diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graph/convert_tensor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graph/convert_tensor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fd1e0511eafef6a9228848ce73a92e9d4682a92a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graph/convert_tensor.mlir @@ -0,0 +1,31 @@ +// RUN: tf-mlir-translate -mlir-to-graph %s -o - | FileCheck %s + +func.func @main() -> (tensor<1x2xf16>, tensor<2xf16>) { + %graph:2 = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<1.0> : tensor<1x2xf16>} : () -> tensor<1x2xf16> loc("const1") +// CHECK: node { +// CHECK-NEXT: name: "const1" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_HALF +// CHECK: half_val: 15360 + %1:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<[1.0, 2.0]> : tensor<2xf16>} : () -> tensor<2xf16> loc("const2") +// CHECK: name: "const2" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_HALF +// CHECK: half_val: 15360 +// CHECK-NEXT: half_val: 16384 + %2:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = bf16, value = dense<[4.900000e+01, 8.200000e+02]> : tensor<2xbf16>} : () -> tensor loc("const3") +// CHECK: name: "const3" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_BFLOAT16 +// CHECK: half_val: 16964 +// CHECK: half_val: 17485 + %3:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = bf16, value = dense<0.000000e+00> : tensor} : () -> tensor loc("const4") +// CHECK: name: "const4" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_BFLOAT16 +// CHECK-NOT: half_val: 0 + tf_executor.fetch %0#0, %1#0 : tensor<1x2xf16>, tensor<2xf16> + } + func.return %graph#0, %graph#1 : tensor<1x2xf16>, tensor<2xf16> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..85f568d34f188cb2147f2a2b7decdb995d3c64dc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/BUILD @@ -0,0 +1,24 @@ +load("//tensorflow:tensorflow.default.bzl", "filegroup") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +licenses(["notice"]) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "@llvm-project//mlir:run_lit.sh", + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-mlir-translate", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/aliasing_arg_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/aliasing_arg_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..465cd75b3dcef7686ce1c48342b8ac7b32fb39e8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/aliasing_arg_attr.mlir @@ -0,0 +1,16 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -tf-graph-as-function -o - | FileCheck %s + +// Verify tf.aliasing_output attributes on args are dropped during export. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 121 : i32}} { + func.func @main(%arg0: tensor<*xf32> {tf.aliasing_output = 0 : i64}, %arg1: tensor<2x4x6x8xi32>) -> (tensor<*xf32>, tensor<2x4x6x8xi32> {tf.device = "/CPU:1"}) + attributes {tf.entry_function = {inputs = "args_0,args_1", outputs = "rets_0,rets_1"}} { + %0:2 = tf_executor.graph { + %1:3 = tf_executor.island wraps "tf.IdentityN"(%arg0, %arg1) {T = ["tfdtype$DT_FLOAT", "tfdtype$DT_INT32"], device = "", name = "identity"} : (tensor<*xf32>, tensor<2x4x6x8xi32>) -> (tensor<*xf32>, tensor<2x4x6x8xi32>) + tf_executor.fetch %1#0, %1#1 : tensor<*xf32>, tensor<2x4x6x8xi32> + } + func.return %0#0, %0#1 : tensor<*xf32>, tensor<2x4x6x8xi32> + } +} + +// CHECK-NOT: aliasing_output diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir new file mode 100644 index 0000000000000000000000000000000000000000..52d20a265380130992f804de7e10dc7b35ab7395 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir @@ -0,0 +1,38 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 486 : i32}} { + func.func @main() { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Case"(%outputs_0, %outputs) {Tin = [i32], Tout = [i32], _lower_using_switch_merge = true, _read_only_resource_inputs = [], branches = [@indexed_case_branch0_40, @indexed_case_branch1_50], device = "", is_stateless = true, output_shapes = [#tf_type.shape<>]} : (tensor, tensor) -> tensor<*xi32> loc("stateless_case") + %outputs_4, %control_5 = tf_executor.island wraps "tf.Identity"(%outputs_2) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.Case"(%outputs_0, %outputs) {Tin = [i32], Tout = [i32], _lower_using_switch_merge = true, _read_only_resource_inputs = [], branches = [@indexed_case_branch0_40, @indexed_case_branch1_50], device = "", is_stateless = false, output_shapes = [#tf_type.shape<>]} : (tensor, tensor) -> tensor<*xi32> loc("regular_case") + tf_executor.fetch + } + func.return + } + + func.func private @indexed_case_branch0_40(%arg0: tensor) -> tensor<*xi32> { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.AddV2"(%arg0, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi32> + tf_executor.fetch %outputs_0 : tensor<*xi32> + } + func.return %0 : tensor<*xi32> + } + + func.func private @indexed_case_branch1_50(%arg0: tensor) -> tensor<*xi32> { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<2> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.AddV2"(%arg0, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi32> + tf_executor.fetch %outputs_0 : tensor<*xi32> + } + func.return %0 : tensor<*xi32> + } +} + +// CHECK: name: "stateless_case" +// CHECK-NEXT: "StatelessCase" +// CHECK: name: "regular_case" +// CHECK-NEXT: "Case" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/convert_tensor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/convert_tensor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6100c337eb3dc15f90d589eb1a36cba2924be239 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/convert_tensor.mlir @@ -0,0 +1,32 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() -> (tensor<1x2xf16>, tensor<2xf16>) { + %graph:2 = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<1.0> : tensor<1x2xf16>} : () -> tensor<1x2xf16> loc("const1") + %1:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<[1.0, 2.0]> : tensor<2xf16>} : () -> tensor<2xf16> loc("const2") + %2:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = bf16, value = dense<[4.900000e+01, 8.200000e+02]> : tensor<2xbf16>} : () -> tensor loc("const3") + %3:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = bf16, value = dense<0.000000e+00> : tensor} : () -> tensor loc("const4") + tf_executor.fetch %0#0, %1#0 : tensor<1x2xf16>, tensor<2xf16> + } + func.return %graph#0, %graph#1 : tensor<1x2xf16>, tensor<2xf16> +} + +// CHECK: node { +// CHECK-NEXT: name: "const1" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_HALF +// CHECK: half_val: 15360 +// CHECK: name: "const2" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_HALF +// CHECK: half_val: 15360 +// CHECK: half_val: 16384 +// CHECK: name: "const3" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_BFLOAT16 +// CHECK: half_val: 16964 +// CHECK: half_val: 17485 +// CHECK: name: "const4" +// CHECK-NEXT: op: "Const" +// CHECK: dtype: DT_BFLOAT16 +// CHECK-NOT: half_val: 0 diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_shape_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_shape_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2a3b2560b6fe5032120ecfec7c74ca40d2eb8210 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_shape_attr.mlir @@ -0,0 +1,28 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// Check that attributes that define derived shapes are exported. + +// CHECK: op: "PlaceholderWithDefault" +// CHECK: shape +// CHECK: unknown_rank: true +// CHECK: name: "static" +// CHECK: op: "PlaceholderWithDefault" +// CHECK: shape { +// CHECK-NEXT: } +// CHECK: name: "static_10" +// CHECK: op: "PlaceholderWithDefault" +// CHECK: shape +// CHECK: dim +// CHECK: size: 10 + +func.func @main() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<0> : tensor<10xi32>} : () -> tensor<10xi32> + %1:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<0> : tensor} : () -> tensor + %2:2 = tf_executor.island wraps "tf.PlaceholderWithDefault"(%1#0) {type = i32} : (tensor) -> tensor<*xi32> loc("unranked") + %3:2 = tf_executor.island wraps "tf.PlaceholderWithDefault"(%1#0) {type = i32} : (tensor) -> tensor loc("static") + %4:2 = tf_executor.island wraps "tf.PlaceholderWithDefault"(%0#0) {type = i32} : (tensor<10xi32>) -> tensor<10xi32> loc("static_10") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_size_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_size_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..11057b110323b0ff972c39b6ea9d717df5d9d458 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/derived_size_attr.mlir @@ -0,0 +1,19 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// CHECK: op: "Split" +// CHECK: attr { +// CHECK: key: "num_split" +// CHECK: value { +// CHECK: i: 2 +// CHECK: } +// CHECK: } + +func.func @main() { + tf_executor.graph { + %dim:2 = tf_executor.island wraps "tf.Const"() {dtype = "tftype$DT_INT32", value = dense<0> : tensor} : () -> tensor + %input:2 = tf_executor.island wraps "tf.Const"() {dtype = "tftype$DT_INT32", value = dense<1.0> : tensor<4x6xf32>} : () -> tensor<4x6xf32> + %split:3 = tf_executor.island wraps "tf.Split"(%dim#0, %input#0) : (tensor, tensor<4x6xf32>) -> (tensor<2x6xf32>, tensor<2x6xf32>) + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/device-arg-retval-attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/device-arg-retval-attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0fa3b24ef4ba8ee900364776ebbaf8f32ef9ef28 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/device-arg-retval-attr.mlir @@ -0,0 +1,54 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -tf-graph-as-function -o - | FileCheck %s + +// Verify arg/ret attributes are exported as device assignment for arg/retval +// nodes. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 121 : i32}} { + func.func @main(%arg0: tensor<*xf32> {tf.device = "/CPU:0"}, %arg1: tensor<2x4x6x8xi32>) -> (tensor<*xf32>, tensor<2x4x6x8xi32> {tf.device = "/CPU:1"}) + attributes {tf.entry_function = {inputs = "args_0,args_1", outputs = "rets_0,rets_1"}} { + %0:2 = tf_executor.graph { + %1:3 = tf_executor.island wraps "tf.IdentityN"(%arg0, %arg1) {T = ["tfdtype$DT_FLOAT", "tfdtype$DT_INT32"], device = "", name = "identity"} : (tensor<*xf32>, tensor<2x4x6x8xi32>) -> (tensor<*xf32>, tensor<2x4x6x8xi32>) + tf_executor.fetch %1#0, %1#1 : tensor<*xf32>, tensor<2x4x6x8xi32> + } + func.return %0#0, %0#1 : tensor<*xf32>, tensor<2x4x6x8xi32> + } +} + +// CHECK: node { +// CHECK-NEXT: name: "args_0" +// CHECK-NEXT: op: "_Arg" +// CHECK: device: "/CPU:0" +// CHECK: attr { +// CHECK: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// +// CHECK: node { +// CHECK-NEXT: name: "args_1" +// CHECK-NEXT: op: "_Arg" +// CHECK-NOT: device +// CHECK: attr { +// CHECK: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 1 +// +// CHECK: node { +// CHECK: op: "IdentityN" +// +// CHECK: node { +// CHECK-NEXT: name: "rets_0" +// CHECK-NEXT: op: "_Retval" +// CHECK-NOT: device +// CHECK: attr { +// CHECK: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// +// CHECK: node { +// CHECK-NEXT: name: "rets_1" +// CHECK-NEXT: op: "_Retval" +// CHECK: device: "/CPU:1" +// CHECK: attr { +// CHECK: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 1 diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/export_main_to_flib.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/export_main_to_flib.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8d79853f5576c8fedffb645d9694d2be595ec282 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/export_main_to_flib.mlir @@ -0,0 +1,20 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef -tf-export-entry-func-to-flib %s -o - 2>&1 | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 458 : i32}} { + func.func @main() attributes {tf.entry_function = {inputs = "", outputs = ""}} { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "TPU:0", name = "const", dtype = "tfdtype$DT_INT32", value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + tf_executor.fetch + } + func.return + } +} + +// CHECK-NOT: node + +// CHECK: library +// CHECK-NEXT: function +// CHECK-NEXT: signature +// CHECK-NEXT: name: "main" +// CHECK: node_def +// CHECK: op: "Const" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/fetch_feed_names.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/fetch_feed_names.mlir new file mode 100644 index 0000000000000000000000000000000000000000..513d9f69e48efe00c265202f8699fdac0ac23646 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/fetch_feed_names.mlir @@ -0,0 +1,31 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s | tf-mlir-translate -graphdef-to-mlir + +// Test graphdef export is producing valid GraphDef. The test imports the +// generated GraphDef to make sure the export is valid. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 890 : i32}} { + func.func @main(%arg0: tensor<100x28x28x3xf32> {}) -> (tensor<*xf32> {}) attributes {tf.entry_function = {control_outputs = "", inputs = "a:0", outputs = "conv2d_2/BiasAdd:0"}} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.StatefulPartitionedCall"(%arg0) {_collective_manager_ids = [], _read_only_resource_inputs = [1, 2], config = "", config_proto = "\0A\07\0A\03CPU\10\01\0A\07\0A\03GPU\10\002\02J\008\01\82\01\00", device = "", executor_type = "", f = @__inference_call_440} : (tensor<100x28x28x3xf32>) -> tensor<*xf32> + %outputs_0, %control_1 = tf_executor.island(%control) wraps "tf.Const"() {value = dense<0.000000e+00> : tensor<16xf32>} : () -> tensor<16xf32> + %outputs_2, %control_3 = tf_executor.island(%control_1) wraps "tf.Const"() {value = dense<0.000000e+00> : tensor<5x5x32x16xf32>} : () -> tensor<5x5x32x16xf32> + %outputs_4, %control_5 = tf_executor.island wraps "tf.Conv2D"(%outputs, %outputs_2) {data_format = "NHWC", device = "", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true} : (tensor<*xf32>, tensor<5x5x32x16xf32>) -> tensor<*xf32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.BiasAdd"(%outputs_4, %outputs_0) {data_format = "NHWC", device = ""} : (tensor<*xf32>, tensor<16xf32>) -> tensor<*xf32> + tf_executor.fetch %outputs_6 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> + } + func.func private @__inference_call_440(%arg0: tensor {tf._user_specified_name = "inputs"}) -> tensor<*xf32> attributes {tf._input_shapes = [#tf_type.shape], tf.signature.is_stateful} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {value = dense<0.000000e+00> : tensor<32xf32>} : () -> tensor<32xf32> + %outputs_0, %control_1 = tf_executor.island(%control) wraps "tf.Const"() {value = dense<0.000000e+00> : tensor<5x5x3x32xf32>} : () -> tensor<5x5x3x32xf32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.Conv2D"(%arg0, %outputs_0) {data_format = "NHWC", device = "", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true} : (tensor, tensor<5x5x3x32xf32>) -> tensor<*xf32> + %outputs_4, %control_5 = tf_executor.island wraps "tf.BiasAdd"(%outputs_2, %outputs) {data_format = "NHWC", device = ""} : (tensor<*xf32>, tensor<32xf32>) -> tensor<*xf32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.Identity"(%outputs_4) {device = ""} : (tensor<*xf32>) -> tensor<*xf32> + tf_executor.fetch %outputs_6 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> + } +} + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..176b5f831bb0299e65e27a8fc550cdd6ce3cfebd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_attr.mlir @@ -0,0 +1,43 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -tf-export-original-func-name | tf-mlir-translate -graphdef-to-mlir | tf-mlir-translate -mlir-to-graphdef -tf-export-original-func-name | FileCheck %s + +// Tests #tf_type.func attributes are exported as AttrValue.NameAttrList attributes +// with its attr field populated with nested attributes. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 458 : i32}} { + func.func @main() { + tf_executor.graph { + %control = tf_executor.island wraps "tf.NoOp"() {_f = #tf_type.func<@callee, {attr2 = true, attr3 = 8.0 : f32}>} : () -> () + %control_1 = tf_executor.island(%control) wraps "tf.LegacyCall"() {f = @callee} : () -> () + tf_executor.fetch + } + func.return + } + func.func @callee() attributes {tf._original_func_name = "original_callee"} { + tf_executor.graph { + tf_executor.fetch + } + func.return + } +} + +// CHECK: op: "NoOp" +// CHECK-NEXT: attr +// CHECK-NEXT: key: "_f" +// CHECK-NEXT: value +// CHECK-NEXT: func +// CHECK-NEXT: name: "original_callee" +// CHECK-NEXT: attr +// CHECK-NEXT: key: "attr2" +// CHECK-NEXT: value +// CHECK-NEXT: b: true +// CHECK: attr +// CHECK-NEXT: key: "attr3" +// CHECK-NEXT: value +// CHECK-NEXT: f: 8 + +// CHECK: op: "original_callee" + +// CHECK: library +// CHECK-NEXT: function +// CHECK-NEXT: signature +// CHECK-NEXT: name: "original_callee" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_list_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_list_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d579a950ee90f99e66c8cc78ef61df616f8b4474 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/func_list_attr.mlir @@ -0,0 +1,67 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { +// CHECK: node { +// CHECK-NEXT: name: "predicate" +// CHECK-NEXT: op: "Const" +// CHECK-NEXT: attr { +// CHECK: key: "dtype" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "value" +// CHECK-NEXT: value { +// CHECK-NEXT: tensor { +// CHECK-NEXT: dtype: DT_INT32 +// CHECK-NEXT: tensor_shape { +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK: } + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<0> : tensor} : () -> tensor loc("predicate") + +// CHECK: node { +// CHECK-NEXT: name: "Case" +// CHECK-NEXT: op: "Case" +// CHECK-NEXT: input: "predicate" +// CHECK: attr { +// CHECK: key: "branches" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: func { +// CHECK-NEXT: name: "foo" +// CHECK-NEXT: } +// CHECK-NEXT: func { +// CHECK-NEXT: name: "bar" +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK: } + %1:2 = tf_executor.island wraps "tf.Case"(%0#0) {Tin = [], Tout = ["tfdtype$DT_FLOAT"], branches = [@foo, @bar], device = "", output_shapes = [], is_stateless = false} : (tensor) -> tensor<*xf32> loc("Case") + tf_executor.fetch + } + func.return +} + +// CHECK-DAG: name: "foo" +func.func @foo() -> tensor<10xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", value = dense<1.000000e+00> : tensor<10xf32>} : () -> tensor<10xf32> loc("const_1") + tf_executor.fetch %1#0 : tensor<10xf32> + } + func.return %0 : tensor<10xf32> +} + +// CHECK-DAG: name: "bar" +func.func @bar() -> tensor<10xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", value = dense<2.000000e+00> : tensor<10xf32>} : () -> tensor<10xf32> loc("const_2") + tf_executor.fetch %1#0 : tensor<10xf32> + } + func.return %0 : tensor<10xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-control-ret.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-control-ret.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5c5ec1a2bf3d5ce637d264c3dd2db151dd596f52 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-control-ret.mlir @@ -0,0 +1,26 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 29 : i32}} { + func.func @main() { + tf_executor.graph { + %0 = tf_executor.island wraps "tf.PartitionedCall"() {Tin = [], Tout = [], config = "", config_proto = "", device = "", executor_type = "", f = @foo, name = "Call_foo"} : () -> () + tf_executor.fetch + } + func.return + } + func.func @foo() { + tf_executor.graph { + %0:2 = tf_executor.island { + %1 = "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<5> : tensor} : () -> tensor loc("control_const") + tf_executor.yield %1 : tensor + } + // CHECK: control_output: "control_const" + // CHECK: control_ret { + // CHECK-NEXT: key: "control_const" + // CHECK-NEXT: value: "control_const" + // CHECK-NEXT: } + tf_executor.fetch %0#1 : !tf_executor.control + } + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-order.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-order.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5df2f7c2e7e497543b3d1d8a37babde58f99068e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-order.mlir @@ -0,0 +1,85 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + + +func.func @main() { + tf_executor.graph { + // CHECK: node { + // CHECK-NEXT: name: "tf.PartitionedCall" + // CHECK-NEXT: op: "PartitionedCall" + // CHECK: func { + // CHECK: name: "foo" + // CHECK: } + // CHECK: } + %0 = tf_executor.island wraps "tf.PartitionedCall"() {Tin = [], Tout = [], config = "", config_proto = "", device = "", executor_type = "", f = @foo, name = "Call_foo"} : () -> () + tf_executor.fetch + } + func.return +} + +// CHECK: library { +// CHECK: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "bar" +// CHECK-NEXT: } +// CHECK: node_def { +// CHECK-NEXT: name: "tf.Const" +// CHECK-NEXT: op: "Const" +// CHECK-NEXT: attr { +// CHECK: key: "dtype" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "value" +// CHECK-NEXT: value { +// CHECK-NEXT: tensor { +// CHECK-NEXT: dtype: DT_INT32 +// CHECK-NEXT: tensor_shape { +// CHECK-NEXT: } +// CHECK-NEXT: int_val: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK: } +// CHECK: node_def { +// CHECK-NEXT: name: "tf.Empty" +// CHECK-NEXT: op: "Empty" +// CHECK-NEXT: input: "tf.Const:output:0" +// CHECK-NEXT: attr { +// CHECK: key: "dtype" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_FLOAT +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK: } +// CHECK-NEXT: } +func.func @bar() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", name = "tf.Const", value = dense<1> : tensor} : () -> tensor + %1:2 = tf_executor.island wraps "tf.Empty"(%0#0) {dtype = "tfdtype$DT_FLOAT", name = "tf.Empty"} : (tensor) -> tensor<*xf32> + tf_executor.fetch + } + func.return +} + +// CHECK: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "foo" +// CHECK-NEXT: } +// CHECK-NEXT: node_def { +// CHECK-NEXT: name: "tf.PartitionedCall" +// CHECK-NEXT: op: "PartitionedCall" +// CHECK: func { +// CHECK: name: "bar" +// CHECK: } +// CHECK: } +// CHECK-NEXT: } +// CHECK: } +func.func @foo() { + tf_executor.graph { + %0 = tf_executor.island wraps "tf.PartitionedCall"() {Tin = [], Tout = [], config = "", config_proto = "", device = "", executor_type = "", f = @bar, name = "Call_bar"} : () -> () + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args-handle-info.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args-handle-info.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c37cf5cd237fccf618e1bb071c60eb022e31ac39 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args-handle-info.mlir @@ -0,0 +1,42 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor<*x!tf_type.resource>>) -> tensor<8x1xf32> { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<8x1xf32> + tf_executor.fetch %outputs : tensor<8x1xf32> + } + func.return %0 : tensor<8x1xf32> +} + +// Check that we generate _handle_dtypes and _handle_shapes for the resource +// argument. + +// CHECK: op: "_Arg" + +// CHECK: key: "_handle_dtypes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: type: DT_FLOAT +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } + +// CHECK: key: "_handle_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 8 +// CHECK-NEXT: } +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } + +// CHECK: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// CHECK-NEXT: } +// CHECK-NEXT: } + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f4b5f859074ab6627df7cd44ad620409728650ef --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/function-resource-args.mlir @@ -0,0 +1,61 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() -> tensor<*x!tf_type.resource> attributes {tf.entry_function = {inputs = "", outputs = "func_call"}} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.VarHandleOp"() {container = "a", device = "/CPU:0", dtype = i64, shape = "tfshape$", shared_name = "x"} : () -> tensor>> loc("x") + %outputs_0, %control_1 = tf_executor.island wraps "tf.LegacyCall"(%outputs, %outputs) {_disable_call_shape_inference = true, f = @test_func_name0} : (tensor>>, tensor>>) -> tensor<*x!tf_type.resource> loc("called") + tf_executor.fetch %outputs_0 : tensor<*x!tf_type.resource> + } + func.return %0 : tensor<*x!tf_type.resource> +} +func.func @test_func_name0(%arg0: tensor<*x!tf_type.resource> {tf._resource_arg_unique_id = 0 : i64}, %arg1: tensor<*x!tf_type.resource> {tf._resource_arg_unique_id = 0 : i64}) -> tensor<*x!tf_type.resource> attributes {tf._disable_call_shape_inference = true} { + %0 = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<*x!tf_type.resource> + } + func.return %0 : tensor<*x!tf_type.resource> +} + +// Check that the `tf._resource_arg_unique_id` argument attributes of +// test_func_name0 are propagated to the function's arg_attr and +// resource_arg_unique_id. + +// CHECK: name: "x" +// CHECK: op: "VarHandleOp" + +// CHECK: name: "func_call" +// CHECK: input: "called" + +// CHECK: library +// CHECK: function +// CHECK: signature +// CHECK: input_arg +// CHECK: type: DT_RESOURCE +// CHECK: input_arg +// CHECK: type: DT_RESOURCE +// CHECK: output_arg +// CHECK: type: DT_RESOURCE +// CHECK: ret + +// Check _resource_arg_unique_id for each argument. Since they alias each other, +// both values are 0. +// CHECK: arg_attr +// CHECK-NEXT: key: 0 +// CHECK-NEXT: value +// CHECK: key: "_resource_arg_unique_id" +// CHECK-NEXT: value +// CHECK-NEXT: i: 0 +// CHECK: arg_attr +// CHECK-NEXT: key: 1 +// CHECK-NEXT: value +// CHECK: key: "_resource_arg_unique_id" +// CHECK-NEXT: value +// CHECK-NEXT: i: 0 + +// Check resource_arg_unique_id for each argument. Since they alias each other, +// both values are 0. +// CHECK: resource_arg_unique_id +// CHECK-NEXT: key: 0 +// CHECK-NEXT: value: 0 +// CHECK: resource_arg_unique_id +// CHECK-NEXT: key: 1 +// CHECK-NEXT: value: 0 diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-if-ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-if-ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..55edfa5fb3fa8d7fd1593b00d0ab128f740c162b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-if-ops.mlir @@ -0,0 +1,65 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor, %arg1: tensor, %arg2: tensor<4xf32>, %arg3: tensor<4xf32>) -> (tensor<4xf32>, tensor<4xf32>) { + %0:2 = tf_executor.graph { + %outputs_2, %control_3 = tf_executor.island wraps "tf.Less"(%arg0, %arg1) : (tensor, tensor) -> tensor + %outputs_4, %control_5 = tf_executor.island wraps "tf.If"(%outputs_2, %arg2, %arg3) {else_branch = @cond_false, is_stateless = false, then_branch = @cond_true} : (tensor, tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> loc("StatefulIf") + %outputs_6, %control_7 = tf_executor.island wraps "tf.If"(%outputs_2, %arg2, %arg3) {else_branch = @cond_false, is_stateless = true, then_branch = @cond_true} : (tensor, tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> loc("StatelessIf") + tf_executor.fetch %outputs_4, %outputs_6 : tensor<4xf32>, tensor<4xf32> + } + func.return %0#0, %0#1 : tensor<4xf32>, tensor<4xf32> +} + +func.func @cond_true(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Add"(%arg0, %arg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + tf_executor.fetch %outputs : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +func.func @cond_false(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Mul"(%arg0, %arg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + tf_executor.fetch %outputs : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// Verify that If op is mapped to TensorFlow StatelessIf op if the is_stateless +// attribute is present and otherwise it is mapped to TensorFlow If op. In both +// cases, the additional attribute should be dropped. + +// CHECK: name: "StatefulIf" +// CHECK-NOT: name: +// CHECK: op: "If" +// CHECK-NOT: is_stateless +// CHECK: attr { +// CHECK: key: "output_shapes" +// CHECK: value { +// CHECK: list { +// CHECK: shape { +// CHECK: dim { +// CHECK: size: 4 +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } + +// CHECK: name: "StatelessIf" +// CHECK-NOT: name: +// CHECK: op: "StatelessIf" +// CHECK-NOT: is_stateless +// CHECK: attr { +// CHECK: key: "output_shapes" +// CHECK: value { +// CHECK: list { +// CHECK: shape { +// CHECK: dim { +// CHECK: size: 4 +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-while-ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-while-ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..83c017ab3fa604b8f5c54db08565f75f936a105b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/functional-while-ops.mlir @@ -0,0 +1,89 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor, %arg1: tensor<5xf32>) -> (tensor<5xf32>, tensor<5xf32>, tensor<5xf32>) { + %0:3 = tf_executor.graph { + %outputs_2:2, %control_3 = tf_executor.island wraps "tf.While"(%arg0, %arg1) {body = @body, cond = @cond, is_stateless = false} : (tensor, tensor<5xf32>) -> (tensor, tensor<5xf32>) loc("StatefulWhile") + %outputs_4:2, %control_5 = tf_executor.island wraps "tf.While"(%arg0, %arg1) {body = @body, cond = @cond, is_stateless = true} : (tensor, tensor<5xf32>) -> (tensor, tensor<5xf32>) loc("StatelessWhile") + %outputs_6:2, %control_7 = tf_executor.island wraps "tf.While"(%arg0, %arg1) {body = @body, cond = @cond, is_stateless = false, shape_invariant} : (tensor, tensor<5xf32>) -> (tensor, tensor<5xf32>) loc("WhileWithOutputShapes") + tf_executor.fetch %outputs_2#1, %outputs_4#1, %outputs_6#1 : tensor<5xf32>, tensor<5xf32>, tensor<5xf32> + } + func.return %0#0, %0#1, %0#2 : tensor<5xf32>, tensor<5xf32>, tensor<5xf32> +} + +func.func @cond(%arg0: tensor<*xi32>, %arg1: tensor<*xf32>) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Greater"(%arg0, %outputs) : (tensor<*xi32>, tensor) -> tensor + tf_executor.fetch %outputs_0 : tensor + } + func.return %0 : tensor +} + +func.func @body(%arg0: tensor<*xi32>, %arg1: tensor<*xf32>) -> (tensor<*xi32>, tensor<*xf32>) { + %0:2 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Sub"(%arg0, %outputs) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.Add"(%arg1, %arg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + tf_executor.fetch %outputs_0, %outputs_2 : tensor<*xi32>, tensor<*xf32> + } + func.return %0#0, %0#1 : tensor<*xi32>, tensor<*xf32> +} + +// Verify that While op is mapped to TensorFlow StatelessWhile op if the +// is_stateless attribute is present and otherwise it is mapped to TensorFlow +// While op. In both cases, the additional attribute should be dropped. + +// CHECK: name: "StatefulWhile" +// CHECK-NOT: name: +// CHECK: op: "While" +// CHECK-NOT: is_stateless +// CHECK-NOT: shape_invariant +// CHECK: attr { +// CHECK: key: "output_shapes" +// CHECK: value { +// CHECK: list { +// CHECK: shape { +// CHECK: dim { +// CHECK: size: 5 +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } + + +// CHECK: name: "StatelessWhile" +// CHECK-NOT: name: +// CHECK: op: "StatelessWhile" +// CHECK-NOT: is_stateless +// CHECK-NOT: shape_invariant +// CHECK: attr { +// CHECK: key: "output_shapes" +// CHECK: value { +// CHECK: list { +// CHECK: shape { +// CHECK: dim { +// CHECK: size: 5 +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } + +// CHECK: name: "WhileWithOutputShapes" +// CHECK-NOT: name: +// CHECK: op: "While" +// CHECK-NOT: is_stateless +// CHECK-NOT: shape_invariant +// CHECK: attr { +// CHECK: key: "output_shapes" +// CHECK: value { +// CHECK: list { +// CHECK: shape { +// CHECK: dim { +// CHECK: size: 5 +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/graph-as-function.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/graph-as-function.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8a06aebf4c2923244094179bc59d80af1b1b0fa0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/graph-as-function.mlir @@ -0,0 +1,79 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -tf-graph-as-function -o - | FileCheck %s + +func.func @main(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor<*xf32>, %arg3: tensor<2x4x6x8xi32>) -> (tensor, tensor) +attributes {tf.entry_function = {inputs = "args_0,args_1,args_2,args_3", outputs = "rets_0_RetVal,rets_1_RetVal"}} { + %graph:2 = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", value = dense<0.000000e+00> : tensor} : () -> tensor loc("const") + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) {T = "tfdtype$DT_FLOAT", device = ""} : (tensor) -> tensor loc("identity") + %2:2 = tf_executor.island wraps "tf.StatefulPartitionedCall"(%0#0, %arg1) {Tin = ["tfdtype$DT_FLOAT", "tfdtype$DT_RESOURCE"], Tout = ["tfdtype$DT_FLOAT"], _gradient_op_type = "PartitionedCall-1205", config = "", config_proto = "\0A\07\0A\03GPU\10\00\0A\07\0A\03CPU\10\012\02J\008\01", device = "", executor_type = "", f = @function0} : (tensor, tensor<*x!tf_type.resource>>) -> tensor loc("statefulpartitionedcall") + tf_executor.fetch %1#0, %2#0 : tensor, tensor + } + func.return %graph#0, %graph#1 : tensor, tensor +} + +func.func @function0(%arg0: tensor<*xf32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<*xf32> +attributes {tf.signature.is_stateful} { + %graph = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Identity"(%arg0) {T = "tfdtype$DT_FLOAT", device = ""} : (tensor<*xf32>) -> tensor<*xf32> loc("Identity@function0") + tf_executor.fetch %0#0 : tensor<*xf32> + } + func.return %graph : tensor<*xf32> +} + +// CHECK: node { +// CHECK-NEXT: name: "args_0" +// CHECK-NEXT: op: "_Arg" +// CHECK: i: 0 +// CHECK: node { +// CHECK-NEXT: name: "args_1" +// CHECK-NEXT: op: "_Arg" +// CHECK: i: 1 +// CHECK: node { +// CHECK-NEXT: name: "args_2" +// CHECK-NEXT: op: "_Arg" +// CHECK: i: 2 +// CHECK: node { +// CHECK-NEXT: name: "args_3" +// CHECK-NEXT: op: "_Arg" +// CHECK: i: 3 +// CHECK: node { +// CHECK-NEXT: name: "const" +// CHECK-NEXT: op: "Const" +// CHECK: node { +// CHECK-NEXT: name: "identity" +// CHECK-NEXT: op: "Identity" +// CHECK-NEXT: input: "const" +// CHECK: node { +// CHECK-NEXT: name: "statefulpartitionedcall" +// CHECK-NEXT: op: "StatefulPartitionedCall" +// CHECK-NEXT: input: "const" +// CHECK-NEXT: input: "args_1" +// CHECK: key: "f" +// CHECK-NEXT: value { +// CHECK-NEXT: func { +// CHECK-NEXT: name: "function0" +// CHECK: node { +// CHECK-NEXT: name: "rets_0_RetVal" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "identity" +// CHECK: node { +// CHECK-NEXT: name: "rets_1_RetVal" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "statefulpartitionedcall" +// CHECK: library { +// CHECK-NEXT: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "function0" +// CHECK-NEXT: input_arg { +// CHECK-NEXT: name: "function0" +// CHECK: input_arg { +// CHECK-NEXT: name: "function01" +// CHECK: output_arg { +// CHECK-NEXT: name: "function02" +// CHECK: node_def { +// CHECK-NEXT: name: "[[NAME:[^"]*]]" +// CHECK-NEXT: op: "Identity" +// CHECK-NEXT: input: "function0" +// CHECK: ret { +// CHECK-NEXT: key: "function02" +// CHECK-NEXT: value: "[[NAME]]:output:0" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/infer_derived_attribute.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/infer_derived_attribute.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6e9b83484d9bcd099b5494bc4852100f6cd65fb7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/infer_derived_attribute.mlir @@ -0,0 +1,26 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + // The operation does not have any attributes, but TensorFlow OpDef expects + // a `dtype` to be added on the NodeDef. We verify that we correctly use the + // DerivedAttr to populate the NodeDef. + // CHECK: key: "dtype" + // CHECK-NEXT: value { + // CHECK-NEXT: type: DT_FLOAT + // CHECK: float_val: 2 + // CHECK: key: "dtype" + // CHECK-NEXT: value { + // CHECK-NEXT: type: DT_FLOAT + // CHECK: float_val: 3 + // CHECK: key: "dtype" + // CHECK-NEXT: value { + // CHECK-NEXT: type: DT_DOUBLE + // CHECK: double_val: 4 + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {value = dense<2.000000e+00> : tensor} : () -> tensor + %1:2 = tf_executor.island(%0#1) wraps "tf.Const"() {value = dense<3.000000e+00> : tensor} : () -> tensor + %2:2 = tf_executor.island(%1#1) wraps "tf.Const"() {value = dense<4.000000e+00> : tensor} : () -> tensor + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/invalid_input.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/invalid_input.mlir new file mode 100644 index 0000000000000000000000000000000000000000..668a0b4e73c1d61522cd25e2366c8ccf048337a0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/invalid_input.mlir @@ -0,0 +1,96 @@ +// RUN: not tf-mlir-translate -split-input-file -mlir-to-graphdef %s -o - 2>&1 | FileCheck %s + +// Tests function with multiple blocks. + +func.func @main() { + ^bb: + cf.br ^bb1 + ^bb1: + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: only single block functions are supported + +// ----- + +// Tests invalid functions for exporting to Graph/GraphDef. + +func.func @main() { + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: first op in function is not a tf_executor.graph + +// ----- + +func.func @main() { + tf_executor.graph { + tf_executor.fetch + } + tf_executor.graph { + tf_executor.fetch + } + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: function does not only contain a single tf_executor.graph + +// ----- + +func.func @main() { + tf_executor.graph { + %0 = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: tf_executor.island must perfectly wrap a single op + +// ----- + +func.func @main() { + tf_executor.graph { + %0 = tf_executor.island { + %1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: tf_executor.island must perfectly wrap a single op + +// ----- + +func.func @main() { + tf_executor.graph { + %0 = tf_executor.island { + %1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: tf_executor.island must perfectly wrap a single op + +// ----- + +func.func @main(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + tf_executor.yield %1#1, %1#0 : tensor, tensor + } + tf_executor.fetch + } + func.return +} + +// CHECK: functions must be of a single Graph with single op Islands: tf_executor.island must perfectly wrap a single op diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/legalized_name.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/legalized_name.mlir new file mode 100644 index 0000000000000000000000000000000000000000..09f5671a4c16fa6937102a6b423f08d368464246 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/legalized_name.mlir @@ -0,0 +1,22 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + // CHECK: name: ".foo" + %0:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<0> : tensor} : () -> (tensor) loc("^foo") + // CHECK: name: "fo.o" + %1:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<1> : tensor} : () -> (tensor) loc("fo{o") + // CHECK: name: "foo" + %2:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<2> : tensor} : () -> (tensor) loc("foo@1") + // CHECK: name: "ba.r" + %3:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<2> : tensor} : () -> (tensor) loc("ba r") + // CHECK: name: "2" + %4:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<3> : tensor} : () -> (tensor) loc("2") + // CHECK: name: "_3" + %5:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<3> : tensor} : () -> (tensor) loc("_3") + // CHECK: name: "foo_" + %6:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<3> : tensor} : () -> (tensor) loc("foo_") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/missing-main.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/missing-main.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fde0428f26b10260c0faa115ba8bbfeb88572b64 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/missing-main.mlir @@ -0,0 +1,11 @@ +// RUN: not tf-mlir-translate -mlir-to-graphdef %s -o - 2>&1 | FileCheck %s + +// CHECK: entry function `main` must be present + +func.func @const() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "TPU:0", name = "const", dtype = "tfdtype$DT_INT32", value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/noop.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/noop.mlir new file mode 100644 index 0000000000000000000000000000000000000000..1690d149e74e250ab2b32bbad01413d07ed8200d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/noop.mlir @@ -0,0 +1,16 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + tf_executor.island wraps "tf.NoOp"() {} : () -> () loc("noop") + tf_executor.fetch + } + func.return +} + +// CHECK: node { +// CHECK-NEXT: name: "noop" +// CHECK-NEXT: op: "NoOp" +// CHECK-NEXT: experimental_debug_info { +// CHECK-NEXT: } +// CHECK-NEXT: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/optional_symbol_ref.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/optional_symbol_ref.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3030946de2b6e1b81c607012a764c55b2a1829e2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/optional_symbol_ref.mlir @@ -0,0 +1,20 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s | tf-mlir-translate -graphdef-to-mlir | tf-mlir-translate -mlir-to-graphdef | FileCheck %s + +// Verifies that optional symbol ref attributes that aren't optional in TensorFlow are handled by setting the value to an empty string. +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 458 : i32}} { + func.func @main() { + tf_executor.graph { + %control = tf_executor.island wraps "tf.XlaHostCompute"() {_xla_original_oc_node_name = "Add", _xla_token_input_nodes = ["_xla_token_arg_node"], ancestors = [], cost_estimate_ns = 1024 : i64, key = "host_compute_channel_1_retvals", send_key = "", recv_key = "", shapes = [], tpu_core = 0 : i64} : () -> () + tf_executor.fetch + } + func.return + } +} + +// CHECK: op: "XlaHostCompute" + +// CHECK: attr { +// CHECK: key: "shape_inference_graph" +// CHECK: value { +// CHECK: func { +// CHECK-NEXT } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/output-shapes-attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/output-shapes-attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..823a2b3feb0d6298a3827ec190038a026e2421df --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/output-shapes-attr.mlir @@ -0,0 +1,98 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor<10xi32>) -> tensor<10xi32> +attributes {tf.entry_function = {inputs = "input0", outputs = "output0"}} { + %graph = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<10xi32> + } + func.return %graph : tensor<10xi32> +} + +// CHECK: node { +// CHECK-NEXT: name: "input0" +// CHECK-NEXT: op: "_Arg" +// CHECK: key: "T" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK: key: "_output_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 10 +// CHECK: name: "output0" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "input0" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "T" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// CHECK-NEXT: } +// CHECK-NEXT: } + +func.func private @simple_callee(%arg0: tensor<2xi32>) -> tensor<2xi32> { + %graph = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<2xi32> + } + func.return %graph#0 : tensor<2xi32> +} + +// Test with a TPUPartitionedCallOp that is not inlined. +func.func @dont_inline_tpu_partitioned_call(%arg0: tensor<2xi32> {tf._user_specified_name = "inputs_0"}, %arg1: tensor<2xi32> {tf._user_specified_name = "inputs_1"}) -> tensor<2xi32> { + %graph = tf_executor.graph { + %result:2 = tf_executor.island wraps "tf.TPUPartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @simple_callee} : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + tf_executor.fetch %result#0 : tensor<2xi32> + } + func.return %graph : tensor<2xi32> +} + +// CHECK: library { +// CHECK-NEXT: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "simple_callee" +// CHECK: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "dont_inline_tpu_partitioned_call" +// CHECK-NEXT: input_arg { +// CHECK-NEXT: name: "dont_inline_tpu_partitioned_call" +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK-NEXT: input_arg { +// CHECK-NEXT: name: "dont_inline_tpu_partitioned_call1" +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK: arg_attr { +// CHECK-NEXT: key: 0 +// CHECK-NEXT: value { +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "_output_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 2 +// CHECK: attr { +// CHECK-NEXT: key: "_user_specified_name" +// CHECK-NEXT: value { +// CHECK-NEXT: s: "inputs_0" +// CHECK: arg_attr { +// CHECK-NEXT: key: 1 +// CHECK-NEXT: value { +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "_output_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 2 +// CHECK: attr { +// CHECK-NEXT: key: "_user_specified_name" +// CHECK-NEXT: value { +// CHECK-NEXT: s: "inputs_1" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example.mlir new file mode 100644 index 0000000000000000000000000000000000000000..377645b1f7b6f5028dbd0726b0171ab84a62dd5c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example.mlir @@ -0,0 +1,61 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// CHECK: name: "tf.ParseExample" +// CHECK-NEXT: op: "ParseExample" +// CHECK-NEXT: input: "tf.Const{{_.*_3}}" +// CHECK-NEXT: input: "tf.Const" +// CHECK-NEXT: input: "tf.Const{{_.*_1}}" +// CHECK-NEXT: input: "tf.Const{{_.*_2}}" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "Ndense" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "Nsparse" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "Tdense" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: type: DT_INT64 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK: key: "dense_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "sparse_types" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 413 : i32}} { + func.func @main() -> tensor<*xi64> attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "result"}} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<"value"> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<"value"> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<-1> : tensor} : () -> tensor + %outputs_4, %control_5 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %outputs_6, %control_7 = tf_executor.island wraps "tf.ParseExample"(%outputs_4, %outputs, %outputs_0, %outputs_2) {dense_shapes = [#tf_type.shape<1>], device = "", operandSegmentSizes = array, resultSegmentSizes = array} : (tensor<1x!tf_type.string>, tensor<1x!tf_type.string>, tensor, tensor) -> tensor<*xi64> + tf_executor.fetch %outputs_6 : tensor<*xi64> + } + func.return %0 : tensor<*xi64> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example_v2.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example_v2.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a2c0e1d6f249d049ab1e817c6e4dba06a71d3d8a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/parse_example_v2.mlir @@ -0,0 +1,83 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 175 : i32}} { + func.func @main(%arg0: tensor<32x!tf_type.string>) -> (tensor) attributes {tf.entry_function = {inputs = "input0", outputs = "ParseExample/ParseExampleV2"}} { + + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", dtype = f32, value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", dtype = f32, value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "", dtype = !tf_type.string, value = dense<""> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + %outputs_4, %control_5 = tf_executor.island wraps "tf.Const"() {device = "", dtype = !tf_type.string, value = dense<""> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %outputs_6, %control_7 = tf_executor.island wraps "tf.Const"() {device = "", dtype = !tf_type.string, value = dense<""> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %outputs_8, %control_9 = tf_executor.island wraps "tf.Const"() {device = "", dtype = !tf_type.string, value = dense<""> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + + %outputs_10:8, %control_11 = tf_executor.island wraps "tf.ParseExampleV2"(%arg0, %outputs_4, %outputs_8, %outputs_2, %outputs_6, %outputs, %outputs_0) {Tdense = ["tfdtype$DT_FLOAT", "tfdtype$DT_FLOAT"], dense_shapes = [#tf_type.shape<>, #tf_type.shape<>], device = "", num_sparse = 2 : i64, ragged_split_types = [], ragged_value_types = [], resultSegmentSizes = array, sparse_types = ["tfdtype$DT_STRING", "tfdtype$DT_INT64"]} : (tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0xf32>, tensor<0xf32>) -> (tensor, tensor, tensor, tensor, tensor<2xi64>, tensor<2xi64>, tensor<32xf32>, tensor<32xf32>) loc("ParseExample") + // CHECK: name: "ParseExample" + // CHECK-NEXT: op: "ParseExampleV2" + // CHECK-NEXT: input: "input0" + // CHECK-NEXT: input: "tf.Const{{_.*_3}}" + // CHECK-NEXT: input: "tf.Const{{_.*_5}}" + // CHECK-NEXT: input: "tf.Const{{_.*_2}}" + // CHECK-NEXT: input: "tf.Const{{_.*_4}}" + // CHECK-NEXT: input: "tf.Const" + // CHECK-NEXT: input: "tf.Const{{_.*_1}}" + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "Tdense" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: type: DT_FLOAT + // CHECK-NEXT: type: DT_FLOAT + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK: key: "dense_shapes" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: shape { + // CHECK-NEXT: } + // CHECK-NEXT: shape { + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "num_sparse" + // CHECK-NEXT: value { + // CHECK-NEXT: i: 2 + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "ragged_split_types" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "ragged_value_types" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "sparse_types" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: type: DT_STRING + // CHECK-NEXT: type: DT_INT64 + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: } + + tf_executor.fetch %outputs_10#0 : tensor + } + func.return %0#0 : tensor + // CHECK: name: "ParseExample/ParseExampleV2" + // CHECK-NEXT: op: "_Retval" + // CHECK-NEXT: input: "ParseExample" + + } +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/preserve-entry-func-names.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/preserve-entry-func-names.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8e31091744e1ce8196a237d096e72e518d2c4f27 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/preserve-entry-func-names.mlir @@ -0,0 +1,28 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> +attributes {tf.entry_function = {inputs = "foo,bar", outputs = "Add"}} { + %graph = tf_executor.graph { + // This node would be renamed to bar1 [note: if imported from TF graphdef this would not be possible] + %2:2 = tf_executor.island wraps "tf.Identity"(%arg1) {device = "", dtype = "tfdtype$DT_INT32"} : (tensor<10xi32>) -> tensor<10xi32> loc ("bar") + // The following node would be renamed to bar2 + %3:2 = tf_executor.island wraps "tf.Identity"(%2) {device = "", dtype = "tfdtype$DT_INT32"} : (tensor<10xi32>) -> tensor<10xi32> loc ("bar") + %4:2 = tf_executor.island wraps "tf.Add"(%arg0, %3) {T = "tfdtype$DT_INT32", device = ""} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> loc("Add") + tf_executor.fetch %4#0 : tensor<10xi32> + } + func.return %graph : tensor<10xi32> +} + +// CHECK: name: "foo" +// CHECK-NEXT: op: "_Arg" +// CHECK: name: "bar" +// CHECK-NEXT: op: "_Arg" +// CHECK: name: "[[BAR_ID_0:.*]]" +// CHECK-NEXT: op: "Identity" +// CHECK-NEXT: input: "bar" +// CHECK: name: "[[BAR_ID_1:.*]]" +// CHECK-NEXT: op: "Identity" +// CHECK-NEXT: input: "[[BAR_ID_0]]" +// CHECK: name: "Add" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "Add{{_.*_1}}" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-type-attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-type-attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2b48ed502b3983e67729924b7701f9946b9449fc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-type-attr.mlir @@ -0,0 +1,20 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// Verify that attribute T is not set as ref type. +// CHECK-LABEL: name: "Ref_Variable" +// CHECK-LABEL: name: "foo" +// CHECK: attr { +// CHECK-NEXT: key: "T" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32{{$}} +// CHECK-NEXT: } +// CHECK-NEXT: } + +func.func @main() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.VariableV2"() {dtype = "tfdtype$DT_INT32", value = dense<2> : tensor, shape = #tf_type.shape<2>, container = "", shared_name = ""} : () -> tensor loc("Ref_Variable") + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor<*x!tf_type.int32ref> loc("foo") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-while-loop.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-while-loop.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d0a4c00c13ab6e9bc3a08c50de038ca5c1dabab2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/ref-while-loop.mlir @@ -0,0 +1,26 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// Verify the ops generated when Ref type is used in a while loop. +func.func @main() { + // CHECK: op: "RefEnter" + // CHECK: op: "RefMerge" + // CHECK: op: "RefSwitch" + // CHECK: op: "RefExit" + // CHECK: op: "RefNextIteration" + tf_executor.graph { + %0:3 = tf_executor.NextIteration.Source : tensor<*x!tf_type.int32ref> {device = "", T = "tfdtype$DT_INT32"} loc("while/NextIteration") + %1:2 = tf_executor.island wraps "tf.VariableV2"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<0> : tensor, shape = #tf_type.shape<0>, container = "", shared_name = ""} : () -> tensor loc("Ref_Variable") + %2:2 = tf_executor.Enter %1#0 frame "while/while_context" parallel_iterations 10 : (tensor) -> (tensor<*x!tf_type.int32ref>, !tf_executor.control) {device = "", T = "tfdtype$DT_INT32"} loc("while/Enter") + %3:3 = tf_executor.Merge %2#0, %0#0 : tensor<*x!tf_type.int32ref> {device = "", N = 2, T = "tfdtype$DT_INT32"} loc("while/Merge") + %4:2 = tf_executor.island(%3#2) wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<10> : tensor} : () -> tensor loc("while/Less/y") + %5:2 = tf_executor.island(%3#2) wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_BOOL", value = dense : tensor} : () -> tensor loc("while/Less") + %6:2 = tf_executor.LoopCond %5#0 : (tensor) -> (tensor, !tf_executor.control) {device = ""} loc("while/LoopCond") + %7:3 = tf_executor.Switch %3#0, %6#0 : (tensor<*x!tf_type.int32ref>, tensor) -> (tensor<*x!tf_type.int32ref>, tensor<*x!tf_type.int32ref>, !tf_executor.control) {device = "", T = "tfdtype$DT_INT32", _class = ["loc:@while/Merge"]} loc("while/Switch") + %8:2 = tf_executor.Exit %7#1 : tensor<*x!tf_type.int32ref> {device = "", T = "tfdtype$DT_INT32"} loc("while/Exit") + %10:2 = tf_executor.island(%7#2) wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<1> : tensor} : () -> tensor loc("while/Add/y") + %11:2 = tf_executor.island wraps "tf.AssignAdd"(%7#0, %10#0) {device = "", T = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.int32ref>, tensor) -> tensor<*x!tf_type.int32ref> loc("while/Add") + tf_executor.NextIteration.Sink [%0#1] %11#0 : tensor<*x!tf_type.int32ref> {device = "", T = "tfdtype$DT_INT32"} loc("while/NextIteration") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/shape_list_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/shape_list_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e2216d5b13759f57d5216f8a37ebc60430effc60 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/shape_list_attr.mlir @@ -0,0 +1,35 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + + +// CHECK: attr { +// CHECK: key: "dtypes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: type: DT_FLOAT +// CHECK-NEXT: type: DT_INT16 + +// CHECK: attr { +// CHECK-NEXT: key: "shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 3 +// CHECK: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 4 +// CHECK-NEXT: } +// CHECK-NEXT: dim { +// CHECK-NEXT: size: -1 +// CHECK: shape { +// CHECK-NEXT: unknown_rank: true + + +func.func @main() { + tf_executor.graph { + %0:4 = tf_executor.island wraps "tf.InfeedDequeueTuple"() : () -> (tensor<3xi32>, tensor<4x?xf32>, tensor<*xi16>) + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/simple.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/simple.mlir new file mode 100644 index 0000000000000000000000000000000000000000..1e0d2aad9952444c780b0fdb2ef0decfccb64c20 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/simple.mlir @@ -0,0 +1,29 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + // CHECK: node { + // CHECK-NEXT: name: "Empty/shape" + // CHECK-NEXT: op: "Const" + // CHECK: attr { + // CHECK: key: "dtype" + // CHECK-NEXT: value { + // CHECK-NEXT: type: DT_INT32 + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: attr { + // CHECK-NEXT: key: "value" + // CHECK-NEXT: value { + // CHECK-NEXT: tensor { + // CHECK-NEXT: dtype: DT_INT32 + // CHECK-NEXT: tensor_shape { + // CHECK-NEXT: dim { + // CHECK-NEXT: size: 2 + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK-NEXT: tensor_content: "\200\000\000\000\200\000\000\000" + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", dtype = "tfdtype$DT_INT32", value = #tf_type : tensor<2xi32>} : () -> tensor<2xi32> loc("Empty/shape") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/stringescape.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/stringescape.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f671ffc96ccde56781aad2352c039763243473e4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/stringescape.mlir @@ -0,0 +1,18 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { +^bb0: + // CHECK: key: "listvalue" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: s: " \n" + // CHECK-NEXT: } + // CHECK: key: "value" + // CHECK-NEXT: value { + // CHECK-NEXT: s: " 0\n\000\000" + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Placeholder"() {name = "dummy", dtype = "tfdtype$DT_INT32", value = "\200\n\00\00", listvalue = ["\20\0A"]} : () -> tensor<2xi32> + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/switchn.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/switchn.mlir new file mode 100644 index 0000000000000000000000000000000000000000..608c39efefd6e683e55e4192dfeb83529dc87db6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/switchn.mlir @@ -0,0 +1,59 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +"builtin.module"() ({ + "func.func"() ({ + "tf_executor.graph"() ({ + %outputs, %control = "tf_executor.island"() ({ + %0 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + "tf_executor.yield"(%0) : (tensor) -> () + }) : () -> (tensor, !tf_executor.control) + %outputs_0:3, %control_1 = "tf_executor._SwitchN"(%outputs, %outputs) {T = i32, device = "", num_outs = 3 : i64} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, !tf_executor.control) + %outputs_2, %control_3 = "tf_executor.island"() ({ + %0 = "tf.Identity"(%outputs_0#0) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + "tf_executor.yield"(%0) : (tensor<*xi32>) -> () + }) : () -> (tensor<*xi32>, !tf_executor.control) + %outputs_4, %control_5 = "tf_executor.island"(%control_3) ({ + %0 = "tf.Const"() {device = "", value = dense<2.000000e+00> : tensor} : () -> tensor + "tf_executor.yield"(%0) : (tensor) -> () + }) : (!tf_executor.control) -> (tensor, !tf_executor.control) + %outputs_6, %control_7 = "tf_executor.island"() ({ + %0 = "tf.Identity"(%outputs_0#1) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + "tf_executor.yield"(%0) : (tensor<*xi32>) -> () + }) : () -> (tensor<*xi32>, !tf_executor.control) + %outputs_8, %control_9 = "tf_executor.island"(%control_7) ({ + %0 = "tf.Const"() {device = "", value = dense<3.000000e+00> : tensor} : () -> tensor + "tf_executor.yield"(%0) : (tensor) -> () + }) : (!tf_executor.control) -> (tensor, !tf_executor.control) + %outputs_10, %control_11 = "tf_executor.island"() ({ + %0 = "tf.Identity"(%outputs_0#2) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + "tf_executor.yield"(%0) : (tensor<*xi32>) -> () + }) : () -> (tensor<*xi32>, !tf_executor.control) + %outputs_12, %control_13 = "tf_executor.island"(%control_11) ({ + %0 = "tf.Const"() {device = "", value = dense<4.000000e+00> : tensor} : () -> tensor + "tf_executor.yield"(%0) : (tensor) -> () + }) : (!tf_executor.control) -> (tensor, !tf_executor.control) + %outputs_14, %control_15 = "tf_executor.island"() ({ + %0 = "tf.Const"() {device = "", value = dense<1.000000e+00> : tensor} : () -> tensor + "tf_executor.yield"(%0) : (tensor) -> () + }) : () -> (tensor, !tf_executor.control) + %outputs_16:2, %control_17 = "tf_executor._SwitchN"(%outputs_14, %outputs) {T = f32, _class = ["Case/input_0"], device = "", num_outs = 2 : i64} : (tensor, tensor) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) + %outputs_18, %control_19 = "tf_executor.island"() ({ + %0 = "tf.Mul"(%outputs_16#0, %outputs_4) {device = ""} : (tensor<*xf32>, tensor) -> tensor<*xf32> + "tf_executor.yield"(%0) : (tensor<*xf32>) -> () + }) : () -> (tensor<*xf32>, !tf_executor.control) + %outputs_20, %control_21 = "tf_executor.island"() ({ + %0 = "tf.Mul"(%outputs_16#1, %outputs_8) {device = ""} : (tensor<*xf32>, tensor) -> tensor<*xf32> + "tf_executor.yield"(%0) : (tensor<*xf32>) -> () + }) : () -> (tensor<*xf32>, !tf_executor.control) + %output, %value_index, %control_22 = "tf_executor.Merge"(%outputs_18, %outputs_20) {N = 2 : i64, T = f32, device = ""} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xi32>, !tf_executor.control) + %control_23 = "tf_executor.island"() ({ + "tf._Retval"(%output) {T = f32, device = "/job:localhost/replica:0/task:0/device:CPU:0", index = 0 : i64} : (tensor<*xf32>) -> () + "tf_executor.yield"() : () -> () + }) : () -> !tf_executor.control + "tf_executor.fetch"() : () -> () + }) : () -> () + "func.return"() : () -> () + }) {sym_name = "main", function_type = () -> ()} : () -> () +}) {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 126 : i32}} : () -> () + +// CHECK: _SwitchN diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-gradient-attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-gradient-attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..33491807d35376cdefe6e7a9626e490013c5f0fb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-gradient-attr.mlir @@ -0,0 +1,47 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + // CHECK: node { + // CHECK-NEXT: name: "Const" + // CHECK-NEXT: op: "Const" + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", value = dense<2.500000e-01> : tensor} : () -> tensor loc("Const") + + // CHECK: node { + // CHECK-NEXT: name: "tf.PartitionedCall" + // CHECK-NEXT: op: "PartitionedCall" + // CHECK: func { + // CHECK: name: "foo" + // CHECK: } + %1:2 = tf_executor.island wraps "tf.PartitionedCall"(%0) {Tin = [], Tout = [], config = "", config_proto = "", device = "", executor_type = "", f = @foo, name = "Call_foo"} : (tensor) -> tensor<*xf32> + tf_executor.fetch + } + func.return +} + +// CHECK: library { +// CHECK-NEXT: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "foo" +// CHECK: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "foo_grad" +// CHECK: gradient { +// CHECK-NEXT: function_name: "foo" +// CHECK-NEXT: gradient_func: "foo_grad" +// CHECK-NEXT: } +// CHECK-NEXT: } +func.func @foo_grad(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %graph = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %graph : tensor<*xf32> +} + +func.func @foo(%arg0: tensor<*xf32>) -> tensor<*xf32> + attributes {tf.gradient = @foo_grad} { + %graph = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %graph : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-legacy-call.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-legacy-call.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c72ca6fc9c37f123e777eab656d4b28b8cbbbf6f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf-legacy-call.mlir @@ -0,0 +1,36 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Constant", value = dense<0> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.LegacyCall"(%outputs) {_tpu_replicate = "cluster", device = "", f = @foo0} : (tensor) -> tensor + tf_executor.fetch + } + func.return +} +func.func @foo0(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<*xi32> + } + func.return %0 : tensor<*xi32> +} + +// CHECK: node { +// CHECK: name: "tf.LegacyCall" +// CHECK-NEXT: op: "foo0" +// CHECK: attr { +// CHECK-NEXT: key: "_output_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK: attr { +// CHECK-NEXT: key: "_tpu_replicate" +// CHECK-NEXT: value { +// CHECK-NEXT: s: "cluster" +// CHECK-NEXT: } +// CHECK-NEXT: } + +// CHECK: library { +// CHECK-NEXT: function { +// CHECK-NEXT: signature { +// CHECK-NEXT: name: "foo0" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_add.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_add.mlir new file mode 100644 index 0000000000000000000000000000000000000000..760e06e6c6abc4a7d1cccf86e8437656d46446ca --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_add.mlir @@ -0,0 +1,41 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> +attributes {tf.entry_function = {inputs = "input0,input1", outputs = "Add"}} { + %graph = tf_executor.graph { + %2:2 = tf_executor.island wraps "tf.Add"(%arg0, %arg1) {T = "tfdtype$DT_INT32", device = ""} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> loc("Add") + tf_executor.fetch %2 : tensor<10xi32> + } + func.return %graph : tensor<10xi32> +} + +// CHECK: node { +// CHECK-NEXT: name: "input0" +// CHECK-NEXT: op: "_Arg" +// CHECK: node { +// CHECK-NEXT: name: "input1" +// CHECK-NEXT: op: "_Arg" +// CHECK: node { +// CHECK-NEXT: name: "Add{{_.*_1}}" +// CHECK-NEXT: op: "Add" +// CHECK-NEXT: input: "input0" +// CHECK-NEXT: input: "input1" +// CHECK: node { +// CHECK-NEXT: name: "Add" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "Add{{_.*_1}}" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "T" +// CHECK-NEXT: value { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "index" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: library { +// CHECK-NEXT: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_identity_n.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_identity_n.mlir new file mode 100644 index 0000000000000000000000000000000000000000..af64e7bcb6763fab4950e85301b125b33987f2d9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_identity_n.mlir @@ -0,0 +1,25 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() -> tensor<2x3xi32> { + %graph = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {value = dense<5> : tensor<2x3xi32>} : () -> tensor<2x3xi32> loc("Const0") + %1:2 = tf_executor.island wraps "tf.Const"() {value = dense<4.2> : tensor<4x5xf32>} : () -> tensor<4x5xf32> loc("Const1") + %2:3 = tf_executor.island wraps "tf.IdentityN"(%0, %1) : (tensor<2x3xi32>, tensor<4x5xf32>) -> (tensor<2x3xi32>, tensor<4x5xf32>) loc("MyIdentityN") + tf_executor.fetch %2#0 : tensor<2x3xi32> + } + func.return %graph : tensor<2x3xi32> +} + +// CHECK: name: "MyIdentityN" +// CHECK-NEXT: op: "IdentityN" +// CHECK-NEXT: input: "Const0" +// CHECK-NEXT: input: "Const1" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "T" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: type: DT_FLOAT +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_tpu_embedding_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_tpu_embedding_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..dbf231bed34b281add753c3b59a112da946e67a7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/tf_tpu_embedding_ops.mlir @@ -0,0 +1,62 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.RecvTPUEmbeddingActivations"() {config = "test_config_recv_embedding"} : () -> tensor<512x256xf32> loc("RecvTPUEmbedding") + %1:1 = tf_executor.island wraps "tf.SendTPUEmbeddingGradients"(%0) {N = 1 : i64, NN = 0 : i64, config = "test_config_send_embedding", operandSegmentSizes = array} : (tensor<512x256xf32>) -> () loc("SendTPUEmbedding") + tf_executor.fetch + } + func.return +} + +// CHECK: name: "RecvTPUEmbedding" +// CHECK-NEXT: op: "RecvTPUEmbeddingActivations" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "_output_shapes" +// CHECK-NEXT: value { +// CHECK-NEXT: list { +// CHECK-NEXT: shape { +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 512 +// CHECK-NEXT: } +// CHECK-NEXT: dim { +// CHECK-NEXT: size: 256 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "config" +// CHECK-NEXT: value { +// CHECK-NEXT: s: "test_config_recv_embedding" +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "num_outputs" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } + +// CHECK: name: "SendTPUEmbedding" +// CHECK-NEXT: op: "SendTPUEmbeddingGradients" +// CHECK-NEXT: input: "RecvTPUEmbedding" +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "N" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 1 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "NN" +// CHECK-NEXT: value { +// CHECK-NEXT: i: 0 +// CHECK-NEXT: } +// CHECK-NEXT: } +// CHECK-NEXT: attr { +// CHECK-NEXT: key: "config" +// CHECK-NEXT: value { +// CHECK-NEXT: s: "test_config_send_embedding" +// CHECK-NEXT: } +// CHECK-NEXT: } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7557bd929c8567b711e2afe5bc9ea257b77202c0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_attr.mlir @@ -0,0 +1,41 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +// Check that attributes that define types are exported. + +// CHECK: key: "Tinputs" +// CHECK-NEXT: value +// CHECK-NEXT: list +// CHECK-NEXT: type: DT_FLOAT + +// CHECK: key: "Toutputs" +// CHECK-NEXT: value +// CHECK-NEXT: list +// CHECK-NEXT: type: DT_FLOAT + +// CHECK: "extra_type_attr" +// CHECK-NEXT: value +// CHECK-NEXT: list +// CHECK-NEXT: type: DT_INT32 +// CHECK-NEXT: type: DT_FLOAT + +// CHECK-LABEL: function +// CHECK: name: "plain" +// CHECK: Placeholder +// CHECK: key: "type" +// CHECK: type: DT_INT8 + +func.func @main(%arg0 : tensor<16xf32>) { + tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.MlirPassthroughOp"(%arg0) {extra_type_attr = [tensor<5xi32>, tensor<16xf32>], Tinputs = [tensor<16xf32>], Toutputs = [tensor<16xf32>], mlir_module = ""} : (tensor<16xf32>) -> tensor<16xf32> + tf_executor.fetch + } + func.return +} + +func.func @plain() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Placeholder"() {type = i8} : () -> tensor<16xi8> + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_list_attr.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_list_attr.mlir new file mode 100644 index 0000000000000000000000000000000000000000..607e917fe7ada7013e431bf87d1042f099acab45 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/type_list_attr.mlir @@ -0,0 +1,21 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + // CHECK: key: "emptylist" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: } + // CHECK-NEXT: } + // CHECK: key: "typelist" + // CHECK-NEXT: value { + // CHECK-NEXT: list { + // CHECK-NEXT: type: DT_INT32 + // CHECK-NEXT: type: DT_FLOAT + // CHECK-NEXT: } + // CHECK-NEXT: } + %0:2 = tf_executor.island wraps "tf.Placeholder"() {name = "dummy", dtype = "tfdtype$DT_FLOAT", emptylist = [], typelist = ["tfdtype$DT_INT32", "tfdtype$DT_FLOAT"]} : () -> tensor<*xi32> + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_name.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_name.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8abf7fab499e16b5fd930481bae95e4e7bd4bb0d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_name.mlir @@ -0,0 +1,20 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + tf_executor.graph { + // CHECK: name: "foo" + %0:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<0> : tensor} : () -> (tensor) loc("foo") + // CHECK: name: "foo{{_.*_1}}" + %1:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<1> : tensor} : () -> (tensor) loc("foo") + // CHECK: name: "foo1" + %2:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<2> : tensor} : () -> (tensor) loc("foo1") + // CHECK: name: "foo{{_.*_2}}" + %3:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<2> : tensor} : () -> (tensor) loc("foo") + // CHECK: name: "2" + %4:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<3> : tensor} : () -> (tensor) loc("2") + // CHECK: name: "3" + %5:2 = tf_executor.island wraps "tf.Const"() {dtype = "tfdtype$DT_INT32", value = dense<3> : tensor} : () -> (tensor) loc("3") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_output_name.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_output_name.mlir new file mode 100644 index 0000000000000000000000000000000000000000..120d9e9160a143c1cb8ec1b48f2c6e2725895e7e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/unique_output_name.mlir @@ -0,0 +1,28 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +module attributes {tf.versions = {producer = 946 : i32}, tf_saved_model.semantics, tfl.description = "MLIR Converted.", tfl.schema_version = 3 : i32} { + func.func @main(%arg0: tensor<2x4x2x2xf32> {tf_saved_model.index_path = ["x"]}) -> (tensor<2x2x1x2xi32> {tf_saved_model.index_path = ["output_1"]}, tensor<2x2x1x2xf32> {tf_saved_model.index_path = ["output_0"]}) attributes {tf.entry_function = {inputs = "serving_default_x:0", outputs = "PartitionedCall:1,PartitionedCall:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0:2 = tf_executor.graph { + %outputs:2, %control = tf_executor.island wraps "tf.MaxPoolWithArgmax"(%arg0) {T = f32, Targmax = i32, include_batch_in_index = false, ksize = [1, 2, 2, 1], padding = "SAME", strides = [1, 2, 2, 1]} : (tensor<2x4x2x2xf32>) -> (tensor<2x2x1x2xf32>, tensor<2x2x1x2xi32>) + tf_executor.fetch %outputs#1, %outputs#0 : tensor<2x2x1x2xi32>, tensor<2x2x1x2xf32> + } + func.return %0#0, %0#1 : tensor<2x2x1x2xi32>, tensor<2x2x1x2xf32> + } +} + +// CHECK: name: "serving_default_x" +// CHECK-NEXT: op: "_Arg" + +// CHECK: name: "tf.MaxPoolWithArgmax" +// CHECK-NEXT: op: "MaxPoolWithArgmax" +// CHECK-NEXT: input: "serving_default_x" + + +// CHECK: name: "PartitionedCall" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "tf.MaxPoolWithArgmax:1" + +// CHECK: name: "PartitionedCall1" +// CHECK-NOT: name: "PartitionedCall" +// CHECK-NEXT: op: "_Retval" +// CHECK-NEXT: input: "tf.MaxPoolWithArgmax" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/while-loop.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/while-loop.mlir new file mode 100644 index 0000000000000000000000000000000000000000..cb2de41b64ea12e29d023a3aa942354a2a4e4ade --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/while-loop.mlir @@ -0,0 +1,27 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +func.func @main() { + // CHECK: name: "while/Merge" + // CHECK-NEXT: op: "Merge" + // CHECK-NEXT: input: "while/Enter" + // CHECK-NEXT: input: "while/NextIteration" + // CHECK: name: "while/NextIteration" + // CHECK-NEXT: op: "NextIteration" + // CHECK-NEXT: input: "while/Add" + tf_executor.graph { + %0:3 = tf_executor.NextIteration.Source : tensor<*xi32> {device = "", T = "tfdtype$DT_INT32"} loc("while/NextIteration") + %1:2 = tf_executor.island wraps "tf.VariableV2"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<0> : tensor, shape = #tf_type.shape<0>, container = "", shared_name = ""} : () -> tensor loc("Ref_Variable") + %2:2 = tf_executor.Enter %1#0 frame "while/while_context" parallel_iterations 10 : (tensor) -> (tensor<*xi32>, !tf_executor.control) {device = "", T = "tfdtype$DT_INT32"} loc("while/Enter") + %3:3 = tf_executor.Merge %2#0, %0#0 : tensor<*xi32> {device = "", N = 2, T = "tfdtype$DT_INT32"} loc("while/Merge") + %4:2 = tf_executor.island(%3#2) wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<10> : tensor} : () -> tensor loc("while/Less/y") + %5:2 = tf_executor.island wraps "tf.Less"(%3#0, %4#0) {device = "", T = "tfdtype$DT_INT32"} : (tensor<*xi32>, tensor) -> tensor<*xi1> loc("while/Less") + %6:2 = tf_executor.LoopCond %5#0 : (tensor<*xi1>) -> (tensor<*xi1>, !tf_executor.control) {device = ""} loc("while/LoopCond") + %7:3 = tf_executor.Switch %3#0, %6#0 : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi32>, !tf_executor.control) {device = "", T = "tfdtype$DT_INT32", _class = ["loc:@while/Merge"]} loc("while/Switch") + %8:2 = tf_executor.Exit %7#1 : tensor<*xi32> {device = "", T = "tfdtype$DT_INT32"} loc("while/Exit") + %10:2 = tf_executor.island(%7#2) wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", value = dense<1> : tensor} : () -> tensor loc("while/Add/y") + %11:2 = tf_executor.island wraps "tf.AssignAdd"(%7#0, %10#0) {device = "", T = "tfdtype$DT_INT32"} : (tensor<*xi32>, tensor) -> tensor<*xi32> loc("while/Add") + tf_executor.NextIteration.Sink [%0#1] %11#0 : tensor<*xi32> {device = "", T = "tfdtype$DT_INT32"} loc("while/NextIteration") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlprogram.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlprogram.mlir new file mode 100644 index 0000000000000000000000000000000000000000..eccee9f9e7115569141cd463144d93c3afbeccc2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/mlprogram.mlir @@ -0,0 +1,136 @@ +// RUN: tf-opt --split-input-file -tf-lower-to-mlprogram-and-hlo %s -o - | FileCheck %s + +module attributes {tf_saved_model.semantics} { + // CHECK-LABEL: func @lowers_to_stablehlo + func.func @lowers_to_stablehlo(%arg0: tensor {tf_saved_model.index_path = []}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["lowers_to_stablehlo"]} + { + // CHECK-DAG: [[one:%.*]] = stablehlo.constant dense<1> + // CHECK-DAG: [[twenty:%.*]] = stablehlo.constant dense<20> + // CHECK-DAG: [[r3:%.*]] = stablehlo.subtract [[twenty]], %arg0 + // CHECK-DAG: [[zero:%.*]] = stablehlo.constant dense<0> + // CHECK-DAG: [[r4:%.*]] = stablehlo.divide [[r3]], [[one]] + // CHECK-DAG: [[r5:%.*]] = stablehlo.compare NE + // CHECK-DAG: [[r6:%.*]] = stablehlo.compare LT + // CHECK: [[result:%.*]] = stablehlo.select + // CHECK-NEXT: return [[result]] + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<20> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_4, %control_5 = tf_executor.island wraps "tf.Range"(%outputs_2, %outputs_0, %outputs) {device = ""} : (tensor, tensor, tensor) -> tensor + %outputs_6, %control_7 = tf_executor.island wraps "tf.Sub"(%outputs_0, %arg0) {device = ""} : (tensor, tensor) -> tensor + %outputs_8, %control_9 = tf_executor.island wraps "tf.FloorDiv"(%outputs_6, %outputs) {device = ""} : (tensor, tensor) -> tensor + tf_executor.fetch %outputs_8 : tensor + } + func.return %0 : tensor + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // CHECK-LABEL: func @removes_dead_code + func.func @removes_dead_code(%arg0: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0", tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["removes_dead_code"]} + { + // CHECK-NEXT: return + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<20> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_4, %control_5 = tf_executor.island wraps "tf.Range"(%outputs_2, %outputs_0, %outputs) {device = ""} : (tensor, tensor, tensor) -> tensor + %outputs_6, %control_7 = tf_executor.island wraps "tf.Sub"(%outputs_0, %outputs_2) {device = ""} : (tensor, tensor) -> tensor + %outputs_8, %control_9 = tf_executor.island wraps "tf.FloorDiv"(%outputs_6, %outputs) {device = ""} : (tensor, tensor) -> tensor + tf_executor.fetch %control_9 : !tf_executor.control + } + return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // CHECK-LABEL: func @lowers_variable_ops + func.func @lowers_variable_ops() + attributes {tf_saved_model.exported_names = ["lowers_variable_ops"]} + { + // CHECK: ml_program.global_store + tf_executor.graph { + %0, %c0 = tf_executor.island wraps "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %1, %c1 = tf_executor.island wraps "tf.VarHandleOp"() {container = "", shared_name = "v"} : () -> tensor>> + %c2 = tf_executor.island wraps "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor) -> () + tf_executor.fetch %c2 : !tf_executor.control + } + return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + func.func private @while_cond(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = tf_executor.graph { + %1, %c0 = tf_executor.island wraps "tf.Const"() {value = dense<1> : tensor} : () -> tensor + tf_executor.fetch %1 : tensor + } + return %0 : tensor + } + + func.func private @while_body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + tf_executor.graph { + %1, %c1 = tf_executor.island wraps "tf.VarHandleOp"() {container = "", shared_name = "v"} : () -> tensor>> + %c2 = tf_executor.island wraps "tf.AssignVariableOp"(%1, %arg1) : (tensor>>, tensor) -> () + tf_executor.fetch %c2 : !tf_executor.control + } + return %arg0, %arg1 : tensor, tensor + } + + // CHECK-LABEL: func @handles_variables_in_while_loops + func.func @handles_variables_in_while_loops(%arg0: tensor>> {tf._user_specified_name = "arg0", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0", tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["handles_variables_in_while_loops"]} + { + // CHECK: stablehlo.while + tf_executor.graph { + %0, %c0 = tf_executor.island wraps "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %1, %c1 = tf_executor.island wraps "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %2, %3, %c2 = tf_executor.island wraps "tf.While"(%0, %1) { + body = @while_body , cond = @while_cond, is_stateless = true + } : (tensor, tensor) -> (tensor, tensor) + tf_executor.fetch %c2 : !tf_executor.control + } + return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // CHECK: func @a + // CHECK: func @b + // CHECK: func @c + // CHECK-LABEL @expands_exported_names + func.func @expands_exported_names() + attributes {tf_saved_model.exported_names = ["a", "b", "c"]} { + return + } +} + +// ----- + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1512 : i32}} { + // CHECK-LABEL @lowers_string_ops + // CHECK-DAG: ml_program.global public @vars.Variable_1([]) : tensor + func.func @lowers_string_ops(%arg0: tensor<128xi32>, %arg1: tensor<128xi32>, %arg2: tensor<128x1xi32>, %arg3: tensor<128x90xi32>, %arg4: tensor<128x90xi32>, %arg5: tensor<128x90xi32>, %arg6: tensor<128x90x64xf32>, %arg7: tensor<128x90x64xf32>) -> tensor { + // CHECK: %[[v0:.*]] = ml_program.global_load @vars.Variable_1 : tensor + %0 = tf_executor.graph { + %outputs_4, %control_5 = tf_executor.island wraps "tf.VarHandleOp"() {container = "", shared_name = "Variable"} : () -> tensor>> + %outputs_10, %control_11 = tf_executor.island wraps "tf.VarHandleOp"() {container = "", shared_name = "Variable_1"} : () -> tensor>> + %outputs_14, %control_15 = tf_executor.island wraps "tf.ReadVariableOp"(%outputs_10) : (tensor>>) -> tensor + %control_18 = tf_executor.island wraps "tf.AssignVariableOp"(%outputs_4, %outputs_14) : (tensor>>, tensor) -> () + tf_executor.fetch %outputs_14 : tensor + } + func.return %0 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/move_tpu_compile_to_front.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/move_tpu_compile_to_front.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b234eb89d221adf3be9273b12ca4b1694154748e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/move_tpu_compile_to_front.mlir @@ -0,0 +1,47 @@ +// RUN: tf-opt %s -allow-unregistered-dialect --tf-move-tpu-compile-to-front --split-input-file | FileCheck %s + +module { + +// CHECK-LABEL: does_basic_reordering +func.func @does_basic_reordering() -> () { + // CHECK: _TPUCompileMlir + // CHECK-SAME: X + // CHECK: _TPUCompileMlir + // CHECK-SAME: Y + // CHECK: OpA + // CHECK: OpB + // CHECK: OpC + "tf.OpA"() : () -> () + %status_x, %program_x = "tf._TPUCompileMlir"() { metadata = "X", mlir_module = "..." } : () -> (tensor, tensor) + "tf.OpB"() : () -> () + %status_y, %program_y = "tf._TPUCompileMlir"() { metadata = "Y", mlir_module = "..." } : () -> (tensor, tensor) + "tf.OpC"() : () -> () +} + +// CHECK-LABEL: does_reordering_for_nested_compiles +func.func @does_reordering_for_nested_compiles() -> () { + // CHECK: _TPUCompileMlir + // CHECK-SAME: Z + // CHECK: tf_device.launch + // CHECK-NEXT: _TPUCompileMlir + // CHECK-SAME: X + // CHECK: tf_device.launch + // CHECK-NEXT: _TPUCompileMlir + // CHECK-SAME: Y + // CHECK: OpA + // CHECK: OpB + // CHECK: OpC + "tf.OpA"() : () -> () + "tf_device.launch"() ({ + %status_x, %program_x = "tf._TPUCompileMlir"() { metadata = "X", mlir_module = "..." } : () -> (tensor, tensor) + tf_device.return + }) {device = ""} : () -> () + "tf.OpB"() : () -> () + "tf_device.launch"() ({ + %status_y, %program_y = "tf._TPUCompileMlir"() { metadata = "Y", mlir_module = "..." } : () -> (tensor, tensor) + tf_device.return + }) {device = ""} : () -> () + %status_z, %program_z = "tf._TPUCompileMlir"() { metadata = "Z", mlir_module = "..." } : () -> (tensor, tensor) + "tf.OpC"() : () -> () +} +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/name_anonymous_iterators.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/name_anonymous_iterators.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8bf4291a46eaedbb5ddbe1fc8ff8cdf0e7cf6950 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/name_anonymous_iterators.mlir @@ -0,0 +1,36 @@ +// RUN: tf-opt %s --tf-name-anonymous-iterators --split-input-file | FileCheck %s + +// CHECK-LABEL: gives_a_name_to_anonymous_iterators +func.func private @gives_a_name_to_anonymous_iterators() { + // CHECK: "tf.Iterator" + // CHECK-SAME: output_shapes{{.*}}200x28x28x1{{.*}}200x10 + // CHECK-SAME: output_types = [f32, f32] + // CHECK-SAME: shared_name = "_iterator1" + %0 = "tf.AnonymousIteratorV3"() {output_shapes = [ + #tf_type.shape<200x28x28x1>, + #tf_type.shape<200x10>], output_types = [f32, f32]} : () -> tensor + // CHECK: "tf.Iterator" + // CHECK-SAME: shared_name = "_iterator2" + %1 = "tf.AnonymousIteratorV3"() {output_shapes = [ + #tf_type.shape<200x10>], output_types = [f32]} : () -> tensor + return +} + +// ----- + +// CHECK-LABEL: handles_all_versions +func.func private @handles_all_versions() { + // CHECK: "tf.Iterator" + // CHECK-SAME: 1x42 + %0 = "tf.AnonymousIterator"() {output_shapes = [ + #tf_type.shape<1x42>], output_types = [f32]} : () -> tensor + // CHECK: "tf.Iterator" + // CHECK-SAME: 2x42 + %1, %2 = "tf.AnonymousIteratorV2"() {output_shapes = [ + #tf_type.shape<2x42>], output_types = [f32]} : () -> (tensor, tensor) + // CHECK: "tf.Iterator" + // CHECK-SAME: 3x42 + %3 = "tf.AnonymousIteratorV3"() {output_shapes = [ + #tf_type.shape<3x42>], output_types = [f32]} : () -> tensor + return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize-arg-operand-constraint.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize-arg-operand-constraint.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b3ce2b7ed8966879c51c60ad298352ebf4acb434 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize-arg-operand-constraint.mlir @@ -0,0 +1,13 @@ +// RUN: tf-opt -tf-optimize %s | FileCheck %s + +// Check passing an argument into DefinedByConv2D constraint does not crash. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1xf32>) -> tensor<1xf32> +attributes {tf.entry_function = {inputs = "input", outputs = "output_node"}} { + %0 = arith.constant dense<2.000000e+00> : tensor + %1 = arith.constant dense<1.000000e+00> : tensor + %2 = "tf.AddV2"(%arg0, %1) {T = "tfdtype$DT_FLOAT", device = "", name = "StatefulPartitionedCall/add"} : (tensor<1xf32>, tensor) -> tensor<1xf32> + %3 = "tf.Mul"(%2, %0) {T = "tfdtype$DT_FLOAT", device = "", name = "output_node"} : (tensor<1xf32>, tensor) -> tensor<1xf32> + func.return %3 : tensor<1xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize.mlir new file mode 100644 index 0000000000000000000000000000000000000000..9ed01eda65f57fd29f3581f68cb3b980e16829e6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/optimize.mlir @@ -0,0 +1,54 @@ +// RUN: tf-opt -tf-optimize %s -o %t && FileCheck %s < %t + +// CHECK-LABEL: convbiasaddmul +func.func @convbiasaddmul(%arg: tensor<256x32x32x3xf32>) -> tensor<256x8x7x16xf32> { + %filter = arith.constant dense<2.0> : tensor<3x3x3x16xf32> + %bias = arith.constant dense<3.0> : tensor<16xf32> + %value = arith.constant dense<4.0> : tensor<16xf32> + %0 = "tf.Conv2D"(%arg, %filter) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", dilations = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 5, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x8x7x16xf32> + %1 = "tf.BiasAdd"(%0, %bias) {T = "tfdtype$DT_FLOAT", data_format = "NHWC"}: (tensor<256x8x7x16xf32>, tensor<16xf32>) -> tensor<256x8x7x16xf32> + %2 = "tf.Mul"(%1, %value) {T = "tfdtype$DT_FLOAT"} : (tensor<256x8x7x16xf32>, tensor<16xf32>) -> tensor<256x8x7x16xf32> + func.return %2 : tensor<256x8x7x16xf32> + +// CHECK-DAG: %[[cst:.*]] = "tf.Const{{.*}} dense<8.000000e+00> : tensor<3x3x3x16xf32> +// CHECK-DAG: %[[cst_0:.*]] = "tf.Const{{.*}} dense<1.200000e+01> : tensor<16xf32> +// CHECK-NEXT: %[[conv:.*]] = "tf.Conv2D"(%arg0, %[[cst]]) +// CHECK-NEXT: %[[bias:.*]] = "tf.AddV2"(%[[conv]], %[[cst_0]]) +// CHECK-NEXT: return %[[bias]] : tensor<256x8x7x16xf32> +} + +// CHECK-LABEL: convaddv2mul +func.func @convaddv2mul(%arg: tensor<256x32x32x3xf32>) -> tensor<256x8x7x16xf32> { + %filter = arith.constant dense<2.0> : tensor<3x3x3x16xf32> + %bias = arith.constant dense<3.0> : tensor<16xf32> + %value = arith.constant dense<4.0> : tensor<16xf32> + %0 = "tf.Conv2D"(%arg, %filter) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", dilations = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 5, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x8x7x16xf32> + %1 = "tf.AddV2"(%0, %bias) {T = "tfdtype$DT_FLOAT"}: (tensor<256x8x7x16xf32>, tensor<16xf32>) -> tensor<256x8x7x16xf32> + %2 = "tf.Mul"(%1, %value) {T = "tfdtype$DT_FLOAT"} : (tensor<256x8x7x16xf32>, tensor<16xf32>) -> tensor<256x8x7x16xf32> + func.return %2 : tensor<256x8x7x16xf32> + +// CHECK-DAG: %[[cst:.*]] = "tf.Const{{.*}} dense<8.000000e+00> : tensor<3x3x3x16xf32> +// CHECK-DAG: %[[cst_0:.*]] = "tf.Const{{.*}} dense<1.200000e+01> : tensor<16xf32> +// CHECK-NEXT: %[[conv:.*]] = "tf.Conv2D"(%arg0, %[[cst]]) +// CHECK-NEXT: %[[add:.*]] = "tf.AddV2"(%[[conv]], %[[cst_0]]) +// CHECK-NEXT: return %[[add]] : tensor<256x8x7x16xf32> +} + +// CHECK-LABEL: fold_cast_fft_to_rfft +func.func @fold_cast_fft_to_rfft(%arg0: tensor<10x20x30xf32>) -> tensor<10x20x30xcomplex> { + %0 = "tf.Cast"(%arg0) : (tensor<10x20x30xf32>) -> tensor<10x20x30xcomplex> + %1 = "tf.FFT"(%0) : (tensor<10x20x30xcomplex>) -> tensor<10x20x30xcomplex> + func.return %1: tensor<10x20x30xcomplex> + +// CHECK: %[[cst:.*]] = arith.constant dense<30> : tensor<1xi32> +// CHECK: %[[rff:.*]] = "tf.RFFT"(%arg0, %[[cst]]) : (tensor<10x20x30xf32>, tensor<1xi32>) -> tensor<10x20x30xcomplex> +} + +// CHECK-LABEL: not_fold_cast_fft_to_rfft +func.func @not_fold_cast_fft_to_rfft(%arg0: tensor<10x20x30xcomplex>) -> tensor<10x20x30xcomplex> { + %0 = "tf.Cast"(%arg0) : (tensor<10x20x30xcomplex>) -> tensor<10x20x30xcomplex> + %1 = "tf.FFT"(%0) : (tensor<10x20x30xcomplex>) -> tensor<10x20x30xcomplex> + func.return %1: tensor<10x20x30xcomplex> + +// CHECK: %[[fft:.*]] = "tf.FFT"(%0) : (tensor<10x20x30xcomplex>) -> tensor<10x20x30xcomplex> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_by_dialect.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_by_dialect.mlir new file mode 100644 index 0000000000000000000000000000000000000000..bbf6191071d0248e4471e06345be5c46ff70fe5f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_by_dialect.mlir @@ -0,0 +1,217 @@ +// RUN: tf-opt %s -allow-unregistered-dialect --tf-order-by-dialect --split-input-file | FileCheck %s + +// CHECK-LABEL: @interleave +func.func @interleave(%arg0: f32) -> (f32, f32, f32) attributes {ignore_side_effects_for_testing} { + %0 = "x.a"(%arg0) : (f32) -> f32 + %1 = "y.a"(%arg0) : (f32) -> f32 + %2 = "z.a"(%arg0) : (f32) -> f32 + %3 = "x.b"(%0) : (f32) -> f32 + %4 = "y.b"(%1) : (f32) -> f32 + %5 = "z.b"(%2) : (f32) -> f32 + %6 = "x.c"(%3) : (f32) -> f32 + %7 = "y.c"(%4) : (f32) -> f32 + %8 = "z.c"(%5) : (f32) -> f32 + func.return %6, %7, %8 : f32, f32, f32 +} +// CHECK: x.a +// CHECK-NEXT: x.b +// CHECK-NEXT: x.c +// CHECK-NEXT: y.a +// CHECK-NEXT: y.b +// CHECK-NEXT: y.c +// CHECK-NEXT: z.a +// CHECK-NEXT: z.b +// CHECK-NEXT: z.c + +// ----- + +// CHECK-LABEL: @terminator +func.func @terminator(%arg0: f32) -> (f32) attributes {ignore_side_effects_for_testing} { + func.call @terminator(%arg0) : (f32) -> (f32) + "x.a"(%arg0) : (f32) -> () + "y.a"(%arg0) : (f32) -> () + "z.a"(%arg0) : (f32) -> () + func.return %arg0 : f32 +} +// CHECK: x.a +// CHECK: y.a +// CHECK: z.a +// CHECK: return +// CHECK-NOT: call +// CHECK: } + +// ----- + +// CHECK-LABEL: @fanout +func.func @fanout(%arg0: f32) -> (f32) attributes {ignore_side_effects_for_testing} { + %0 = "x.a"(%arg0) : (f32) -> (f32) + %1 = "y.a"(%0) : (f32) -> (f32) + %2 = "y.b"(%0) : (f32) -> (f32) + %3 = "y.c"(%0) : (f32) -> (f32) + %4 = "y.d"(%0) : (f32) -> (f32) + %5 = "x.b"(%1, %2, %3, %4) : (f32, f32, f32, f32) -> (f32) + func.return %5 : f32 +} +// CHECK: x.a +// CHECK-NEXT: y.a +// CHECK-NEXT: y.b +// CHECK-NEXT: y.c +// CHECK-NEXT: y.d +// CHECK-NEXT: x.b + +// ----- + +// CHECK-LABEL: @constants +func.func @constants() -> f32 attributes {ignore_side_effects_for_testing} { + %0 = "a.x"() : () -> f32 + %1 = "b.x"() : () -> f32 + %2 = "c.x"() : () -> f32 + %3 = "d.x"(%0, %1, %2) : (f32, f32, f32) -> f32 + return %3 : f32 +} +// CHECK-DAG: a.x +// CHECK-DAG: b.x +// CHECK-DAG: c.x +// CHECK-NEXT: d.x + +// ----- + +// CHECK-LABEL: @tf_and_mhlo +func.func @tf_and_mhlo(%arg0: tensor<32x28x28x1xf32>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>, %arg4: tensor>>) -> (tensor<32x10xf32>) { + %0 = mhlo.constant dense<0.000000e+00> : tensor<32x10xf32> + %1 = mhlo.constant dense<0.000000e+00> : tensor<32x28x28x5xf32> + %2 = "tf.ReadVariableOp"(%arg4) : (tensor>>) -> tensor<10xf32> + %3 = "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor<5xf32> + %4 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor<3x3x1x5xf32> + %5 = "tf.ReadVariableOp"(%arg3) : (tensor>>) -> tensor<3920x10xf32> + %6 = mhlo.convolution(%arg0, %4) dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f], window = {stride = [1, 1], pad = [[1, 1], [1, 1]], rhs_dilate = [1, 1]} {batch_group_count = 1 : i64, feature_group_count = 1 : i64} : (tensor<32x28x28x1xf32>, tensor<3x3x1x5xf32>) -> tensor<32x28x28x5xf32> + %7 = "mhlo.broadcast_in_dim"(%3) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<5xf32>) -> tensor<32x28x28x5xf32> + %8 = mhlo.add %6, %7 : tensor<32x28x28x5xf32> + %9 = mhlo.maximum %8, %1 : tensor<32x28x28x5xf32> + %10 = "mhlo.reshape"(%9) : (tensor<32x28x28x5xf32>) -> tensor<32x3920xf32> + %11 = "mhlo.dot"(%10, %5) : (tensor<32x3920xf32>, tensor<3920x10xf32>) -> tensor<32x10xf32> + %12 = "mhlo.broadcast_in_dim"(%2) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> : (tensor<10xf32>) -> tensor<32x10xf32> + %13 = mhlo.add %11, %12 : tensor<32x10xf32> + %14 = mhlo.maximum %13, %0 : tensor<32x10xf32> + return %14 : tensor<32x10xf32> +} +// CHECK: ReadVariableOp +// CHECK: mhlo.convolution +// CHECK: mhlo.add +// CHECK: mhlo.maximum +// CHECK: mhlo.reshape +// CHECK: mhlo.dot +// CHECK: mhlo.add +// CHECK: mhlo.maximum +// CHECK: return{{.*}}tensor<32x10xf32> + +// ----- + +// CHECK-LABEL: @mhlo_while +func.func private @mhlo_while() { + // CHECK-NEXT: mhlo.constant + // CHECK-NEXT: mhlo.constant + // CHECK-NEXT: mhlo.constant + %0 = mhlo.constant dense<-1> : tensor + %1 = mhlo.constant dense<0> : tensor + %2 = mhlo.constant dense<20> : tensor + // CHECK-NEXT: mhlo.while + %3, %4, %5 = mhlo.while(%iterArg = %1, %iterArg_0 = %0, %iterArg_1 = %1) : tensor, tensor, tensor + cond { + %17 = mhlo.compare LT, %iterArg_1, %2 : (tensor, tensor) -> tensor + mhlo.return %17 : tensor + } do { + mhlo.return %0, %0, %0 : tensor, tensor, tensor + } + return +} + +// ----- + +// CHECK-LABEL: @nested_regions +func.func @nested_regions(%arg0: f32) attributes {ignore_side_effects_for_testing} { + %0 = "x.a"(%arg0) : (f32) -> f32 + %1 = "y.a"(%arg0) : (f32) -> f32 + %2 = "x.b"(%arg0) : (f32) -> f32 + "nested1"() ({ + %3 = "y.a"(%arg0) : (f32) -> f32 + %4 = "x.b"(%1) : (f32) -> f32 + %5 = "y.b"(%2) : (f32) -> f32 + "nested2"() ({ + %6 = "x.b"(%3) : (f32) -> f32 + %7 = "y.c"(%4) : (f32) -> f32 + %8 = "x.d"(%arg0) : (f32) -> f32 + %9 = "y.e"(%6) : (f32) -> f32 + }) : () -> () + }) : () -> () +} +// CHECK: x.a +// CHECK-NEXT: x.b +// CHECK-NEXT: y.a +// CHECK-NEXT: nested1 +// CHECK-NEXT: y.a +// CHECK-NEXT: y.b +// CHECK-NEXT: x.b +// CHECK-NEXT: nested2 +// CHECK-NEXT: x.b +// CHECK-NEXT: x.d +// CHECK-NEXT: y.c +// CHECK-NEXT: y.e + +// ----- + +// CHECK-LABEL: interleaved_tf_and_mhlo +func.func private @interleaved_tf_and_mhlo() { + %m0 = mhlo.constant dense<0> : tensor + %t0 = "tf.Const"() { value = dense<0> : tensor<1xi32> } : () -> tensor<1xi32> + %m1 = mhlo.constant dense<1> : tensor + %t1 = "tf.Const"() { value = dense<1> : tensor<1xi32> } : () -> tensor<1xi32> + %m2 = mhlo.constant dense<1> : tensor + %t2 = "tf.Const"() { value = dense<1> : tensor<1xi32> } : () -> tensor<1xi32> + %m3 = mhlo.constant dense<1> : tensor + %t3 = "tf.Const"() { value = dense<1> : tensor<1xi32> } : () -> tensor<1xi32> + // CHECK: mhlo.constant + // CHECK: mhlo.constant + // CHECK: mhlo.constant + // CHECK: mhlo.constant + // CHECK: tf.Const + // CHECK: tf.Const + // CHECK: tf.Const + // CHECK: tf.Const + return +} + +// ----- + +// CHECK-LABEL: variable_ops +func.func private @variable_ops(%arg0: tensor>>) { + %t3 = "tf.Const"() { value = dense<0> : tensor<0xi32> } : () -> tensor<0xi32> + // Without side effect analysis, we would now schedule tf.ReadVariableOp next, + // since all its operands are ready. Check that we don't. + %0 = mhlo.constant dense<0.> : tensor + "tf.AssignVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + %1 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: tf.Const + // CHECK: mhlo.constant + // CHECK: tf.Assign + // CHECK: tf.Read + return +} + +// ----- + +func.func private @id(%arg0: tensor) -> tensor { + return %arg0 : tensor +} + +// CHECK-LABEL: iterators +func.func private @iterators(%arg0 : tensor) { + %0 = "tf.Iterator"() {container = "", output_shapes = [#tf_type.shape<200x28x28x1>, #tf_type.shape<200x10>], output_types = [f32, f32], shared_name = "_iterator1"} : () -> tensor + %1 = func.call @id(%arg0) : (tensor) -> tensor + "tf.MakeIterator"(%1, %0) {_class = ["loc:@BatchDatasetV2"], device = ""} : (tensor, tensor) -> () + %2:2 = "tf.IteratorGetNext"(%0) {_class = ["loc:@iterator"], device = ""} : (tensor) -> (tensor<200x28x28x1xf32>, tensor<200x10xf32>) + // CHECK: tf.Iterator + // CHECK: tf.MakeIterator + // CHECK: tf.IteratorGetNext + return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_for_program_key.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_for_program_key.mlir new file mode 100644 index 0000000000000000000000000000000000000000..20ccb2262a64de4919859bd605761fb4fd19851a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/order_for_program_key.mlir @@ -0,0 +1,59 @@ +// RUN: tf-opt %s -allow-unregistered-dialect --tf-order-for-program-key --split-input-file | FileCheck %s + +// CHECK-LABEL: func @test_reorder_launches +func.func @test_reorder_launches() { + // CHECK: TPUCompileMlir + // CHECK: tf.OpA + "tf_device.launch"() ({ + %cst_0 = "tf.Const"() {value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + "tf.OpA"(%cst_0) { mini_batch_splits = ""} : (tensor<1x!tf_type.string>) -> () + tf_device.return + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> () + %0:2 = "tf_device.launch"() ({ + %compilation_status, %program = "tf._TPUCompileMlir"() { metadata = "...", mlir_module = "..." } : () -> (tensor, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program : tensor, tensor<3x!tf_type.string> + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>) + return +} + +// ----- + +// CHECK-LABEL: func @test_reorder_ops +func.func @test_reorder_ops() { + // CHECK: TPUCompileMlir + // CHECK: tf.OpA + // CHECK: tf.OpB + "tf.OpA"() : () -> () + %compilation_status, %program = "tf._TPUCompileMlir"() { metadata = "...", mlir_module = "..." } : () -> (tensor, tensor<3x!tf_type.string>) + "tf.OpB"() : () -> () + return +} + +// ----- + +// CHECK-LABEL: func @test_compile_with_dependencies +func.func @test_compile_with_dependencies() { + // CHECK: tf.OpA + // CHECK: TPUCompileMlir + // CHECK: tf.OpB + %a = "tf.OpA"() : () -> tensor + %b = "tf.OpB"() : () -> tensor + %compilation_status, %program = "tf._TPUCompileMlir"(%a) { metadata = "...", mlir_module = "..." } : (tensor) -> (tensor, tensor<3x!tf_type.string>) + return +} + +// ----- + +// CHECK-LABEL: func @test_side_effects +func.func @test_side_effects(%arg0: tensor>>) { + // CHECK: ReadVariable + // CHECK: TPUCompileMlir + // CHECK: AssignVariable + %1 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %compilation_status, %program = "tf._TPUCompileMlir"(%1) { metadata = "...", mlir_module = "..." } : (tensor) -> (tensor, tensor<3x!tf_type.string>) + %2 = "tf.Cast"(%compilation_status) : (tensor) -> tensor + %3 = builtin.unrealized_conversion_cast to tensor + "tf.AssignVariableOp"(%arg0, %2) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg0, %3) : (tensor>>, tensor) -> () + return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands.mlir new file mode 100644 index 0000000000000000000000000000000000000000..69ad4073987d12696bec1edaf354fe463b2615f2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands.mlir @@ -0,0 +1,236 @@ +// RUN: tf-opt %s -tf-parallel-execute-to-islands=legacy-graph-export=false | env FILECHECK_OPTS="" FileCheck %s + +// CHECK-LABEL: func @testEmptyRegions +func.func @testEmptyRegions() { + tf_executor.graph { + tf_executor.island() { + "tf_device.parallel_execute"() ({ + tf_device.return + }, { + tf_device.return + }) {} : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: [[ISLAND_0_CTRL:%.+]] = tf_executor.island { +// CHECK: tf_executor.yield {_parallel_execution_ids = "p0:0"} +// CHECK: [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK: tf_executor.yield {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.fetch + + +// CHECK-LABEL: func @testDataOperandsAndResults +// CHECK-SAME: ([[ARG_0:%.+]]: tensor) +func.func @testDataOperandsAndResults(%arg0 : tensor) { + %0:2 = tf_executor.graph { + %1:2 = tf_executor.island { + %2 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %2 : tensor + } + %3:3 = tf_executor.island() { + %4:2 = "tf_device.parallel_execute"() ({ + %5 = "tf.opB"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }, { + %5 = "tf.opC"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %4#0, %4#1 : tensor, tensor + } + tf_executor.fetch %3#0, %3#1 : tensor, tensor + } + func.return +} + +// CHECK: [[INPUT_A:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"([[INPUT_A]]) {_parallel_execution_ids = "p0:0"} +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: [[ISLAND_1_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_C_OUTPUT:%.+]] = "tf.opC"([[INPUT_A]]) {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.yield [[OP_C_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]], [[ISLAND_1_OUTPUT]] : + + +// CHECK-LABEL: func @testControlOperands +func.func @testControlOperands() { + %0:2 = tf_executor.graph { + %1 = tf_executor.island { + "tf.someOp"() : () -> () + tf_executor.yield + } + %2:3 = tf_executor.island(%1) { + %3:2 = "tf_device.parallel_execute"() ({ + %4 = "tf.opA"() : () -> tensor + tf_device.return %4 : tensor + }, { + %4 = "tf.opB"() : () -> tensor + tf_device.return %4 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %3#0, %3#1 : tensor, tensor + } + tf_executor.fetch %2#0, %2#1 : tensor, tensor + } + func.return +} + +// CHECK: [[INPUT_CTRL:%.+]] = tf_executor.island { +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island([[INPUT_CTRL]]) { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() {_parallel_execution_ids = "p0:0"} +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_1_OUTPUT:%.+]], {{%.+}} = tf_executor.island([[INPUT_CTRL]]) { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]], [[ISLAND_1_OUTPUT]] : + + +// CHECK-LABEL: func @testControlResults +func.func @testControlResults() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = "tf_device.parallel_execute"() ({ + %2 = "tf.opA"() : () -> tensor + tf_device.return %2 : tensor + }, { + %2 = "tf.opB"() : () -> tensor + tf_device.return %2 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + %3 = tf_executor.island(%0#2) { + "tf.someOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch %3 : !tf_executor.control + } + func.return +} + +// CHECK: {{%.+}}, [[ISLAND_0_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() {_parallel_execution_ids = "p0:0"} +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: {{%.+}}, [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: [[OUTPUT_CTRL:%.+]] = tf_executor.island([[ISLAND_0_CTRL]], [[ISLAND_1_CTRL]]) { +// CHECK: [[FETCH_ISLAND:%.+]] = tf_executor.island([[OUTPUT_CTRL]]) { +// CHECK: tf_executor.fetch [[FETCH_ISLAND]] : !tf_executor.control + + +// CHECK-LABEL: func @testSomeRegionNoUsers +func.func @testSomeRegionNoUsers() { + %0 = tf_executor.graph { + %1:3 = tf_executor.island { + %2:2 = "tf_device.parallel_execute"() ({ + %3 = "tf.opA"() : () -> tensor + tf_device.return %3 : tensor + }, { + %3 = "tf.opB"() : () -> tensor + tf_device.return %3 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %2#0, %2#1 : tensor, tensor + } + tf_executor.fetch %1#0 : tensor + } + func.return +} + +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() {_parallel_execution_ids = "p0:0"} +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: {{%.+}}, [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]] : + +// CHECK-LABEL: @testRegionContainsMultipleOps +func.func @testRegionContainsMultipleOps() { + %0:2 = tf_executor.graph { + %outputs:2, %control = tf_executor.island { + %1:2 = "tf_device.parallel_execute"() ({ + %2 = "tf.opA"() : () -> tensor + %3 = "tf.opB"(%2) : (tensor) -> tensor + tf_device.return %3 : tensor + }, { + %2 = "tf.opC"() : () -> tensor + %3 = "tf.opD"(%2) : (tensor) -> tensor + tf_device.return %3 : tensor + }) : () -> (tensor, tensor) + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch %outputs#0, %outputs#1 : tensor, tensor + } + return +} + +// CHECK: [[OUTPUT_0:%.*]], {{%.*}} = tf_executor.island wraps "tf.opA"() {_parallel_execution_ids = "p0:0"} +// CHECK: [[OUTPUT_1:%.*]], {{%.*}} = tf_executor.island wraps "tf.opB"([[OUTPUT_0:%.*]]) {_parallel_execution_ids = "p0:0"} +// CHECK: [[OUTPUT_2:%.*]], {{%.*}} = tf_executor.island wraps "tf.opC"() {_parallel_execution_ids = "p0:1"} +// CHECK: [[OUTPUT_3:%.*]], {{%.*}} = tf_executor.island wraps "tf.opD"([[OUTPUT_2:%.*]]) {_parallel_execution_ids = "p0:1"} +// CHECK: tf_executor.fetch [[OUTPUT_1:%.*]], [[OUTPUT_3:%.*]] + +// ----- + +// Tests a ParallelExecute with a single region. + +// CHECK-LABEL: func @testSingleton +// CHECK-SAME: ([[ARG_0:%.+]]: tensor) +func.func @testSingleton(%arg0 : tensor) { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + %2 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %2 : tensor + } + %3:2 = tf_executor.island() { + %4 = "tf_device.parallel_execute"() ({ + %5 = "tf.opB"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }) {} : () -> tensor + tf_executor.yield %4 : tensor + } + tf_executor.fetch %3#0 : tensor + } + func.return +} + +// CHECK: [[INPUT_A:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"([[INPUT_A]]) {_parallel_execution_ids = "p0:0"} +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]] : + +// ----- +// Tests parallel_group attr can merge correctly. +// CHECK-LABEL: func @merge_of_parallel_group_attr +func.func @merge_of_parallel_group_attr() { + %0:2 = tf_executor.graph { + %outputs:2, %control = tf_executor.island { + %1:2 = "tf_device.parallel_execute" () ({ + %2 = "tf.opA"() : () -> tensor + %3 = "tf.opB"(%2) : (tensor) -> tensor + tf_device.return %3 : tensor + }, { + %2 = "tf.opC"() : () -> tensor + %3 = "tf.opD"(%2) : (tensor) -> tensor + tf_device.return %2 : tensor + }) {_parallel_execution_ids = "r4:5"} : () -> (tensor, tensor) + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch %outputs#0, %outputs#1 : tensor, tensor + } + return +} + +// CHECK: [[OUTPUT_0:%.*]], {{%.*}} = tf_executor.island wraps "tf.opA"() {_parallel_execution_ids = "r4:5,p0:0"} +// CHECK: [[OUTPUT_1:%.*]], {{%.*}} = tf_executor.island wraps "tf.opB"([[OUTPUT_0:%.*]]) {_parallel_execution_ids = "r4:5,p0:0"} +// CHECK: [[OUTPUT_2:%.*]], {{%.*}} = tf_executor.island wraps "tf.opC"() {_parallel_execution_ids = "r4:5,p0:1"} +// CHECK: [[OUTPUT_3:%.*]], {{%.*}} = tf_executor.island wraps "tf.opD"([[OUTPUT_2:%.*]]) {_parallel_execution_ids = "r4:5,p0:1"} +// CHECK: tf_executor.fetch [[OUTPUT_1:%.*]], [[OUTPUT_3:%.*]] diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands_legacy.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands_legacy.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ec09f26ba77314f201f5bd015f14e59bcf051d2c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/parallel_execute_to_islands_legacy.mlir @@ -0,0 +1,182 @@ +// RUN: tf-opt %s -tf-parallel-execute-to-islands=legacy-graph-export=true | env FILECHECK_OPTS="" FileCheck %s + +// CHECK-LABEL: func @testEmptyRegions +func.func @testEmptyRegions() { + tf_executor.graph { + tf_executor.island() { + "tf_device.parallel_execute"() ({ + tf_device.return + }, { + tf_device.return + }) {} : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: [[ISLAND_0_CTRL:%.+]] = tf_executor.island { +// CHECK: tf_executor.yield +// CHECK: [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK: tf_executor.yield +// CHECK: tf_executor.fetch [[ISLAND_0_CTRL]], [[ISLAND_1_CTRL]] : + + +// CHECK-LABEL: func @testDataOperandsAndResults +// CHECK-SAME: ([[ARG_0:%.+]]: tensor) +func.func @testDataOperandsAndResults(%arg0 : tensor) { + %0:2 = tf_executor.graph { + %1:2 = tf_executor.island { + %2 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %2 : tensor + } + %3:3 = tf_executor.island() { + %4:2 = "tf_device.parallel_execute"() ({ + %5 = "tf.opB"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }, { + %5 = "tf.opC"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %4#0, %4#1 : tensor, tensor + } + tf_executor.fetch %3#0, %3#1 : tensor, tensor + } + func.return +} + +// CHECK: [[INPUT_A:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"([[INPUT_A]]) +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: [[ISLAND_1_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_C_OUTPUT:%.+]] = "tf.opC"([[INPUT_A]]) +// CHECK: tf_executor.yield [[OP_C_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]], [[ISLAND_1_OUTPUT]] : + + +// CHECK-LABEL: func @testControlOperands +func.func @testControlOperands() { + %0:2 = tf_executor.graph { + %1 = tf_executor.island { + "tf.someOp"() : () -> () + tf_executor.yield + } + %2:3 = tf_executor.island(%1) { + %3:2 = "tf_device.parallel_execute"() ({ + %4 = "tf.opA"() : () -> tensor + tf_device.return %4 : tensor + }, { + %4 = "tf.opB"() : () -> tensor + tf_device.return %4 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %3#0, %3#1 : tensor, tensor + } + tf_executor.fetch %2#0, %2#1 : tensor, tensor + } + func.return +} + +// CHECK: [[INPUT_CTRL:%.+]] = tf_executor.island { +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island([[INPUT_CTRL]]) { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_1_OUTPUT:%.+]], {{%.+}} = tf_executor.island([[INPUT_CTRL]]) { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]], [[ISLAND_1_OUTPUT]] : + + +// CHECK-LABEL: func @testControlResults +func.func @testControlResults() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = "tf_device.parallel_execute"() ({ + %2 = "tf.opA"() : () -> tensor + tf_device.return %2 : tensor + }, { + %2 = "tf.opB"() : () -> tensor + tf_device.return %2 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + %3 = tf_executor.island(%0#2) { + "tf.someOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch %3 : !tf_executor.control + } + func.return +} + +// CHECK: {{%.+}}, [[ISLAND_0_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: {{%.+}}, [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: [[OUTPUT_CTRL:%.+]] = tf_executor.island([[ISLAND_0_CTRL]], [[ISLAND_1_CTRL]]) { +// CHECK: [[FETCH_ISLAND:%.+]] = tf_executor.island([[OUTPUT_CTRL]]) { +// CHECK: tf_executor.fetch [[FETCH_ISLAND]] : !tf_executor.control + + +// CHECK-LABEL: func @testSomeRegionNoUsers +func.func @testSomeRegionNoUsers() { + %0 = tf_executor.graph { + %1:3 = tf_executor.island { + %2:2 = "tf_device.parallel_execute"() ({ + %3 = "tf.opA"() : () -> tensor + tf_device.return %3 : tensor + }, { + %3 = "tf.opB"() : () -> tensor + tf_device.return %3 : tensor + }) {} : () -> (tensor, tensor) + tf_executor.yield %2#0, %2#1 : tensor, tensor + } + tf_executor.fetch %1#0 : tensor + } + func.return +} + +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"() +// CHECK: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: {{%.+}}, [[ISLAND_1_CTRL:%.+]] = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"() +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]], [[ISLAND_1_CTRL]] : + +// ----- + +// Tests a ParallelExecute with a single region. + +// CHECK-LABEL: func @testSingleton +// CHECK-SAME: ([[ARG_0:%.+]]: tensor) +func.func @testSingleton(%arg0 : tensor) { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + %2 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %2 : tensor + } + %3:2 = tf_executor.island() { + %4 = "tf_device.parallel_execute"() ({ + %5 = "tf.opB"(%1#0) : (tensor) -> tensor + tf_device.return %5 : tensor + }) {} : () -> tensor + tf_executor.yield %4 : tensor + } + tf_executor.fetch %3#0 : tensor + } + func.return +} + +// CHECK: [[INPUT_A:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_A_OUTPUT:%.+]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: tf_executor.yield [[OP_A_OUTPUT]] : +// CHECK: [[ISLAND_0_OUTPUT:%.+]], {{%.+}} = tf_executor.island { +// CHECK-NEXT: [[OP_B_OUTPUT:%.+]] = "tf.opB"([[INPUT_A]]) +// CHECK: tf_executor.yield [[OP_B_OUTPUT]] : +// CHECK: tf_executor.fetch [[ISLAND_0_OUTPUT]] : diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/prepare_tpu_computation_for_tf_export.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/prepare_tpu_computation_for_tf_export.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4bc42e5d2af052694c3849e6d6d715e8ce03d0db --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/prepare_tpu_computation_for_tf_export.mlir @@ -0,0 +1,191 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -prepare-tpu-computation-for-tf-export | FileCheck %s + +// CHECK-LABEL: @ShardingAttr +func.func @ShardingAttr(%arg0: tensor<128x10xf32> {mhlo.sharding = "\08\03\1A\02\01\02\22\02\00\01"}, %arg1: tensor<10x1024xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor<128x1024xf32> {mhlo.sharding = ""}) -> (tensor<128x10xf32>, tensor<10x1024xf32>, tensor<128x1024xf32>) { + + // CHECK: %[[SHARDED_ARG0:.*]] = "tf.XlaSharding"(%arg0) <{_XlaSharding = "\08\03\1A\02\01\02\22\02\00\01", sharding = "\08\03\1A\02\01\02\22\02\00\01"}> + // CHECK: %[[SHARDED_ARG1:.*]] = "tf.XlaSharding"(%arg1) <{_XlaSharding = "\08\01\1A\01\01\22\01\00", sharding = "\08\01\1A\01\01\22\01\00"}> + + // CHECK: "tf.Identity"(%[[SHARDED_ARG1]]) + %0 = "tf.Identity"(%arg1) : (tensor<10x1024xf32>) -> tensor<10x1024xf32> + + // CHECK: "tf.Identity"(%arg2) + %1 = "tf.Identity"(%arg2) : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + func.return %arg0, %0, %1 : tensor<128x10xf32>, tensor<10x1024xf32>, tensor<128x1024xf32> +} + +// CHECK-LABEL: @RewriteHostComputeMlirOp +func.func @RewriteHostComputeMlirOp(%arg0: tensor<*xf32>, %arg1: tensor<3x?xf64>) -> (tensor<*xf32>) { + + // CHECK: "tf.XlaHostCompute"(%arg0, %arg1) + // CHECK-SAME: ancestors = [] + // CHECK-SAME: cost_estimate_ns = 1000000 : i64 + // CHECK-SAME: key = "" + // CHECK-SAME: recv_key = "host_compute_channel_recv" + // CHECK-SAME: send_key = "host_compute_channel_send" + // CHECK-SAME: shape_inference_graph = @host_func + // CHECK-SAME: shapes = [#tf_type.shape<*>, #tf_type.shape<3x?>] + // CHECK-SAME: tpu_core = 0 : i64 + // CHECK: func @host_func + // CHECK: %[[RECV_OUTPUT:[0-9]*]]:2 = "tf._XlaRecvAtHost" + // CHECK-SAME: key = "host_compute_channel_send" + // CHECK-SAME: -> (tensor<*xf32>, tensor<3x?xf64>) + // CHECK: %[[ID_OUTPUT:[0-9]*]] = "tf.Identity"(%[[RECV_OUTPUT]]#0) + // CHECK: "tf._XlaSendFromHost" + // CHECK-SAME: [[ID_OUTPUT]] + // CHECK-SAME: key = "host_compute_channel_recv" + + %0:2 = "tf._XlaHostComputeMlir"(%arg0, %arg1) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = "module {\0A func.func @host_func(%arg0: tensor<*xf32>, %arg1: tensor<3x?xf64>) -> (tensor<*xf32>, tensor<3x?xf64>) {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A func.return %0, %arg1 : tensor<*xf32>, tensor<3x?xf64> \0A } \0A} \0A"} : (tensor<*xf32>, tensor<3x?xf64>) -> (tensor<*xf32>, tensor<3x?xf64>) + func.return %0#0 : tensor<*xf32> +} + +// CHECK-LABEL: @RewriteSendRecvOps +func.func @RewriteSendRecvOps() -> () { + // CHECK: key = "recv_key_htod_0" + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + + // CHECK: key = "send_key_dtoh_0" + "tf.XlaSendToHost"(%0) {key = "send_key"} : (tensor) -> () + + func.return +} + +// CHECK-LABEL: @CommunicateOpTokenAttrs +func.func @CommunicateOpTokenAttrs() -> () { + // CHECK: _xla_original_oc_node_name = [[NODE_NAME1:.*]], _xla_token_input_nodes = ["_xla_token_arg_node"] + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + + // CHECK: _xla_original_oc_node_name = [[NODE_NAME2:.*]], _xla_token_input_nodes = {{\[}}[[NODE_NAME1]]{{\]}} + "tf.XlaSendToHost"(%0) {key = "send_key"} : (tensor) -> () + + // CHECK: _xla_original_oc_node_name = [[NODE_NAME3:.*]], _xla_token_input_nodes = {{\[}}[[NODE_NAME2]]{{\]}} + %1 = "tf._XlaHostComputeMlir"(%0) {recv_key = "host_compute_channel_recv1", send_key = "host_compute_channel_send1", host_mlir_module = ""} : (tensor) -> (tensor) + func.return +} + +// CHECK-LABEL: @IfOpTokenAttrs +func.func @IfOpTokenAttrs(%arg0: tensor, %arg1: tensor) -> (tensor) { + // CHECK: tf.IfRegion + %0 = "tf.IfRegion"(%arg0) ({ + // CHECK: tf.XlaRecvFromHost + // CHECK-SAME-DAG: _xla_original_oc_node_name = + // CHECK-SAME-DAG: _xla_token_input_nodes = ["_xla_token_arg_node"] + %recv = "tf.XlaRecvFromHost"() {key = "if_op_token_recv_key", shape = #tf_type.shape<>} : () -> tensor + + // CHECK: tf.Yield + "tf.Yield"(%recv) : (tensor) -> () + }, { + // CHECK-NOT: _xla_token_input_nodes + %add = "tf.Add"(%arg1, %arg1) : (tensor, tensor) -> tensor + + // CHECK: tf.Yield + "tf.Yield"(%add) : (tensor) -> () + }) { is_stateless = true}: (tensor) -> tensor + // CHECK: _xla_token_input_nodes = ["_xla_token_arg_node"] + + func.return %0 : tensor +} + +// Verifies that If ops that don't have any communication ops don't have token +// input nodes attribute even if the parent region has token argument. + +// CHECK-LABEL: @IfOpWithoutCommunicationOps +func.func @IfOpWithoutCommunicationOps(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: tf.IfRegion + %0 = "tf.IfRegion"(%arg0) ({ + %mul = "tf.Add"(%arg1, %arg1) : (tensor, tensor) -> tensor + "tf.Yield"(%mul) : (tensor) -> () + }, { + %add = "tf.Add"(%arg1, %arg1) : (tensor, tensor) -> tensor + "tf.Yield"(%add) : (tensor) -> () + }) { is_stateless = true}: (tensor) -> tensor + // CHECK-NOT: _xla_token_input_nodes + + // CHECK: tf.IfRegion + %1 = "tf.IfRegion"(%arg0) ({ + %recv = "tf.XlaRecvFromHost"() {key = "if_op_token_recv_key", shape = #tf_type.shape<>} : () -> tensor + "tf.Yield"(%recv) : (tensor) -> () + }, { + %add = "tf.Add"(%arg1, %arg1) : (tensor, tensor) -> tensor + "tf.Yield"(%add) : (tensor) -> () + }) { is_stateless = true}: (tensor) -> tensor + // CHECK: _xla_token_input_nodes = ["_xla_token_arg_node"] + + func.return %0, %1 : tensor, tensor +} + + +// Next four functions are used to verify handling of a call chain. + +// CHECK-LABEL: func @IdentityFunc +func.func @IdentityFunc(%arg0: tensor) -> tensor { + // CHECK-NOT: _xla_token_input_nodes + %1 = "tf.Identity"(%arg0) : (tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @PartitionedCall3 +func.func @PartitionedCall3(%arg0: tensor) -> tensor { + // CHECK: _xla_original_oc_node_name = [[NODE_NAME1:.*]], _xla_token_input_nodes = ["_xla_token_arg_node"] + "tf.XlaSendToHost"(%arg0) {key = "send_key_call3"} : (tensor) -> () + // CHECK: _xla_original_oc_node_name = [[NODE_NAME2:.*]], _xla_token_input_nodes = {{\[}}[[NODE_NAME1]]{{\]}} + %1 = "tf.XlaRecvFromHost"() {key = "recv_key_call3", shape = #tf_type.shape<>} : () -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @PartitionedCall2 +func.func @PartitionedCall2(%arg0: tensor) -> tensor { + // CHECK: _xla_original_oc_node_name = [[NODE_NAME1:.*]], _xla_token_input_nodes = ["_xla_token_arg_node"] + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @PartitionedCall3} : (tensor) -> (tensor) + // CHECK-NOT: _xla_token_input_nodes + %1 = "tf.PartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @IdentityFunc} : (tensor) -> (tensor) + func.return %1 : tensor +} + +// CHECK-LABEL: func @PartitionedCall1 +func.func @PartitionedCall1(%arg0: tensor) -> tensor { + // CHECK: _xla_original_oc_node_name = [[NODE_NAME1:.*]], _xla_token_input_nodes = ["_xla_token_arg_node"] + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @PartitionedCall2} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +func.func @Callee(%arg0: tensor) -> tensor { + "tf.XlaSendToHost"(%arg0) {key = "send_key_call3"} : (tensor) -> () + %1 = "tf.XlaRecvFromHost"() {key = "recv_key_call3", shape = #tf_type.shape<>} : () -> tensor + func.return %1 : tensor +} + +func.func @UnsupportedOp(%arg0: tensor) -> tensor { + // expected-error @+1 {{does not support subcomputations with tf/xla communication ops}} + %0 = "tf.CustomTestOp"(%arg0) {config = "", config_proto = "", executor_type = "", f = @Callee} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +// _XlaHostComputeMlir with manual_sharding should not fall back to +// XlaHostCompute, because XlaHostCompute does not support manual_sharding. +// Instead, it is skipped and the MlirXlaOpKernel is expected to handle it. + +func.func @HostComputeManualNoFallback(%arg0: tensor) -> () { + // CHECK: "tf._XlaHostComputeMlir" + %1 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv1", send_key = "host_compute_channel_send1", host_mlir_module = "", manual_sharding = true} : (tensor) -> (tensor) + func.return +} + +// ----- + +// CHECK-LABEL: test_xla_call_module_with_host_communicative_subcomputation +func.func @test_xla_call_module_with_host_communicative_subcomputation() { + "tf.XlaCallModule"() {Sout = [], device = "", dim_args_spec = [], function_list = [@callee], module = "", platforms = [], version = 4 : i64} : () -> () + func.return +} + +// CHECK-LABEL: callee +func.func private @callee(%arg0: tensor) { + "tf.XlaHostCompute"(%arg0) <{ancestors = [], key = "@host_func", recv_key = "", send_key = "", shapes = []}> {_xla_original_oc_node_name = "hcb0", _xla_token_input_nodes = ["_xla_token_arg_node"]} : (tensor) -> () + return + } + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/print.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/print.mlir new file mode 100644 index 0000000000000000000000000000000000000000..64371e5de49309c3940990b11cb3dbb15f27f361 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/print.mlir @@ -0,0 +1,12 @@ +// RUN: tf-opt %s --tf-print | FileCheck %s + +module { +// Smoke test. We don't expect any modifications of the MLIR. + +// CHECK-LABEL: foo +// CHECK: return +func.func @foo(%arg0: tensor, %arg1: tensor) -> tensor { + return %arg0 : tensor +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args.mlir new file mode 100644 index 0000000000000000000000000000000000000000..cb48a3064194ef4215a86688651f97db934b0f18 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args.mlir @@ -0,0 +1,398 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-promote-resources-to-args | env FILECHECK_OPTS="" FileCheck %s + +// One resource, one read. The initial value of the resource is read. +// CHECK-LABEL: func @main(%arg0: tensor, %arg1: tensor {tf.resource_name = "x"}) -> tensor<2xf32> +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() + // CHECK: %[[ADD:[0-9]*]] = "tf.AddV2"(%arg1, %[[CONST]]) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%[[CONST]], %[[ADD]]) + // CHECK: return %[[PACK]] + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.AddV2"(%2, %0) : (tensor, tensor) -> tensor + %4 = "tf.Pack"(%0, %3) : (tensor, tensor) -> tensor<2xf32> + func.return %4 : tensor<2xf32> +} + +// ----- + +// One resource, one read. _is_initialized is false, shouldn't be promoted. +// CHECK-LABEL: func @main() +func.func @main() -> tensor { + // CHECK: "tf.VarHandleOp" + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x", _is_initialized = false} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + func.return %2 : tensor +} + +// ----- + +// One resource, one read. _is_initialized is true, should be promoted. +// CHECK-LABEL: func @main +// CHECK-SAME: ({{%.+}}: tensor {tf.resource_name = "x"}) +func.func @main() -> tensor { + // CHECK-NOT: "tf.VarHandleOp" + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x", _is_initialized = true} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + func.return %2 : tensor +} + +// ----- + +// One resource, one write. The initial value of the resource is not read. +// CHECK-LABEL: func @main(%arg0: tensor) -> (tensor {tf.resource_name = "x"}) +func.func @main(%arg0: tensor) { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK: %[[RES:.*]] = "tf.Const"() + // CHECK: return %[[RES]] + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor) -> () + func.return +} + +// ----- + +// One resource, two reads using different resource handles. +// CHECK-LABEL: func @main(%arg0: tensor, %arg1: tensor {tf.resource_name = "x"}) -> tensor<2xf32> +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<4.200000e+01> : tensor}> + // CHECK: %[[ADD1:[0-9]*]] = "tf.AddV2"(%arg1, %[[CONST]]) + // CHECK: %[[ADD2:[0-9]*]] = "tf.AddV2"(%[[ADD1]], %arg1) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%[[CONST]], %[[ADD2]]) + // CHECK: return %[[PACK]] + + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.AddV2"(%2, %0) : (tensor, tensor) -> tensor + %4 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %5 = "tf.ReadVariableOp"(%4) : (tensor>>) -> tensor + %6 = "tf.AddV2"(%3, %5) : (tensor, tensor) -> tensor + %7 = "tf.Pack"(%0, %6) : (tensor, tensor) -> tensor<2xf32> + func.return %7 : tensor<2xf32> +} + +// ----- + +// Two resources, two reads using different resources. +// CHECK-LABEL: func @main(%arg0: tensor, %arg1: tensor {tf.resource_name = "x"}, %arg2: tensor {tf.resource_name = "y"}) -> tensor<2xf32> +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() + // CHECK: %[[ADD1:[0-9]*]] = "tf.AddV2"(%arg1, %[[CONST]]) + // CHECK: %[[ADD2:[0-9]*]] = "tf.AddV2"(%[[ADD1]], %arg2) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%[[CONST]], %[[ADD2]]) + // CHECK: return %[[PACK]] + + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.AddV2"(%2, %0) : (tensor, tensor) -> tensor + %4 = "tf.VarHandleOp"() {container = "", shared_name = "y"} : () -> tensor>> + %5 = "tf.ReadVariableOp"(%4) : (tensor>>) -> tensor + %6 = "tf.AddV2"(%3, %5) : (tensor, tensor) -> tensor + %7 = "tf.Pack"(%0, %6) : (tensor, tensor) -> tensor<2xf32> + func.return %7 : tensor<2xf32> +} + +// ----- + +// One resource with read and write. The initial value of the resource is read. +// CHECK-LABEL: func @main(%arg0: tensor, %arg1: tensor {tf.aliasing_output = 1 : i64, tf.resource_name = "x"}) -> (tensor<2xf32>, tensor) +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() + // CHECK: %[[ADD1:[0-9]*]] = "tf.AddV2"(%arg1, %[[CONST]]) + // CHECK: %[[ADD2:[0-9]*]] = "tf.AddV2"(%[[ADD1]], %[[ADD1]]) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%arg1, %[[ADD2]]) + // CHECK: return %[[PACK]], %[[ADD1]] + + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %4 = "tf.AddV2"(%3, %0) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%1, %4) : (tensor>>, tensor) -> () + %5 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %6 = "tf.AddV2"(%4, %5) : (tensor, tensor) -> tensor + %7 = "tf.Pack"(%2, %6) : (tensor, tensor) -> tensor<2xf32> + func.return %7 : tensor<2xf32> +} + +// ----- + +// One resource with read and write. The initial value of the resource is not read. +// CHECK-LABEL: func @main(%arg0: tensor) -> (tensor<2xf32>, tensor {tf.resource_name = "x"}) +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<4.200000e+01> : tensor}> + // CHECK: %[[ADD1:[0-9]*]] = "tf.AddV2"(%[[CONST]], %[[CONST]]) + // CHECK: %[[ADD2:[0-9]*]] = "tf.AddV2"(%[[ADD1]], %[[ADD1]]) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%[[CONST]], %[[ADD2]]) + // CHECK: return %[[PACK]], %[[ADD1]] + + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor) -> () + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %4 = "tf.AddV2"(%3, %0) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%1, %4) : (tensor>>, tensor) -> () + %5 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %6 = "tf.AddV2"(%4, %5) : (tensor, tensor) -> tensor + %7 = "tf.Pack"(%2, %6) : (tensor, tensor) -> tensor<2xf32> + func.return %7 : tensor<2xf32> +} + +// ----- + +// A resource is passed into tf.If +func.func @cond_false(%arg0: tensor>>, %arg1: tensor) -> tensor { + func.return %arg1 : tensor +} + +func.func @cond_true(%arg0: tensor>>, %arg1: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %1 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %2 = "tf.AddV2"(%1, %0) {T = f32, device = ""} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func @main(%arg0: tensor, %arg1: tensor {tf.resource_name = "x"}) -> tensor<2xf32> +func.func @main(%arg0: tensor) -> tensor<2xf32> attributes {tf.entry_function = {inputs = "", outputs = "result"}} { + %0 = "tf.Const"() {value = dense<1.050000e+03> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.Less"(%2, %0) : (tensor, tensor) -> tensor + %4 = "tf.If"(%3, %1, %2) {Tcond = i1, Tin = ["tfdtype$DT_RESOURCE", "tfdtype$DT_FLOAT"], Tout = ["tfdtype$DT_FLOAT"], + else_branch = @cond_false, is_stateless = false,then_branch = @cond_true} : + (tensor, tensor>>, tensor) -> tensor + %5 = "tf.Identity"(%4) : (tensor) -> tensor + %6 = "tf.Pack"(%2, %5) {N = 2 : i64, T = f32, axis = 0 : i64, device = ""} : (tensor, tensor) -> tensor<2xf32> + func.return %6 : tensor<2xf32> +} + +// ----- + +// Tests resource passed in as an argument is not modified and not returned. + +// CHECK-LABEL: func @main +// CHECK-SAME: %arg0: tensor +// CHECK-SAME: %[[ARG_1:[a-z0-9]+]]: tensor +func.func @main(%arg0: tensor, %arg1: tensor>>) { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + // CHECK-NEXT: "tf.AddV2"(%[[ARG_1]], %[[ARG_1]]) + %1 = "tf.AddV2"(%0, %0) : (tensor, tensor) -> tensor + // CHECK-NEXT: return + func.return +} + +// ----- + +// Tests resource passed in as an argument is modified but not returned. + +// CHECK-LABEL: func @main +// CHECK-SAME: %{{[a-z0-9]+}}: tensor {tf.aliasing_output = 0 : i64} +// CHECK-SAME: %arg1: tensor +// CHECK-SAME: -> tensor +func.func @main(%arg0: tensor>>, %arg1: tensor) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const" + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]] : tensor + func.return +} + +// ----- + +// Tests last resource assign is returned as a result. + +// CHECK-LABEL: func @main +// CHECK-SAME: %{{[a-z0-9]+}}: tensor {tf.aliasing_output = 0 : i64} +// CHECK-SAME: %arg1: tensor +// CHECK-SAME: -> tensor +func.func @main(%arg0: tensor>>, %arg1: tensor) { + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<1.050000e+03> : tensor}> + %1 = "tf.Const"() {value = dense<1.050000e+03> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %1) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]] : tensor + func.return +} + +// ----- + +// Tests last resource assign is returned even when the original function +// returns the same value prior. + +// CHECK-LABEL: func @main +// CHECK-SAME: %{{[a-z0-9]+}}: tensor {tf.aliasing_output = 1 : i64} +// CHECK-SAME: %arg1: tensor +// CHECK-SAME: -> (tensor, tensor) +func.func @main(%arg0: tensor>>, %arg1: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<1.050000e+03> : tensor}> + %1 = "tf.Const"() {value = dense<1.050000e+03> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %1) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]], %[[CONST]] : tensor, tensor + func.return %1 : tensor +} + +// ----- + +// Tests read interleaved between writes. + +// CHECK-LABEL: func @main +// CHECK-SAME: %{{[a-z0-9]+}}: tensor {tf.aliasing_output = 1 : i64} +// CHECK-SAME: %arg1: tensor +// CHECK-SAME: -> (tensor, tensor) +func.func @main(%arg0: tensor>>, %arg1: tensor) -> tensor { + // CHECK-NEXT: %[[CONST_0:.*]] = "tf.Const"() <{value = dense<4.200000e+01> : tensor}> + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + %1 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK-NEXT: %[[ADD:[a-z0-9]+]] = "tf.AddV2"(%[[CONST_0]], %[[CONST_0]]) + %2 = "tf.AddV2"(%1, %1) : (tensor, tensor) -> tensor + // CHECK-NEXT: %[[CONST_1:.*]] = "tf.Const"() <{value = dense<1.050000e+03> : tensor}> + %3 = "tf.Const"() {value = dense<1.050000e+03> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %3) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[ADD]], %[[CONST_1]] : tensor, tensor + func.return %2 : tensor +} + +// ----- + +// Tests resource write takes on value that is from an argument. + +// CHECK-LABEL: func @main +// CHECK-SAME: %[[ARG_0:[a-z0-9]+]]: tensor {tf.aliasing_output = 0 : i64} +// CHECK-SAME: %[[ARG_1:[a-z0-9]+]]: tensor +// CHECK-SAME: -> tensor +func.func @main(%arg0: tensor>>, %arg1: tensor) { + "tf.AssignVariableOp"(%arg0, %arg1) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[ARG_1]] : tensor + func.return +} + +// ----- + +// Tests removal of dead local variables. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<2xf32>) { + // CHECK-NOT: tf.MlirLocalVarOp + // CHECK-NOT: tf.AssignVariableOp + %0 = "tf.MlirLocalVarOp"() : () -> tensor>> + "tf.AssignVariableOp"(%0, %arg0) : (tensor>>, tensor<2xf32>) -> () + func.return +} + +// ----- + +// Tests first read of one resource is used as a value to write to another +// resource. + +// CHECK-LABEL: func @main +// CHECK-SAME: %{{[a-z0-9]+}}: tensor {tf.aliasing_output = 0 : i64} +// CHECK-SAME: %[[ARG_1:[a-z0-9]+]]: tensor +// CHECK-SAME: -> tensor +func.func @main(%arg0: tensor>>, %arg1: tensor>>) { + %1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + "tf.AssignVariableOp"(%arg0, %1) : (tensor>>, tensor) -> () + // CHECK-NEXT: return %[[ARG_1]] : tensor + func.return +} + +// ----- + +// Tests if local variables that are dead after resource op lifting are removed. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: tf.MlirLocalVarOp + // CHECK-NOT: tf.AssignVariableOp + %0 = "tf.MlirLocalVarOp"() : () -> tensor>> + %1 = "tf._SomeOp"() : () -> tensor<2xf32> + "tf.AssignVariableOp"(%0, %1) : (tensor>>, tensor<2xf32>) -> () + %2 = "tf.PartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor>>) -> tensor<2xf32> + func.return %2 : tensor<2xf32> +} +func.func private @callee(%arg0: tensor>>) -> tensor<2xf32> { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + + +// ----- + +// Tests main function with multiple blocks. + +// expected-error@+1 {{expects function 'main' to have 1 block, got 2}} +func.func @main() { + cf.br ^bb1 +^bb1: + func.return +} + +// ----- + +// Tests main function is terminated with a non MLIR ReturnOp. + +// expected-error@+1 {{expects function 'main' to have a MLIR ReturnOp}} +func.func @main() { +^bb0: + tf_device.return +} + +// ----- + +// Tests main function with invalid resource argument subtype. + +// expected-error@+1 {{expects resource type of argument 0 to have one subtype, got '!tf_type.resource'}} +func.func @main(%arg0: tensor) { + func.return +} + +// ----- + +// Tests main function with invalid VarHandleOp resource subtype. + +func.func @main() { + // expected-error @+1 {{must have exactly one subtype in the result resource type}} + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor + func.return +} + +// ----- + +// Tests resource argument has users that are not ReadVariableOp or +// AssignVariableOp. + +// expected-error@+1 {{expects users of resource argument 0 to be 'tf.ReadVariableOp' or 'tf.AssignVariableOp', got [tf.UnknownOp, tf.VarIsInitializedOp]}} +func.func @main(%arg0: tensor>>) -> tensor { + %0 = "tf.VarIsInitializedOp"(%arg0) : (tensor>>) -> tensor + %1 = "tf.UnknownOp"(%arg0) : (tensor>>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Tests VarHandleOp has users that are not removed. + +func.func @main() -> tensor { + // expected-error@+1 {{expects users to be 'tf.ReadVariableOp' or 'tf.AssignVariableOp', got [tf.UnknownOp, tf.VarIsInitializedOp]}} + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %1 = "tf.VarIsInitializedOp"(%0) : (tensor>>) -> tensor + %2 = "tf.UnknownOp"(%0) : (tensor>>) -> tensor + func.return %1 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args_functions.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args_functions.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b1cd3fee12fc769b15ba5792effa87782c5d4bb9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_resources_to_args_functions.mlir @@ -0,0 +1,32 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-promote-resources-to-args=functions="add_and_pack,read" | env FILECHECK_OPTS="" FileCheck %s + +module { + + // One resource, one read. The initial value of the resource is read. + // CHECK-LABEL: func @add_and_pack(%arg0: tensor, %arg1: tensor {tf.resource_name = "x"}) -> tensor<2xf32> + func.func @add_and_pack(%arg0: tensor) -> tensor<2xf32> { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[CONST:.*]] = "tf.Const"() + // CHECK: %[[ADD:[0-9]*]] = "tf.AddV2"(%arg1, %[[CONST]]) + // CHECK: %[[PACK:[0-9]*]] = "tf.Pack"(%[[CONST]], %[[ADD]]) + // CHECK: return %[[PACK]] + %0 = "tf.Const"() {value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + %3 = "tf.AddV2"(%2, %0) : (tensor, tensor) -> tensor + %4 = "tf.Pack"(%0, %3) : (tensor, tensor) -> tensor<2xf32> + func.return %4 : tensor<2xf32> + } + + // One resource, one read. _is_initialized is true, should be promoted. + // CHECK-LABEL: func @read(%arg0: tensor {tf.resource_name = "x"}) -> tensor + func.func @read() -> tensor { + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: return %arg0 + %1 = "tf.VarHandleOp"() {container = "", shared_name = "x", _is_initialized = true} : () -> tensor>> + %2 = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor + func.return %2 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_var_handles_to_args.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_var_handles_to_args.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4b5b65faf766ce804a3103227e99ee23d6934e01 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/promote_var_handles_to_args.mlir @@ -0,0 +1,59 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-promote-var-handles-to-args | FileCheck %s + +// Tests main function with multiple blocks. + +// expected-error@+1 {{expects function 'main' to have 1 block, got 2}} +func.func @main() { + cf.br ^bb1 +^bb1: + func.return +} + +// ----- + +// CHECK-LABEL: func @no_args +// CHECK-SAME: (%arg0: tensor>> {tf.resource_name = "x"}) +// CHECK-NOT: "tf.VarHandleOp" +func.func @no_args() { + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + func.return +} + +// CHECK-LABEL: func @some_args +// CHECK-SAME: (%arg0: tensor, %arg1: tensor>> {tf.resource_name = "x"}) +// CHECK-NOT: "tf.VarHandleOp" +func.func @some_args(%arg0: tensor) { + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + func.return +} + +// CHECK-LABEL: func @unique_vars +// CHECK-SAME: (%arg0: tensor>> {tf.resource_name = "x"}, %arg1: tensor>> {tf.resource_name = "y"}) +// CHECK-NOT: "tf.VarHandleOp" +func.func @unique_vars() { + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "y"} : () -> tensor>> + func.return +} + +// CHECK-LABEL: func @duplicate_vars +// CHECK-SAME: (%arg0: tensor>> {tf.resource_name = "x"}) +// CHECK-NOT: "tf.VarHandleOp" +func.func @duplicate_vars() { + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %1 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + func.return +} + +// CHECK-LABEL: func @duplicate_vars_with_users +// CHECK-SAME: (%arg0: tensor, %arg1: tensor>> {tf.resource_name = "x"}) +// CHECK: "tf.ReadVariableOp"(%arg1) +// CHECK: "tf.AssignAddVariableOp"(%arg1, %arg0) +// CHECK-NOT: "tf.VarHandleOp" +func.func @duplicate_vars_with_users(%arg0: tensor) { + %0 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf.VarHandleOp"() {container = "", shape = "tfshape$", shared_name = "x"} : () -> tensor>> + "tf.AssignAddVariableOp"(%2, %arg0) : (tensor>>, tensor) -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/readonly_references_to_resources.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/readonly_references_to_resources.mlir new file mode 100644 index 0000000000000000000000000000000000000000..96a940b2b29ece6987c43f28c53f057694c06916 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/readonly_references_to_resources.mlir @@ -0,0 +1,109 @@ +// RUN: tf-opt -verify-diagnostics -tf-readonly-references-to-resources -split-input-file %s | FileCheck %s + +// Test case: Basic converting. + +func.func @f() { + // CHECK: "tf.VarHandleOp" + // CHECK: "tf.ReadVariableOp" + %val0 = "tf.VariableV2"() {_class = ["loc:@v"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: Basic converting. '_class' attribute is at IdentityOp. + +func.func @f() { + // CHECK: "tf.VarHandleOp" + // CHECK: "tf.ReadVariableOp" + %val0 = "tf.VariableV2"() {container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) {_class = ["loc:@v"]} : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: Two ReadVariable ops. + +func.func @f() { + // CHECK: "tf.VarHandleOp" + + // During lowering to resource variables, this pass will preserve the + // locations of the ReadVariableOps as Identity ops to keep the original graph + // composition and order. + + // CHECK: "tf.ReadVariableOp" + // CHECK: "tf.ReadVariableOp" + %val0 = "tf.VariableV2"() {_class = ["loc:@v"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + %val2 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: No follow-up ReadVariable case. + +func.func @f() { + // CHECK-NOT: "tf.VariableV2" + // CHECK-NOT: "tf.VarHandleOp" + %val0 = "tf.VariableV2"() {_class = ["loc:@v"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + func.return +} + +// ----- + +// Test case: No converting when there is another use case. + +func.func @f() { + // expected-error @+1 {{'tf.VariableV2' op expects all users to be 'tf.Identity', but got user tf.CustomIdentity}} + %val0 = "tf.VariableV2"() {_class = ["loc:@v"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.CustomIdentity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: Get variable name from the shared_name attribute from VariableV2 op. + +func.func @f() { + // CHECK: "tf.VarHandleOp" + // CHECK: "tf.ReadVariableOp" + %val0 = "tf.VariableV2"() {container = "", device = "", shape = #tf_type.shape<96>, shared_name = "test"} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: No class and shared_name attributes on VariableV2 op. + +func.func @f() { + // expected-error @+1 {{'tf.VariableV2' op has no '_class' and 'shared_name' attributes}} + %val0 = "tf.VariableV2"() {container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: No named location found on VariableV2 op. + +func.func @f() { + // expected-error @+1 {{'tf.VariableV2' op expects variable name in '_class' attribute, but got ["unrelated_class"]}} + %val0 = "tf.VariableV2"() {_class = ["unrelated_class"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} + +// ----- + +// Test case: Invalid multiple location information in a class attribute on VariableV2 op. + +func.func @f() { + // expected-error @+1 {{'tf.VariableV2' op expects only one named location in '_class' attribute, but got ["loc:@v1", "loc:@v2"]}} + %val0 = "tf.VariableV2"() {_class = ["loc:@v1", "loc:@v2"], container = "", device = "", shape = #tf_type.shape<96>, shared_name = ""} : () -> tensor<96x!tf_type.f32ref> + %val1 = "tf.Identity"(%val0) : (tensor<96x!tf_type.f32ref>) -> tensor<96xf32> + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/reducer/unsupported-op-test.sh b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/reducer/unsupported-op-test.sh new file mode 100644 index 0000000000000000000000000000000000000000..be3c7aa7688cbe7896567c3edb9bd2fa71510973 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/reducer/unsupported-op-test.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Script for mlir-reduce to test input interestingness. +tf-opt $1 -tf-mark-ops-for-outside-compilation 2>&1 | grep -q "'tf.UnsupportedOp' op isn't compilable for TPU device" + +if [ $? -eq 0 ]; then + exit 1 +else + exit 0 +fi diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/region-control-flow-to-functional.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/region-control-flow-to-functional.mlir new file mode 100644 index 0000000000000000000000000000000000000000..91160998a95e5b85e559f134e5db069a59885a03 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/region-control-flow-to-functional.mlir @@ -0,0 +1,1018 @@ +// RUN: tf-opt %s -tf-region-control-flow-to-functional -split-input-file | FileCheck %s + +// Simple IfRegion +// CHECK: func private @test_else_name(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @test_then_name(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Abs" +func.func @testSimple(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.If" + // CHECK-NOT: attr1 + // CHECK-SAME: else_branch = @test_else_name + // CHECK-SAME: then_branch = @test_then_name + // CHECK-SAME: _attr0 = false + // CHECK-SAME: _xla_propagate_compile_time_consts = true + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Abs"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = true, _attr0 = false, attr1 = "hello", _then_func_name = "test_then_name", _else_func_name = "test_else_name"} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Simple IfRegion with empty branch names +// CHECK: func private @tf.IfRegion_else(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @tf.IfRegion_then(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Abs" +func.func @testSimpleEmptyBranchNames(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.If" + // CHECK-NOT: attr1 + // CHECK-SAME: else_branch = @tf.IfRegion_else + // CHECK-SAME: then_branch = @tf.IfRegion_then + // CHECK-SAME: _attr0 = false + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Abs"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = true, _attr0 = false, attr1 = "hello", _then_func_name = "", _else_func_name = ""} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Use if condition inside the regions +// CHECK: func private @tf.IfRegion_else(%arg0: tensor, %arg1: tensor<2xf32>, %arg2: tensor<2xf32>, %arg3: tensor<2xf32>) -> tensor<2xf32> +// CHECK-NEXT: "tf.Select"(%arg0, %arg2, %arg3) +// CHECK: func private @tf.IfRegion_then(%arg0: tensor, %arg1: tensor<2xf32>, %arg2: tensor<2xf32>, %arg3: tensor<2xf32>) -> tensor<2xf32> +// CHECK-NEXT: "tf.Select"(%arg0, %arg1, %arg2) +func.func @testIfCondition(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.Add"(%arg1, %arg1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + %1 = "tf.Mul"(%arg1, %arg1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + %2 = "tf.Div"(%arg1, %arg1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + + // CHECK: "tf.If"{{.+}}else_branch = @tf.IfRegion_else{{.+}}then_branch = @tf.IfRegion_then + %3 = "tf.IfRegion"(%arg0) ({ + %4 = "tf.Select"(%arg0, %0, %1) : (tensor, tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%4) : (tensor<2xf32>) -> () + }, { + %5 = "tf.Select"(%arg0, %1, %2): (tensor, tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%5) : (tensor<2xf32>) -> () + }) { is_stateless = true} : (tensor) -> tensor<2xf32> + func.return %3 : tensor<2xf32> +} + +// ----- + +// Constant sinking for IfRegion + +// CHECK: func private @tf.IfRegion_else() -> tensor<2xf32> +// CHECK-NEXT: constant dense<1.0 +// CHECK: func private @tf.IfRegion_then() -> tensor<2xf32> +// CHECK-NEXT: constant dense<0.0 +func.func @testIfConstant(%arg0: tensor) -> tensor<2xf32> { + %cst_zero = arith.constant dense<0.0> : tensor<2xf32> + // CHECK: "tf.If"(%arg0) <{else_branch = @tf.IfRegion_else{{.+}}then_branch = @tf.IfRegion_then + %0 = "tf.IfRegion"(%arg0) ({ + "tf.Yield"(%cst_zero) : (tensor<2xf32>) -> () + }, { + %cst_one = arith.constant dense<1.0> : tensor<2xf32> + "tf.Yield"(%cst_one) : (tensor<2xf32>) -> () + }) { is_stateless = true} : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// Nested IfRegions +// CHECK: func private @tf.IfRegion1_else +// CHECK-NEXT: "tf.Acos" +// CHECK-NEXT: "tf.Abs" + +// CHECK: func private @tf.IfRegion1_then +// CHECK-NEXT: "tf.LogicalNot" +// CHECK-NEXT: "tf.Asin" +// CHECK-NEXT: "tf.If"({{.+}}) <{else_branch = @tf.IfRegion_else, {{.+}} then_branch = @tf.IfRegion_then} + +// CHECK: func private @tf.IfRegion_else +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @tf.IfRegion_then +// CHECK-NEXT: "tf.Abs" + +func.func @testNested(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.If"({{.+}}) <{else_branch = @tf.IfRegion1_else, {{.+}} then_branch = @tf.IfRegion1_then} + %0 = "tf.IfRegion"(%arg0) ({ + // Outer Then + %cond = "tf.LogicalNot"(%arg0) : (tensor) -> tensor + %asin = "tf.Asin"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + + // nested IfRegion + %1 = "tf.IfRegion"(%cond) ({ + %2 = "tf.Abs"(%asin) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) { is_stateless = true } : (tensor) -> tensor<*xf32> + + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + // Outer Else + %acos = "tf.Acos"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + %3 = "tf.Abs"(%acos) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%3) : (tensor<*xf32>) -> () + }) { is_stateless = true } : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Match existing function->Region pattern (simple) for IfRegion +func.func private @testIf1Then(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIf1Else(tensor<*xf32>) -> tensor<*xf32> +func.func @testIf1Result(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.If"({{.+}}) <{else_branch = @testIf1Else, {{.+}} then_branch = @testIf1Then} + %0 = "tf.IfRegion"(%arg0) ({ + %1 = func.call @testIf1Then(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %1 = func.call @testIf1Else(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Match existing function->Region pattern (with casts) for IfRegion + +func.func private @testIf1Then(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIf1Else(tensor<*xf32>) -> tensor<*xf32> +func.func @testIf2Result(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: "tf.If"({{.+}}) <{else_branch = @testIf1Else, {{.+}} then_branch = @testIf1Then} + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xf32>) -> tensor<*xf32> + %2 = func.call @testIf1Then(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }, { + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xf32>) -> tensor<*xf32> + %2 = func.call @testIf1Else(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// Match existing function->Region pattern (with multiple casts) for IfRegion + +func.func private @testIf1Then(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIf1Else(tensor<*xf32>) -> tensor<*xf32> +func.func @testIf2Result(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: "tf.If"({{.+}}) <{else_branch = @testIf1Else, {{.+}} then_branch = @testIf1Then} + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xf32>) -> tensor + %2 = "tf.Cast"(%1) {Truncate = false} : (tensor) -> tensor<*xf32> + %3 = func.call @testIf1Then(%2) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%3) : (tensor<*xf32>) -> () + }, { + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xf32>) -> tensor + %2 = "tf.Cast"(%1) {Truncate = false} : (tensor) -> tensor<*xf32> + %3 = func.call @testIf1Else(%2) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%3) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// Do not skip extern incompatible cast for trivial transform. + +func.func private @testIf1Then(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIf1Else(tensor<*xf32>) -> tensor<*xf32> +func.func @testIfExternIncompatibleCastTrivialTransform(%arg0: tensor, %arg1: tensor<2xi64>) -> tensor<2xf32> { + // CHECK: %[[CAST:.*]] = "tf.Cast"(%arg1) <{Truncate = false}> : (tensor<2xi64>) -> tensor<*xf32> + // CHECK: "tf.If"(%arg0, %[[CAST]]) <{else_branch = @testIf1Else, {{.+}} then_branch = @testIf1Then} + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xi64>) -> tensor<*xf32> + %0 = "tf.IfRegion"(%arg0) ({ + %2 = func.call @testIf1Then(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }, { + %2 = func.call @testIf1Else(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// Do not skip incompatible cast for trivial transform. + +// CHECK: func private @tf.IfRegion_else(%arg0: tensor<2xi64>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Cast" +// CHECK: func private @tf.IfRegion_then(%arg0: tensor<2xi64>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Cast" +func.func private @testIf1Then(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIf1Else(tensor<*xf32>) -> tensor<*xf32> +func.func @testIfIncompatibleCastTrivialTransform(%arg0: tensor, %arg1: tensor<2xi64>) -> tensor<2xf32> { + // CHECK: "tf.If"(%arg0, %arg1) <{else_branch = @tf.IfRegion_else{{.+}}then_branch = @tf.IfRegion_then} + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xi64>) -> tensor<*xf32> + %2 = func.call @testIf1Then(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }, { + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2xi64>) -> tensor<*xf32> + %2 = func.call @testIf1Else(%1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// No inputs, some outputs for IfRegion +// CHECK: func private @tf.IfRegion_else() -> tensor<2xf32> +// CHECK-NEXT: constant dense<1.000000e+00> +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @tf.IfRegion_then() -> tensor<2xf32> +// CHECK-NEXT: constant dense<0.000000e+00> +// CHECK-NEXT: "tf.Abs" +func.func @testSimple(%arg0: tensor) -> tensor<2xf32> { + // CHECK: "tf.If"{{.+}}else_branch = @tf.IfRegion_else{{.+}}then_branch = @tf.IfRegion_then + %0 = "tf.IfRegion"(%arg0) ({ + %cst_zero = arith.constant dense<0.0> : tensor<2xf32> + %1 = "tf.Abs"(%cst_zero) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%1) : (tensor<2xf32>) -> () + }, { + %cst_one = arith.constant dense<1.0> : tensor<2xf32> + %2 = "tf.Neg"(%cst_one) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%2) : (tensor<2xf32>) -> () + }) { is_stateless = true } : (tensor) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// No outputs, some inputs for IfRegion +// +// CHECK: func private @tf.IfRegion_else(%arg0: tensor<*xf32>) +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @tf.IfRegion_then(%arg0: tensor<*xf32>) +// CHECK-NEXT: "tf.Abs" +func.func private @printer(tensor<*xf32>) -> () +func.func @testNoOutputs(%arg0: tensor, %arg1: tensor<*xf32>) -> () { + // CHECK: "tf.If"{{.+}}else_branch = @tf.IfRegion_else{{.+}}then_branch = @tf.IfRegion_then + "tf.IfRegion"(%arg0) ({ + %1 = "tf.Abs"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + func.call @printer(%1) : (tensor<*xf32>) -> () + "tf.Yield"() : () -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + func.call @printer(%2) : (tensor<*xf32>) -> () + "tf.Yield"() : () -> () + }) { is_stateless = false } : (tensor) -> () + func.return +} + +// ----- +// Check ToBool folding for IfRegion +// CHECK: func private @tf.IfRegion_else(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Neg" +// CHECK: func private @tf.IfRegion_then(%arg0: tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: "tf.Abs" +// CHECK-LABEL: @testToBoolFold +func.func @testToBoolFold(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK-NEXT: "tf.If"(%arg0, %arg1) + // CHECK-SAME: else_branch = @tf.IfRegion_else + // CHECK-SAME: then_branch = @tf.IfRegion_then + %tobool = "tf.ToBool"(%arg0) : (tensor) -> tensor + %0 = "tf.IfRegion"(%tobool) ({ + %1 = "tf.Abs"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = true} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func private @branch_0(tensor) -> tensor<*xf32> +func.func private @branch_1(tensor) -> tensor<*xf32> + +// CHECK: func private @tf.CaseRegion_branch1{{.*}} +// CHECK: call @branch_1 +// CHECK: func private @tf.CaseRegion_branch0{{.*}} +// CHECK: call @branch_0 +// CHECK-LABEL: func @testCase +func.func @testCase(%arg0: tensor, %arg1: tensor>>) -> tensor<1x2x3xf32> { + // CHECK: [[Result:%.*]] = "tf.Case"(%arg0, %arg1) + // CHECK-SAME: branches = [@tf.CaseRegion_branch0{{.*}}, @tf.CaseRegion_branch1{{.*}}] + // CHECK-SAME: is_stateless = false + %0 = "tf.CaseRegion"(%arg0) ({ + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor>>) -> tensor + %2 = func.call @branch_0(%1) : (tensor) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }, { + %1 = "tf.Cast"(%arg1) {Truncate = false} : (tensor>>) -> tensor + %2 = func.call @branch_1(%1) : (tensor) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<1x2x3xf32> + return %0 : tensor<1x2x3xf32> +} + +// ----- + +// Simple WhileRegion +// CHECK: func private @tf.WhileRegion_body{{.+}} +// CHECK: "tf.Add" +// CHECK: constant dense<1> +// CHECK: "tf.Sub" +// CHECK:func private @tf.WhileRegion_cond{{.+}} +// CHECK: constant dense<0> +// CHECK: "tf.NotEqual" +// CHECK-LABEL: testValidWhileRegion +func.func @testValidWhileRegion(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) + // CHECK-NOT: attr1 + // CHECK-SAME: body = @tf.WhileRegion_body + // CHECK-SAME: cond = @tf.WhileRegion_cond + // CHECK-SAME: _attr0 = false + // CHECK-SAME: _xla_propagate_compile_time_consts = true + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %zero = arith.constant dense<0> : tensor + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false, _attr0 = false, attr1 = "hello"} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with type mismatch +// CHECK: func private @tf.WhileRegion_body{{.+}} +// CHECK: "tf.Add" +// CHECK: constant dense<1> +// CHECK: "tf.Sub" +// CHECK:func private @tf.WhileRegion_cond{{.+}} +// CHECK: constant dense<0> +// CHECK: "tf.NotEqual" +// CHECK-LABEL: testWhileRegionTypeMismatch +func.func @testWhileRegionTypeMismatch(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<4xf32>, %carg1: tensor): + %zero = arith.constant dense<0> : tensor + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<4xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<4xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with constant sinking +// CHECK: func private @tf.WhileRegion_body{{.+}} +// CHECK: constant dense<1> +// CHECK: "tf.Add" +// CHECK: "tf.Sub" +// CHECK:func private @tf.WhileRegion_cond{{.+}} +// CHECK: constant dense<0> +// CHECK: "tf.NotEqual" +// CHECK-LABEL: testWhileRegionConstantSink +func.func @testWhileRegionConstantSink(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + %zero = arith.constant dense<0> : tensor + %one = arith.constant dense<1> : tensor + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<4xf32>, %carg1: tensor): + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<4xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<4xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with implicitly captured extern value in cond +// CHECK: func private @tf.WhileRegion_body(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK: "tf.Add" +// CHECK: constant dense<1> +// CHECK: "tf.Sub" +// CHECK: return %{{.+}}, %{{.+}}, %arg2 : tensor<*xf32>, tensor, tensor +// CHECK: func private @tf.WhileRegion_cond(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK: "tf.NotEqual"(%arg1, %arg2) +// CHECK-LABEL: testWhileRegionExternInCond +func.func @testWhileRegionExternInCond(%arg0 : tensor<*xf32>, %arg1 : tensor, %arg2 : tensor) -> tensor<*xf32> { + %cst = arith.constant dense<4> : tensor + %limit = "tf.Add"(%arg2, %cst) : (tensor, tensor) -> tensor + // CHECK: [[Result:%.*]]:3 = "tf.While"(%arg0, %arg1, %{{.+}} <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %ne = "tf.NotEqual"(%carg1, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with implicitly captured extern value in body +// CHECK: func private @tf.WhileRegion_body(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK: %0 = "tf.Add"(%arg0, %arg0) +// CHECK: %1 = "tf.Sub"(%arg1, %arg2) +// CHECK: return %0, %1, %arg2 + +// CHECK: func private @tf.WhileRegion_cond(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK: constant dense<0> +// CHECK: "tf.NotEqual" + +// CHECK-LABEL: testWhileRegionExternInBody +func.func @testWhileRegionExternInBody(%arg0 : tensor<*xf32>, %arg1 : tensor, %arg2 : tensor) -> tensor<*xf32> { + %zero = arith.constant dense<0> : tensor + %cst = arith.constant dense<4> : tensor + %stride = "tf.Add"(%arg2, %cst) : (tensor, tensor) -> tensor + // CHECK: [[Result:%.*]]:3 = "tf.While"(%arg0, %arg1, %{{.+}} <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %sub = "tf.Sub"(%barg1, %stride) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with implicitly captured extern value in cond and body +// CHECK: func private @tf.WhileRegion_body(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor, %arg3: tensor) +// CHECK: return %{{.+}}, %{{.+}}, %arg2, %arg3 +// CHECK: func private @tf.WhileRegion_cond(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor, %arg3: tensor) +// CHECK-LABEL: testWhileRegionExternInBodyAndCond +func.func @testWhileRegionExternInBodyAndCond(%arg0 : tensor<*xf32>, %arg1 : tensor, %arg2 : tensor) -> tensor<*xf32> { + %cst = arith.constant dense<4> : tensor + %stride = "tf.Add"(%arg2, %cst) : (tensor, tensor) -> tensor + %cst1 = arith.constant dense<44> : tensor + %limit = "tf.Add"(%arg2, %cst1) : (tensor, tensor) -> tensor + // CHECK: [[Result:%.*]]:4 = "tf.While"(%arg0, %arg1, %{{.+}}, %{{.+}} <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %ne = "tf.NotEqual"(%carg1, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %sub = "tf.Sub"(%barg1, %stride) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// WhileRegion with same value implicitly captured in cond and body +// CHECK: func private @tf.WhileRegion_body(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK: return %{{.+}}, %{{.+}}, %arg2 +// CHECK: func private @tf.WhileRegion_cond(%arg0: tensor<*xf32>, %arg1: tensor, %arg2: tensor) +// CHECK-LABEL: testWhileRegionSameExternInBodyAndCond +func.func @testWhileRegionSameExternInBodyAndCond(%arg0 : tensor<*xf32>, %arg1 : tensor, %arg2 : tensor) -> tensor<*xf32> { + %cst = arith.constant dense<4> : tensor + %stride = "tf.Add"(%arg2, %cst) : (tensor, tensor) -> tensor + // CHECK: [[Result:%.*]]:3 = "tf.While"(%arg0, %arg1, %{{.+}} <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %ne = "tf.NotEqual"(%carg1, %stride) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %sub = "tf.Sub"(%barg1, %stride) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Simple trivially transformable while +// CHECK: func private @while_cond +// CHECK: func private @while_body +// CHECK-LABEL: testWhileRegionTrivial +func.func private @while_cond(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<*xf32>, %arg1 : tensor) -> (tensor<*xf32>, tensor) +func.func @testWhileRegionTrivial(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @while_body, cond = @while_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = func.call @while_cond(%carg0, %carg1) : (tensor<*xf32>, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy:2 = func.call @while_body(%barg0, %barg1) : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Trivially transformable with casts +// CHECK: func private @while_cond +// CHECK: func private @while_body +// CHECK-LABEL: testWhileRegionTrivialCasts +func.func private @while_cond(%arg0 : tensor<4xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<4xf32>, %arg1 : tensor) -> (tensor<4xf32>, tensor) +func.func @testWhileRegionTrivialCasts(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @while_body, cond = @while_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond_cast = "tf.Cast"(%carg0) : (tensor<*xf32>) -> tensor<4xf32> + %cond = func.call @while_cond(%cond_cast, %carg1) : (tensor<4xf32>, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy_cast = "tf.Cast"(%barg0) : (tensor<*xf32>) -> tensor<4xf32> + %bdy:2 = func.call @while_body(%bdy_cast, %barg1) : (tensor<4xf32>, tensor) -> (tensor<4xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<4xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Trivially transformable with multiple casts +// CHECK: func private @while_cond +// CHECK: func private @while_body +// CHECK-LABEL: testWhileRegionTrivialMultipleCasts +func.func private @while_cond(%arg0 : tensor<4xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<4xf32>, %arg1 : tensor) -> (tensor<4xf32>, tensor) +func.func @testWhileRegionTrivialMultipleCasts(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @while_body, cond = @while_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond_cast0 = "tf.Cast"(%carg0) : (tensor<*xf32>) -> tensor + %cond_cast1 = "tf.Cast"(%cond_cast0) : (tensor) -> tensor<4xf32> + %cond = func.call @while_cond(%cond_cast1, %carg1) : (tensor<4xf32>, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy_cast0 = "tf.Cast"(%barg0) : (tensor<*xf32>) -> tensor + %bdy_cast1 = "tf.Cast"(%bdy_cast0) : (tensor) -> tensor<4xf32> + %bdy:2 = func.call @while_body(%bdy_cast1, %barg1) : (tensor<4xf32>, tensor) -> (tensor<4xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<4xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Almost trivially transformable with incompatible cast +// CHECK: func private @tf.WhileRegion_body +// CHECK-NEXT: "tf.Cast" +// CHECK: func private @tf.WhileRegion_cond +// CHECK-NEXT: "tf.Cast" +// CHECK-LABEL: testWhileRegionIncompatibleCast +func.func private @while_cond(%arg0 : tensor<4xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<4xf32>, %arg1 : tensor) -> (tensor<4xi64>, tensor) +func.func @testWhileRegionIncompatibleCast(%arg0 : tensor<*xi64>, %arg1 : tensor) -> tensor<*xi64> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xi64>, %carg1: tensor): + %cond_cast = "tf.Cast"(%carg0) : (tensor<*xi64>) -> tensor<4xf32> + %cond = func.call @while_cond(%cond_cast, %carg1) : (tensor<4xf32>, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xi64>, %barg1: tensor): + %bdy_cast = "tf.Cast"(%barg0) : (tensor<*xi64>) -> tensor<4xf32> + %bdy:2 = func.call @while_body(%bdy_cast, %barg1) : (tensor<4xf32>, tensor) -> (tensor<4xi64>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<4xi64>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xi64>, tensor) -> (tensor<*xi64>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xi64> +} + +// ----- + +// Almost trivially transformable with extern values +// CHECK: func private @tf.WhileRegion_body +// CHECK: call @while_body +// CHECK: func private @tf.WhileRegion_cond +// CHECK: call @while_cond +// CHECK-LABEL: testWhileRegionExtern +func.func private @while_cond(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<*xf32>, %arg1 : tensor, %arg2 : tensor<*xf32>) -> (tensor<*xf32>, tensor) +func.func @testWhileRegionExtern(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + %ext = "tf.Neg"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> + // CHECK: [[Result:%.*]]:3 = "tf.While"(%arg0, %arg1, %{{.+}} <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = func.call @while_cond(%carg0, %carg1) : (tensor<*xf32>, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy:2 = func.call @while_body(%barg0, %barg1, %ext) : (tensor<*xf32>, tensor, tensor<*xf32>) -> (tensor<*xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Almost trivially transformable, mismatching block arguments +// CHECK: func private @tf.WhileRegion_body +// CHECK: call @while_body +// CHECK: func private @tf.WhileRegion_cond +// CHECK: call @while_cond +// CHECK-LABEL: testWhileRegionBlockArgMismatch +func.func private @while_cond(%arg0 : tensor, %arg1 : tensor<*xf32>) -> tensor +func.func private @while_body(%arg0 : tensor<*xf32>, %arg1 : tensor) -> (tensor<*xf32>, tensor) +func.func @testWhileRegionBlockArgMismatch(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @tf.WhileRegion_body, cond = @tf.WhileRegion_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = func.call @while_cond(%carg1, %carg0) : (tensor, tensor<*xf32>) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy:2 = func.call @while_body(%barg0, %barg1) : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Simple trivially transformable while with ToBool +// CHECK: func private @while_cond +// CHECK: func private @while_body +// CHECK-LABEL: testWhileRegionTrivial +func.func private @while_cond(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor +func.func private @while_body(%arg0 : tensor<*xf32>, %arg1 : tensor) -> (tensor<*xf32>, tensor) +func.func @testWhileRegionTrivial(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) <{body = @while_body, cond = @while_cond + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond_i32 = func.call @while_cond(%carg0, %carg1) : (tensor<*xf32>, tensor) -> tensor + %cond = "tf.ToBool"(%cond_i32) : (tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %bdy:2 = func.call @while_body(%barg0, %barg1) : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + "tf.Yield"(%bdy#0, %bdy#1) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Test tf.IfRegion device is preserved. +// CHECK-LABEL: func @testIfRegionDevice +func.func @testIfRegionDevice(%arg0: tensor) { + "tf.IfRegion"(%arg0) ({ + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) {is_stateless = false, device = "/device:CPU:0"} : (tensor) -> () + + // CHECK: "tf.If" + // CHECK-SAME: device = "/device:CPU:0" + func.return +} + +// ----- + +// Test tf.WhileRegion device is preserved. +// CHECK-LABEL: func @testWhileRegionDevice +func.func @testWhileRegionDevice() { + "tf.WhileRegion"() ({ + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%0) : (tensor) -> () + }, { + "tf.Yield"() : () -> () + }) {is_stateless = false, device = "/device:CPU:0"} : () -> () + + // CHECK: "tf.While" + // CHECK-SAME: device = "/device:CPU:0" + func.return +} + +// ----- + +// CHECK-LABEL: @testOverrideIfRegionXlaPropageCompileTimeConsts +func.func @testOverrideIfRegionXlaPropageCompileTimeConsts(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.If" + // CHECK-SAME: _xla_propagate_compile_time_consts = true + %0 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.Abs"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %2 = "tf.Neg"(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + }) {is_stateless = true, _attr0 = false, attr1 = "hello", _then_func_name = "test_then_name", _else_func_name = "test_else_name", _xla_propagate_compile_time_consts = false} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Trivial WhileRegion_cond +// CHECK: func private @tf.WhileRegion_body{{.+}} +// CHECK: "tf.Add" +// CHECK: constant dense<1> +// CHECK: "tf.Sub" +// CHECK:func private @tf.WhileRegion_cond{{.+}} +// CHECK: "tf.ToBool" +// CHECK-LABEL: testValidWhileRegion +func.func @testValidWhileRegion(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + // CHECK: [[Result:%.*]]:2 = "tf.While"(%arg0, %arg1) + // CHECK-NOT: attr1 + // CHECK-SAME: body = @tf.WhileRegion_body + // CHECK-SAME: cond = @tf.WhileRegion_cond + // CHECK-SAME: _attr0 = false + // CHECK-SAME: _xla_propagate_compile_time_consts = true + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = "tf.ToBool"(%carg1) : (tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false, _attr0 = false, attr1 = "hello"} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + // CHECK: return [[Result]]#0 + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// Condition with passthrough arguments +// CHECK: func private @tf.WhileRegion_body{{.+}} +// CHECK: func private @tf.WhileRegion_cond{{.+}} +// CHECK: return {{[^,]*}} : +// CHECK-LABEL: testPassThroughCond +func.func @testPassThroughCond(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = "tf.ToBool"(%carg1) : (tensor) -> tensor + "tf.Yield"(%cond, %carg0, %carg1) : (tensor, tensor<*xf32>, tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false, _attr0 = false, attr1 = "hello"} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + func.return %0#0 : tensor<*xf32> +} + +// ----- + +func.func @init(%arg0: tensor<4xf32>) -> tensor<7xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<7xf32> + return %0 : tensor<7xf32> +} +func.func @next(%arg0: tensor<7xf32>, %arg1: tensor<3xf32>) -> tensor<6xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<6xf32> + return %0 : tensor<6xf32> +} +func.func @finalize(%arg0: tensor<6xf32>, %arg1: tensor<2xf32>) -> tensor<5xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<5xf32> + return %0 : tensor<5xf32> +} + +// CHECK-LABEL: testGeneratorDatasetRegion +func.func @testGeneratorDatasetRegion(%arg0: tensor<4xf32>, %arg1: tensor<3xf32>, %arg2: tensor, %arg3: tensor<2xf32>) { + // CHECK: "tf.GeneratorDataset" + // CHECK-DAG: @init + // CHECK-DAG: @next + // CHECK-DAG: @finalize + // CHECK: return + %0 = "tf.GeneratorDatasetRegion"(%arg0, %arg1, %arg2, %arg3) ({ + ^bb0(%arg4: tensor<4xf32>): + %1 = func.call @init(%arg4) : (tensor<4xf32>) -> tensor<7xf32> + "tf.Yield"(%1) : (tensor<7xf32>) -> () + }, { + ^bb0(%arg4: tensor<7xf32>, %arg5: tensor<3xf32>): + %1 = func.call @next(%arg4, %arg5) : (tensor<7xf32>, tensor<3xf32>) -> tensor<6xf32> + "tf.Yield"(%1) : (tensor<6xf32>) -> () + }, { + ^bb0(%arg4: tensor<6xf32>, %arg5: tensor<2xf32>): + %1 = func.call @finalize(%arg4, %arg5) : (tensor<6xf32>, tensor<2xf32>) -> tensor<5xf32> + "tf.Yield"(%1) : (tensor<5xf32>) -> () + }) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", metadata = "", operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string]} : (tensor<4xf32>, tensor<3xf32>, tensor, tensor<2xf32>) -> tensor + return +} + +// ----- + +func.func @init(%arg0: tensor<4xf32>) -> tensor<7xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<7xf32> + return %0 : tensor<7xf32> +} +func.func @next(%arg0: tensor<3xf32>, %arg1: tensor<7xf32>) -> tensor<6xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<6xf32> + return %0 : tensor<6xf32> +} +func.func @finalize(%arg0: tensor<6xf32>, %arg1: tensor<2xf32>) -> tensor<5xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<5xf32> + return %0 : tensor<5xf32> +} + +// CHECK-LABEL: testGeneratorDatasetRegionWithComplexBlocks +func.func @testGeneratorDatasetRegionWithComplexBlocks(%arg0: tensor<4xf32>, %arg1: tensor<3xf32>, %arg2: tensor, %arg3: tensor<2xf32>) { + // CHECK: "tf.GeneratorDataset" + // CHECK-NOT: @init + // CHECK-NOT: @next + // CHECK-NOT: @finalize + // CHECK: -> tensor + // CHECK: return + %0 = "tf.GeneratorDatasetRegion"(%arg0, %arg1, %arg2, %arg3) ({ + ^bb0(%arg4: tensor<4xf32>): + %sum = "tf.Add"(%arg4, %arg4) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %1 = func.call @init(%sum) : (tensor<4xf32>) -> tensor<7xf32> + "tf.Yield"(%1) : (tensor<7xf32>) -> () + }, { + ^bb0(%arg4: tensor<7xf32>, %arg5: tensor<3xf32>): + %1 = func.call @next(%arg5, %arg4) : (tensor<3xf32>, tensor<7xf32>) -> tensor<6xf32> + "tf.Yield"(%1) : (tensor<6xf32>) -> () + }, { + ^bb0(%arg4: tensor<6xf32>, %arg5: tensor<2xf32>): + %1 = func.call @finalize(%arg4, %arg5) : (tensor<6xf32>, tensor<2xf32>) -> tensor<5xf32> + %sum = "tf.Add"(%1, %1) : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xf32> + "tf.Yield"(%sum) : (tensor<5xf32>) -> () + }) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", metadata = "", operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string]} : (tensor<4xf32>, tensor<3xf32>, tensor, tensor<2xf32>) -> tensor + return +} + +// ----- + +func.func private @tf.WhileRegion_body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %1 = builtin.unrealized_conversion_cast to tensor + %2 = builtin.unrealized_conversion_cast to tensor + return %1, %2 : tensor, tensor +} +func.func private @tf.WhileRegion_cond(%arg0: tensor) -> tensor { + %0 = builtin.unrealized_conversion_cast to tensor + return %0 : tensor +} +// CHECK-LABEL: testNameCollision +func.func @testNameCollision(%arg0: tensor) { + %1 = builtin.unrealized_conversion_cast to tensor + %2 = builtin.unrealized_conversion_cast to tensor + // CHECK: "tf.While" + // CHECK-SAME: body = @tf.WhileRegion_body_1 + // CHECK-SAME: cond = @tf.WhileRegion_cond_0 + %3:2 = "tf.WhileRegion"(%1, %2) <{is_stateless = false}> ({ + ^bb0(%arg1: tensor, %arg2: tensor): + %8 = func.call @tf.WhileRegion_cond(%arg1) : (tensor) -> tensor + "tf.Yield"(%8, %arg1, %arg2) : (tensor, tensor, tensor) -> () + }, { + ^bb0(%arg1: tensor, %arg2: tensor): + %8:2 = func.call @tf.WhileRegion_body(%arg1, %arg2) : (tensor, tensor) -> (tensor, tensor) + "tf.Yield"(%8#0, %8#1) : (tensor, tensor) -> () + }) : (tensor, tensor) -> (tensor, tensor) + return +} + +// ----- + +func.func private @my_cond(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = builtin.unrealized_conversion_cast to tensor + return %0 : tensor +} +func.func private @my_body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + return %arg0, %arg1 : tensor, tensor +} +// CHECK-LABEL: testConditionWithPassthroughArgs +func.func @testConditionWithPassthroughArgs(%arg1: tensor, %arg2: tensor) { + // CHECK: "tf.While" + // CHECK-SAME: body = @my_body + // CHECK-SAME: cond = @my_cond + %3:2 = "tf.WhileRegion"(%arg1, %arg2) <{is_stateless = false}> ({ + ^bb0(%barg1: tensor, %barg2: tensor): + %8 = func.call @my_cond(%barg1, %barg2) : (tensor, tensor) -> tensor + "tf.Yield"(%8, %barg1, %barg2) : (tensor, tensor, tensor) -> () + }, { + ^bb0(%barg1: tensor, %barg2: tensor): + %r1, %r2 = func.call @my_body(%barg1, %barg2) : (tensor, tensor) -> (tensor, tensor) + "tf.Yield"(%r1, %r2) : (tensor, tensor) -> () + }) : (tensor, tensor) -> (tensor, tensor) + return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_arguments.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_arguments.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2dff15fee064f588ae6fff3549a7f680d7e7f2ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_arguments.mlir @@ -0,0 +1,237 @@ +// RUN: tf-opt %s -allow-unregistered-dialect --tf-remove-unused-arguments --split-input-file | FileCheck %s + +func.func private @f(%arg0: tensor, %arg1: tensor) -> tensor { + "SomeOp"(%arg1) : (tensor) -> () + return %arg1 : tensor +} + +// CHECK-LABEL: removes_first_arg +func.func @removes_first_arg(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: call{{.*}}(%arg1) + %1 = func.call @f(%arg0, %arg1) : (tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- + +func.func private @f(%arg0: tensor, %arg1: tensor) -> tensor { + "SomeOp"(%arg0) : (tensor) -> () + return %arg0 : tensor +} + +// CHECK-LABEL: removes_last_arg +func.func @removes_last_arg(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: call{{.*}}(%arg0) + %1 = func.call @f(%arg0, %arg1) : (tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- + +func.func @f(%arg0: tensor, %arg1: tensor) -> tensor { + "SomeOp"(%arg1) : (tensor) -> () + return %arg1 : tensor +} + +// CHECK-LABEL: leaves_public_functions_alone +func.func @leaves_public_functions_alone(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: call{{.*}}(%arg0, %arg1) + %1 = func.call @f(%arg0, %arg1) : (tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- + +func.func private @f(%arg0: tensor, %arg1: tensor) { + return +} + +// CHECK-LABEL: removes_all_args +func.func @removes_all_args(%arg0: tensor, %arg1: tensor) { + // CHECK: call{{.*}}() : + func.call @f(%arg0, %arg1) : (tensor, tensor) -> () + return +} + +// ----- + +// CHECK-LABEL: handles_mlprogram +// CHECK-SAME: () { +ml_program.func private @handles_mlprogram(%arg0: tensor, %arg1: tensor) { + ml_program.return +} + +// ----- + +func.func private @f(%arg0: tensor, %arg1: tensor) -> tensor { + return %arg0 : tensor +} + +// CHECK-LABEL: handles_partitioned_function_calls +func.func @handles_partitioned_function_calls(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: PartitionedCall"() + %1 = "tf.PartitionedCall"(%arg0, %arg1) {f = @f} : (tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- + +func.func private @f(%arg0: tensor, %arg1: tensor) -> tensor { + return %arg0 : tensor +} + +// CHECK-LABEL: handles_tpu_partitioned_function_calls_with_device_ordinal +func.func @handles_tpu_partitioned_function_calls_with_device_ordinal(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.TPUOrdinalSelector"() {device = ""} : () -> tensor + // CHECK: [[ordinal:%[0-9]*]] = "tf.TPUOrdinalSelector" + // CHECK: TPUPartitionedCall"([[ordinal]]) + %1 = "tf.TPUPartitionedCall"(%arg0, %arg1, %0) {f = @f} : (tensor, tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- + +func.func private @f(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32) -> f32 { + %0 = "tf.Add2"(%arg0, %arg2) : (f32, f32) -> f32 + return %0 : f32 +} + +// CHECK-LABEL: removes_multiple_args +func.func @removes_multiple_args(%arg0: f32, %arg1: f32) -> f32 { + // CHECK: call{{.*}}(%arg0, %arg0) + %0 = func.call @f(%arg0, %arg1, %arg0, %arg1) : (f32, f32, f32, f32) -> f32 + return %0 : f32 +} + +// ----- + +// CHECK-LABEL: leaves_while_loops_alone +func.func @leaves_while_loops_alone(%arg0: tensor, %arg1: tensor) { + // CHECK: While{{.*}}(%arg0, %arg1) + %0, %1 = "tf.While"(%arg0, %arg1) {body = @body, cond = @cond, is_stateless = false} : + (tensor, tensor) -> (tensor, tensor) + func.return +} + +// CHECK: @body(%arg0{{.*}}, %arg1{{.*}}) +func.func private @body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %0 = "tf.Const"() { value = dense<42> : tensor } : () -> tensor + %1 = "tf.Const"() { value = dense<42.0> : tensor } : () -> tensor + func.return %0, %1 : tensor, tensor +} + +// CHECK: @cond(%arg0{{.*}}, %arg1{{.*}}) +func.func private @cond(%arg0: tensor, %arg1: tensor) -> tensor { + %const = "tf.Const"() { value = dense<1000> : tensor } : () -> tensor + %result = "tf.Less"(%const, %arg0) : (tensor, tensor) -> tensor + func.return %result : tensor +} + +// ----- + +func.func private @f(%arg0: f32, %arg1: f32) -> (f32, f32) { + %1 = "some_op"(%arg1) : (f32) -> f32 + return %arg0, %1 : f32, f32 +} + +// CHECK-LABEL: @removes_first_passthrough_arg +func.func @removes_first_passthrough_arg(%arg0: f32, %arg1: f32) -> (f32, f32) { + // CHECK: %0 = call @f(%arg1) + %0, %1 = call @f(%arg0, %arg1) : (f32, f32) -> (f32, f32) + // CHECK: return %arg0, %0 + return %0, %1 : f32, f32 +} + +// ----- + +func.func private @f(%arg0: f32, %arg1: f32) -> (f32, f32) { + %0 = "some_op"(%arg0) : (f32) -> f32 + return %0, %arg1 : f32, f32 +} + +// CHECK-LABEL: @removes_second_passthrough_arg +func.func @removes_second_passthrough_arg(%arg0: f32, %arg1: f32) -> (f32, f32) { + // CHECK: %0 = call @f(%arg0) + %0, %1 = call @f(%arg0, %arg1) : (f32, f32) -> (f32, f32) + // CHECK: return %0, %arg1 + return %0, %1 : f32, f32 +} + +// ----- + +func.func private @f(%arg0: f32) -> (f32, f32) { + return %arg0, %arg0 : f32, f32 +} + +// CHECK-LABEL: @can_remove_all_results +func.func @can_remove_all_results(%arg0: f32) -> (f32, f32) { + // CHECK: call @f() + %0, %1 = call @f(%arg0) : (f32) -> (f32, f32) + // CHECK: return %arg0, %arg0 + return %0, %1 : f32, f32 +} + +// ----- + +// CHECK-LABEL: @has_inner_function +func.func private @has_inner_function(%arg0: f32) -> (f32, f32) { + builtin.module { + func.func private @inner() -> (tensor, tensor) { + %0, %1 = "some_constant"() : () -> (tensor, tensor) + // CHECK: return + // CHECK-SAME: tensor, tensor + return %0, %1 : tensor, tensor + } + } + // CHECK: return + // CHECK-NOT: arg + return %arg0, %arg0 : f32, f32 +} + +// CHECK-LABEL: @respects_regions +func.func @respects_regions(%arg0: f32) -> (f32, f32) { + // CHECK: call @has_inner_function() + %0, %1 = call @has_inner_function(%arg0) : (f32) -> (f32, f32) + // CHECK: return %arg0, %arg0 + return %0, %1 : f32, f32 +} + +// ----- + +// CHECK-LABEL: @handles_recursion +// CHECK-SAME: %arg0: f32 +func.func private @handles_recursion(%arg0: f32, %arg1: f32) -> (f32, f32) { + // CHECK: call @handles_recursion(%arg0) + %0, %1 = call @handles_recursion(%arg0, %arg0) : (f32, f32) -> (f32, f32) + // CHECK: return + // CHECK-SAME: f32, f32 + return %0, %1 : f32, f32 +} + +// ----- + +// CHECK-LABEL: @handles_multiple_returns +// CHECK-SAME: %arg0 +// CHECK-SAME: %arg1 +func.func private @handles_multiple_returns(%arg0: f32, %arg1: f32) -> (f32, f32) { + "SomeOp"() : () -> () + ^bb0: + return %arg0, %arg1 : f32, f32 + ^bb1: + return %arg1, %arg0 : f32, f32 +} + +// ----- + +// CHECK-LABEL: @removes_multiple_returns +func.func private @removes_multiple_returns(%arg0: f32, %arg1: f32) -> (f32, f32) { + "SomeOp"() : () -> () + ^bb0: + // CHECK: return + // CHECK-NOT: arg + return %arg0, %arg1 : f32, f32 + ^bb1: + // CHECK: return + // CHECK-NOT: arg + return %arg0, %arg1 : f32, f32 +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_while_results.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_while_results.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8b96bff23cb8413995a23077e09bebc707793ca1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/remove_unused_while_results.mlir @@ -0,0 +1,185 @@ +// RUN: tf-opt %s --tf-remove-unused-while-results --split-input-file | FileCheck %s + +// Check that we remove the first result and operand from the loop because the +// result is unused and all other conditions are satisfied, that means: +// - the corresponding first block arguments are unused (except for `tf.OpB` +// which is the defining op of the first result and will be pruned) +// - `tf.OpB` is not stateful +// - `tf.OpB` has only one result, and that result is only passed through to the +// unused result of the while loop + +// CHECK-LABEL: remove_first_result +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK-NOT: tf.OpB +// CHECK: tf.OpC +// CHECK-SAME: %[[BARG]] +func.func @remove_first_result(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"() {is_stateless = true} : () -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpB"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + %2 = "tf.OpC"(%arg3) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#1 : tensor<*xf32> +} + +// ----- + +// Check that we remove the second result and operand from the loop because the +// result is unused and all other conditions are satisfied. + +// CHECK-LABEL: remove_second_result +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpB +// CHECK-SAME: %[[BARG]] +// CHECK-NOT: tf.OpC +func.func @remove_second_result(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"() {is_stateless = true} : () -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpB"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + %2 = "tf.OpC"(%arg3) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#0 : tensor<*xf32> +} + +// ----- + +// Check that we don't remove the first result and operand from the loop (even +// though the result is unused) because the corresponding block argument is used +// in the while condition (`%arg2` is used by `tf.OpA`). + +// CHECK-LABEL: result_used_in_cond +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9_]+]], %[[ARG1:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[CARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[BARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpB +// CHECK: tf.OpC +func.func @result_used_in_cond(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpB"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + %2 = "tf.OpC"(%arg3) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#1 : tensor<*xf32> +} + +// ----- + +// Check that we don't remove the first result and operand from the loop (even +// though the result is unused) because the op that defines the result (passed +// through via `tf.Yield`) is stateful +// (`"tf.OpB"(%arg2) {is_stateless = false}`). + +// CHECK-LABEL: defining_op_is_stateful +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9_]+]], %[[ARG1:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[CARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[BARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpB +// CHECK: tf.OpC +func.func @defining_op_is_stateful(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"() {is_stateless = true} : () -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpB"(%arg2) {is_stateless = false} : (tensor<*xf32>) -> tensor<*xf32> + %2 = "tf.OpC"(%arg3) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#1 : tensor<*xf32> +} + +// ----- + +// Check that we don't remove the first result and operand from the loop (even +// though the result is unused) because the single result of the defining op has +// more than one use (`%1` is used by `tf.OpC` in addition to `tf.Yield`). + +// CHECK-LABEL: defining_op_has_other_used_results +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9_]+]], %[[ARG1:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[CARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[BARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpB +// CHECK: tf.OpC +func.func @defining_op_has_other_used_results(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"() {is_stateless = true} : () -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpB"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> tensor<*xf32> + %2 = "tf.OpC"(%arg3, %1) {is_stateless = true} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#1 : tensor<*xf32> +} + +// ----- + +// Check that we don't remove the first result and operand from the loop (even +// though the result is unused) because the defining op has another result that +// is used (`%1#1` is used by `tf.OpC`). + +// CHECK-LABEL: defining_op_has_other_used_results +// CHECK: tf.WhileRegion +// CHECK-SAME: (%[[ARG0:[a-zA-Z0-9_]+]], %[[ARG1:[a-zA-Z0-9_]+]]) +// CHECK: ^bb0 +// CHECK: (%[[CARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[CARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpA +// CHECK: ^bb0 +// CHECK: (%[[BARG0:[a-zA-Z0-9_]+]]: tensor<*xf32>, %[[BARG1:[a-zA-Z0-9_]+]]: tensor<*xf32>) +// CHECK: tf.OpB +// CHECK: tf.OpC +func.func @defining_op_has_other_used_results(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1 = "tf.OpA"() {is_stateless = true} : () -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor<*xf32>, %arg3: tensor<*xf32>): + %1:2 = "tf.OpB"(%arg2) {is_stateless = true} : (tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + %2 = "tf.OpC"(%arg3, %1#1) {is_stateless = true} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1#0, %2) : (tensor<*xf32>, tensor<*xf32>) -> () + }) {is_stateless = true, parallel_iterations = 10 : i64} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + return %0#1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replica_id_to_device_ordinal.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replica_id_to_device_ordinal.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4050cf60ef9b48e7fa9b7533cb6efb7701b72262 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replica_id_to_device_ordinal.mlir @@ -0,0 +1,98 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics %s -tf-replica-id-to-device-ordinal | FileCheck %s + + +// Tests device ordinal is set correctly for multiple devices. +// CHECK-LABEL: func @device_ordinal_attr_added_multiple_devices +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @device_ordinal_attr_added_multiple_devices() { + tf_executor.graph { + %0 = tf_executor.island { + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 0 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 1 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 2 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 3 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return + } + + // CHECK: tf_executor.island + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = 0 + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = 1 + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = 0 + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = 1 +} + +// ----- + + +// Tests device ordinal is set correctly for single device. +// CHECK-LABEL: func @device_ordinal_attr_added_single_device +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @device_ordinal_attr_added_single_device() { + tf_executor.graph { + %0 = tf_executor.island { + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 0 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return + } + // CHECK: tf_executor.island + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = 0 +} + +// ----- + +// Tests device ordinal is not set for no tpu device. +// CHECK-LABEL: func @device_ordinal_attr_not_added_no_tpu_device +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0"]} { + func.func @device_ordinal_attr_not_added_no_tpu_device() { + tf_executor.graph { + %0 = tf_executor.island { + "tf_device.launch"() ({ + %1 = "tf.opA"() : () -> tensor + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%1){_xla_replica_id = 0 : i64, device_ordinal = -1 : i64} : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return + } + // CHECK: tf_executor.island + // CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" + // CHECK-SAME: device_ordinal = -1 +} + +// ----- + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_invariant_op_hoisting.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_invariant_op_hoisting.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e57af41b86e238ee13e66eaa90c636eaae982780 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_invariant_op_hoisting.mlir @@ -0,0 +1,222 @@ +// RUN: tf-opt %s -tf-replicate-invariant-op-hoisting | FileCheck %s + +// CHECK-LABEL: func @replicate_arg_shape +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<*xf32>) +func.func @replicate_arg_shape(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + %0:4 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {n = 2: i32} { + %1 = "tf.Shape"(%ri) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %2 = "tf.opA"(%1, %ri) : (tensor, tensor<*xf32>) -> tensor<*xi32> + tf_device.return %1, %2 : tensor, tensor<*xi32> + } + func.return +} + +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_0]]) +// CHECK: tf_device.replicate([%[[ARG_0]], %[[ARG_1]]] as %[[RI:[a-z0-9]*]]: tensor<*xf32>) +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA"(%[[SHAPE]], %[[RI]]) +// CHECK: tf_device.return %[[SHAPE]], %[[OP_A]] + + +// CHECK-LABEL: func @invariant_shape +// CHECK-SAME: (%{{[a-z0-9]*}}: tensor<*xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<*xf32>) +func.func @invariant_shape(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + %0:2 = tf_device.replicate([%arg0, %arg0] as %ri: tensor<*xf32>) {n = 2: i32} { + %1 = "tf.Shape"(%arg1) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + tf_device.return %1 : tensor + } + func.return +} + +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_1]]) +// CHECK: tf_device.replicate +// CHECK: tf_device.return %[[SHAPE]] + + +// CHECK-LABEL: func @not_invariant_ordinal_placeholder +func.func @not_invariant_ordinal_placeholder(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + // CHECK: tf_device.replicate + // CHECK: tf._TPUDeviceOrdinalPlaceholder + %0:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {n = 2: i32} { + %1 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + tf_device.return %1 : tensor + } + func.return +} + + +// CHECK-LABEL: func @replicate_resource_var_arg_shape +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>, %[[ARG_1:[a-z0-9]*]]: tensor<*x!tf_type.resource>) +func.func @replicate_resource_var_arg_shape(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>) { + %0:6 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*x!tf_type.resource>) {n = 2: i32} { + %1 = "tf.ReadVariableOp"(%ri) {dtype = "tfdtype$DT_FLOAT"} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %2 = "tf.Shape"(%1) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %3 = "tf.opA"(%1, %2, %ri) : (tensor<*xf32>, tensor, tensor<*x!tf_type.resource>) -> tensor<*xi32> + tf_device.return %1, %2, %3 : tensor<*xf32>, tensor, tensor<*xi32> + } + func.return +} + +// CHECK: %[[VAR_SHAPE:[0-9]*]] = "tf.VariableShape"(%[[ARG_0]]) +// CHECK: tf_device.replicate([%[[ARG_0]], %[[ARG_1]]] as %[[RI:[a-z0-9]*]]: tensor<*x!tf_type.resource>) +// CHECK: %[[READ_VAR:[0-9]*]] = "tf.ReadVariableOp"(%[[RI]]) +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA"(%[[READ_VAR]], %[[VAR_SHAPE]], %[[RI]]) +// CHECK: tf_device.return %[[READ_VAR]], %[[VAR_SHAPE]], %[[OP_A]] + +// CHECK-LABEL: func @replicate_arg_shape_with_packed +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xf32>) +func.func @replicate_arg_shape_with_packed(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>) { + %0:4 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>, %arg2 as %rj : tensor<*xf32>) {n = 2: i32} { + %1 = "tf.Shape"(%rj) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %2 = "tf.opA"(%1, %rj) : (tensor, tensor<*xf32>) -> tensor<*xi32> + tf_device.return %1, %2 : tensor, tensor<*xi32> + } + func.return +} + +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_2]]) +// CHECK: tf_device.replicate([%[[ARG_0]], %[[ARG_1]]] as %[[RI:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_2]] as %[[RJ:[a-z0-9]*]]: tensor<*xf32>) +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA"(%[[SHAPE]], %[[RJ]]) +// CHECK: tf_device.return %[[SHAPE]], %[[OP_A]] + +// CHECK-LABEL: func @replicate_resource_var_arg_shape_with_packed +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>, %[[ARG_1:[a-z0-9]*]]: tensor<*x!tf_type.resource>, %[[ARG_2:[a-z0-9]*]]: tensor<*x!tf_type.resource>) +func.func @replicate_resource_var_arg_shape_with_packed(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<*x!tf_type.resource>) { + %0:6 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*x!tf_type.resource>, %arg2 as %rj : tensor<*x!tf_type.resource>) {n = 2: i32} { + %1 = "tf.ReadVariableOp"(%rj) {dtype = "tfdtype$DT_FLOAT"} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %2 = "tf.Shape"(%1) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %3 = "tf.opA"(%1, %2, %rj) : (tensor<*xf32>, tensor, tensor<*x!tf_type.resource>) -> tensor<*xi32> + tf_device.return %1, %2, %3 : tensor<*xf32>, tensor, tensor<*xi32> + } + func.return +} + +// CHECK: %[[VAR_SHAPE:[0-9]*]] = "tf.VariableShape"(%[[ARG_2]]) +// CHECK: tf_device.replicate([%[[ARG_0]], %[[ARG_1]]] as %[[RI:[a-z0-9]*]]: tensor<*x!tf_type.resource>, %[[ARG_2]] as %[[RJ:[a-z0-9]*]]: tensor<*x!tf_type.resource>) +// CHECK: %[[READ_VAR:[0-9]*]] = "tf.ReadVariableOp"(%[[RJ]]) +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA"(%[[READ_VAR]], %[[VAR_SHAPE]], %[[RJ]]) +// CHECK: tf_device.return %[[READ_VAR]], %[[VAR_SHAPE]], %[[OP_A]] + +// CHECK-LABEL: func @invariant_resource_var_shape +// CHECK-SAME: (%{{[a-z0-9]*}}: tensor<*x!tf_type.resource>, %[[ARG_1:[a-z0-9]*]]: tensor<*x!tf_type.resource>) +func.func @invariant_resource_var_shape(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>) { + %0 = "tf.ReadVariableOp"(%arg1) {dtype = "tfdtype$DT_FLOAT"} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %1:2 = tf_device.replicate([%arg0, %arg0] as %ri: tensor<*x!tf_type.resource>) {n = 2: i32} { + %2 = "tf.Shape"(%0) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + tf_device.return %2 : tensor + } + func.return +} + +// CHECK: %[[READ_VAR:[0-9]*]] = "tf.ReadVariableOp"(%[[ARG_1]]) +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[READ_VAR]]) +// CHECK: tf_device.replicate +// CHECK: tf_device.return %[[SHAPE]] + + +// CHECK-LABEL: func @dependent_invariants +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %{{[a-z0-9]*}}: tensor<*xf32>) +func.func @dependent_invariants(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + %0:6 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {n = 2: i32} { + %1 = "tf.Shape"(%ri) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %2 = "tf.opA"(%1) : (tensor) -> tensor<*xi32> + %3 = "tf.opB"(%1, %2) : (tensor, tensor<*xi32>) -> tensor<*xf32> + tf_device.return %1, %2, %3 : tensor, tensor<*xi32>, tensor<*xf32> + } + func.return +} + +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_0]]) +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA"(%[[SHAPE]]) +// CHECK: %[[OP_B:[0-9]*]] = "tf.opB"(%[[SHAPE]], %[[OP_A]]) +// CHECK: tf_device.replicate +// CHECK: tf_device.return %[[SHAPE]], %[[OP_A]], %[[OP_B]] + + +// CHECK-LABEL: func @nested_ops +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %{{[a-z0-9]*}}: tensor<*xf32>) +func.func @nested_ops(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + %0:8 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {n = 2: i32} { + %1 = "tf.Shape"(%ri) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %2 = "tf_device.launch"() ({ + %a = "tf.opA"(%1) : (tensor) -> tensor<*xi32> + tf_device.return %a : tensor<*xi32> + }) {device = "a"} : () -> tensor<*xi32> + %3 = "tf_device.launch"() ({ + %b = "tf.opB"(%1, %2) : (tensor, tensor<*xi32>) -> tensor<*xf32> + tf_device.return %b : tensor<*xf32> + }) {device = "b"} : () -> tensor<*xf32> + %4 = "tf_device.launch"() ({ + %c = "tf.opC"(%ri, %3) : (tensor<*xf32>, tensor<*xf32>) -> tensor + tf_device.return %c : tensor + }) {device = "c"} : () -> tensor + tf_device.return %1, %2, %3, %4 : tensor, tensor<*xi32>, tensor<*xf32>, tensor + } + func.return +} + +// CHECK: %[[SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_0]]) +// CHECK-NEXT: %[[LAUNCH_A:[0-9]*]] = "tf_device.launch" +// CHECK-SAME: device = "a" +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[SHAPE]]) +// CHECK-NEXT: tf_device.return %[[OP_A]] +// CHECK: %[[LAUNCH_B:[0-9]*]] = "tf_device.launch" +// CHECK-SAME: device = "b" +// CHECK-NEXT: %[[OP_B:[0-9]*]] = "tf.opB"(%[[SHAPE]], %[[LAUNCH_A]]) +// CHECK-NEXT: tf_device.return %[[OP_B]] +// CHECK: tf_device.replicate([{{.*}}] as %[[RI:[a-z0-9]+]]: tensor<*xf32>) +// CHECK-NEXT: %[[LAUNCH_C:[0-9]*]] = "tf_device.launch" +// CHECK-SAME: device = "c" +// CHECK-NEXT: %[[OP_C:[0-9]*]] = "tf.opC"(%[[RI]], %[[LAUNCH_B]]) +// CHECK-NEXT: tf_device.return %[[OP_C]] +// CHECK: tf_device.return %[[SHAPE]], %[[LAUNCH_A]], %[[LAUNCH_B]], %[[LAUNCH_C]] + + +// CHECK-LABEL: func @do_not_hoist_ops_with_virtual_device +// CHECK-SAME: [[VAL_0:%.*]]: tensor<*xf32>, [[VAL_1:%.*]]: tensor<*xf32>) +func.func @do_not_hoist_ops_with_virtual_device(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + %0:8 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2: i32} { + %1 = "tf.Shape"(%ri) {device = "", T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + %2 = "tf.opA"(%1) {device = "TPU_REPLICATED_CORE_0"} : (tensor) -> tensor<*xi32> + %3 = "tf_device.launch"() ({ + %b = "tf.opB"(%1) : (tensor) -> tensor<*xi32> + tf_device.return %b : tensor<*xi32> + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor<*xi32> + %4 = "tf_device.launch"() ({ + %c = "tf.opC"(%1) {device = "TPU_REPLICATED_CORE_0"} : (tensor) -> tensor<*xi32> + tf_device.return %c : tensor<*xi32> + }) {device = "c"} : () -> tensor<*xi32> + tf_device.return %1, %2, %3, %4 : tensor, tensor<*xi32>, tensor<*xi32>, tensor<*xi32> + } + func.return +} + +// CHECK: [[SHAPE:%.*]] = "tf.Shape"([[VAL_0]]) +// CHECK: tf_device.replicate({{\[}}[[VAL_0]], [[VAL_1]]] as [[VAL_4:%.*]]: tensor<*xf32>) {devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { +// CHECK: [[OP_A:%.*]] = "tf.opA"([[SHAPE]]) {device = "TPU_REPLICATED_CORE_0"} : (tensor) -> tensor<*xi32> +// CHECK: [[LAUNCH_B:%.*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> ({ +// CHECK: [[OP_B:%.*]] = "tf.opB"([[SHAPE]]) : (tensor) -> tensor<*xi32> +// CHECK: tf_device.return [[OP_B]] : tensor<*xi32> +// CHECK: }) : () -> tensor<*xi32> +// CHECK: [[LAUNCH_C:%.*]] = "tf_device.launch"() <{device = "c"}> ({ +// CHECK: [[OP_C:%.*]] = "tf.opC"([[SHAPE]]) {device = "TPU_REPLICATED_CORE_0"} : (tensor) -> tensor<*xi32> +// CHECK: tf_device.return [[OP_C]] : tensor<*xi32> +// CHECK: }) : () -> tensor<*xi32> +// CHECK: tf_device.return [[SHAPE]], [[OP_A]], [[LAUNCH_B]], [[LAUNCH_C]] + + +// Checks that the argument to a Shape that has a virtual device is not changed. + +// CHECK-LABEL: func @do_not_mutate_shape_op_with_virtual_device +// CHECK: tf_device.replicate +// CHECK-SAME: as [[RI:%.*]]: tensor<*xf32> +// CHECK: "tf.Shape"([[RI]]) +func.func @do_not_mutate_shape_op_with_virtual_device(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) { + tf_device.replicate([%arg0, %arg1] as %ri: tensor<*xf32>) {devices = {TPU_REPLICATED_HOST_0 = ["/device:CPU:0", "/device:CPU:1"]}, n = 2: i32} { + "tf_device.launch"() ({ + %1 = "tf.Shape"(%ri) {T = "tfdtype$DT_FLOAT", out_type = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + tf_device.return + }) {device = "TPU_REPLICATED_HOST_0"} : () -> () + tf_device.return + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_tensor_list_init_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_tensor_list_init_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6b80dce72f16ed487ad5e5bc8fce2e8c40fb5713 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_tensor_list_init_ops.mlir @@ -0,0 +1,139 @@ +// RUN: tf-opt %s -tf-replicate-tensor-list-init-ops -verify-diagnostics | FileCheck %s + +// CHECK: while_region_op +func.func @while_region_op() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK: TensorListReserve + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:1 = "tf.WhileRegion"(%tl) ({ + ^bb0(%barg1: tensor>>): // no predeceessors + %cond = "tf.false"():()-> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg1: tensor>>): // no predeceessors + "tf.Yield"(%barg1) : (tensor>>) -> () + }) {is_stateless = false} : (tensor>>) -> (tensor>>) + func.return +} + +// CHECK: while_region_op_empty_tensor_list +func.func @while_region_op_empty_tensor_list() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: EmptyTensorList + // CHECK: EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:1 = "tf.WhileRegion"(%tl) ({ + ^bb0(%barg1: tensor>>): // no predeceessors + %cond = "tf.false"():()-> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg1: tensor>>): // no predeceessors + "tf.Yield"(%barg1) : (tensor>>) -> () + }) {is_stateless = false} : (tensor>>) -> (tensor>>) + func.return +} + +// CHECK: while_region_op_twosepargs +func.func @while_region_op_twosepargs() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: %[[RESULT0:.*]] = "tf.TensorListReserve" + // CHECK: %[[RESULT1:.*]] = "tf.TensorListReserve" + // CHECK: %[[RESULT2:.*]] = "tf.TensorListReserve" + // CHECK: tf.WhileRegion + // CHECK-SAME: (%[[RESULT1]], %[[RESULT0]]) + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:2 = "tf.WhileRegion"(%tl, %tl) ({ + ^bb0(%barg1: tensor>>, %barg2: tensor>>): // no predeceessors + %cond = "tf.false"():()-> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg1: tensor>>, %barg2: tensor>>): // no predeceessors + "tf.Yield"(%barg1, %barg2) : (tensor>>, tensor>>) -> () + }) {is_stateless = false} : (tensor>>, tensor>>) -> (tensor>>, tensor>>) + func.return +} + +// CHECK: while_region_op_two_sep_args_empty_tensor_list +func.func @while_region_op_two_sep_args_empty_tensor_list() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: %[[RESULT0:.*]] = "tf.EmptyTensorList" + // CHECK: %[[RESULT1:.*]] = "tf.EmptyTensorList" + // CHECK: %[[RESULT2:.*]] = "tf.EmptyTensorList" + // CHECK: tf.WhileRegion + // CHECK-SAME: (%[[RESULT1]], %[[RESULT0]]) + %tl = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:2 = "tf.WhileRegion"(%tl, %tl) ({ + ^bb0(%barg1: tensor>>, %barg2: tensor>>): // no predeceessors + %cond = "tf.false"():()-> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg1: tensor>>, %barg2: tensor>>): // no predeceessors + "tf.Yield"(%barg1, %barg2) : (tensor>>, tensor>>) -> () + }) {is_stateless = false} : (tensor>>, tensor>>) -> (tensor>>, tensor>>) + func.return +} + +// CHECK: no_while_region_op +func.func @no_while_region_op() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK: TensorListReserve + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<8x1xf32> + %tl_set_item = "tf.TensorListSetItem"(%tl, %one, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + func.return +} + +// CHECK: no_while_region_op_empty_tensor_list +func.func @no_while_region_op_empty_tensor_list() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: EmptyTensorList + // CHECK: EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<8x1xf32> + %tl_set_item = "tf.TensorListSetItem"(%tl, %one, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + func.return +} + +// CHECK: use_two_sep_ops +func.func @use_two_sep_ops() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK: TensorListReserve + // CHECK: TensorListReserve + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %elem_1 = "tf._FirstOp"() : () -> tensor<8x1xf32> + %tl_set_item = "tf.TensorListSetItem"(%tl, %one, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + %elem_2 = "tf._SecondOp"() : () -> tensor<16x1xf32> + %tl_set_item2 = "tf.TensorListSetItem"(%tl, %one, %elem_2) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + func.return +} + +// CHECK: use_two_sep_ops_empty_tensor_list +func.func @use_two_sep_ops_empty_tensor_list() { + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: EmptyTensorList + // CHECK: EmptyTensorList + // CHECK: EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %elem_1 = "tf._FirstOp"() : () -> tensor<8x1xf32> + %tl_set_item = "tf.TensorListSetItem"(%tl, %one, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + %elem_2 = "tf._SecondOp"() : () -> tensor<16x1xf32> + %tl_set_item2 = "tf.TensorListSetItem"(%tl, %one, %elem_2) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir new file mode 100644 index 0000000000000000000000000000000000000000..900d5d5dd27829ac77c44998166a60d9730fc63f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir @@ -0,0 +1,439 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics %s -tf-replicate-to-island=legacy-graph-export=false | FileCheck %s + +// Tests per replica island has same control operands as island holding +// replicate. +// CHECK-LABEL: func @controls_per_replica +func.func @controls_per_replica() { + tf_executor.graph { + %1 = tf_executor.ControlTrigger {} + %2 = tf_executor.ControlTrigger {} + %3 = tf_executor.island(%1, %2) { + tf_device.replicate {n = 2 : i32} { + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: %[[CT_0:.*]] = tf_executor.ControlTrigger +// CHECK: %[[CT_1:.*]] = tf_executor.ControlTrigger +// CHECK: %{{.*}} = tf_executor.island(%[[CT_0]], %[[CT_1]]) +// CHECK: %{{.*}} = tf_executor.island(%[[CT_0]], %[[CT_1]]) + + +// Tests devices are not remapped if no devices were defined in replicate. +// CHECK-LABEL: func @no_devices +func.func @no_devices() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "CORE_0"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf.opA" +// device = "CORE_0" +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: "tf.opA" +// device = "CORE_0" +// CHECK: _parallel_execution_ids = "r0:1" + + +// Tests devices are not remapped if device is not in replicate devices. +// CHECK-LABEL: func @no_override_device +func.func @no_override_device() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "/TPU:2"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf.opA" +// device = "/TPU:2" +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: "tf.opA" +// device = "/TPU:2" +// CHECK: _parallel_execution_ids = "r0:1" + + +// Tests devices are remapped if device is in replicate devices. +// CHECK-LABEL: func @remap_device +func.func @remap_device() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "CORE_0"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf.opA" +// device = "/CPU:0" +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: "tf.opA" +// device = "/GPU:1" +// CHECK: _parallel_execution_ids = "r0:1" + + +// Tests replicate with control dependency output has each expanded replica +// control pinned to a sink island. +// CHECK-LABEL: func @replicate_control +func.func @replicate_control() { + tf_executor.graph { + %1 = tf_executor.island { + tf_device.replicate {n = 2 : i32} { + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch %1 : !tf_executor.control + } + func.return +} + +// CHECK: %[[REPLICA_0:.*]] = tf_executor.island +// CHECK: %[[REPLICA_1:.*]] = tf_executor.island +// CHECK: %[[SINK:.*]] = tf_executor.island(%[[REPLICA_0]], %[[REPLICA_1]]) +// CHECK: tf_executor.fetch %[[SINK]] + + +// Tests unused replica are pinned to the graph fetch. +// CHECK-LABEL: func @unused_replica +func.func @unused_replica(%arg0: tensor) { + %0 = tf_executor.graph { + %1:3 = tf_executor.island { + %2:2 = tf_device.replicate([%arg0, %arg0] as %ri0: tensor) {n = 2 : i32} { + tf_device.return %ri0 : tensor + } + tf_executor.yield %2#0, %2#1 : tensor, tensor + } + tf_executor.fetch %1#1 : tensor + } + func.return +} + +// CHECK: {{%.*}}, [[REPLICA_0_CONTROL:%.*]] = tf_executor.island +// CHECK: [[REPLICA_1_OUTPUT:%.*]], {{%.*}} = tf_executor.island +// CHECK: tf_executor.fetch [[REPLICA_1_OUTPUT]] : + + +// Tests replicate results are remapped correctly. +// CHECK-LABEL: func @replicate_result +func.func @replicate_result(%arg0: tensor, %arg1: tensor) { + %0:4 = tf_executor.graph { + %1:5 = tf_executor.island { + %2:4 = tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {n = 2 : i32} { + %3 = "tf.opA"(%arg2) : (tensor) -> tensor + %4 = "tf.opB"(%arg2) : (tensor) -> tensor + tf_device.return %3, %4 : tensor, tensor + } + tf_executor.yield %2#0, %2#1, %2#2, %2#3 : tensor, tensor, tensor, tensor + } + tf_executor.fetch %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor + } + func.return +} + +// CHECK: %[[REPLICA_OPA_1:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opA"(%arg0) +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: %[[REPLICA_OPB_1:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opB"(%arg0) +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: %[[REPLICA_OPA_2:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opA"(%arg1) +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: %[[REPLICA_OPB_2:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opB"(%arg1) +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: tf_executor.fetch %[[REPLICA_OPA_1]], %[[REPLICA_OPA_2]], %[[REPLICA_OPB_1]], %[[REPLICA_OPB_2]] + +// Tests replicate results are remapped correctly with packed inputs. +// CHECK-LABEL: func @replicate_with_packed_input +func.func @replicate_with_packed_input(%arg0: tensor, %arg1: tensor) { + %0:4 = tf_executor.graph { + %1:5 = tf_executor.island { + %2:4 = tf_device.replicate(%arg0 as %arg2: tensor, %arg1 as %arg3: tensor) + {n = 2 : i32, _packed_input_indices = [0, 1]} { + %3 = "tf.opA"(%arg2) : (tensor) -> tensor + %4 = "tf.opB"(%arg3) : (tensor) -> tensor + tf_device.return %3, %4 : tensor, tensor + } + tf_executor.yield %2#0, %2#1, %2#2, %2#3 : tensor, tensor, tensor, tensor + } + tf_executor.fetch %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor + } + func.return +} + +// CHECK: %[[REPLICA_OPA_1:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opA"(%arg0) +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: %[[REPLICA_OPB_1:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opB"(%arg1) +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: %[[REPLICA_OPA_2:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opA"(%arg0) +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: %[[REPLICA_OPB_2:.*]], %{{.*}} = tf_executor.island wraps +// CHECK: "tf.opB"(%arg1) +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: tf_executor.fetch %[[REPLICA_OPA_1]], %[[REPLICA_OPA_2]], %[[REPLICA_OPB_1]], %[[REPLICA_OPB_2]] + +// Tests replica id is added correctly. +// CHECK-LABEL: func @replica_id_attr_added +func.func @replica_id_attr_added(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {n = 2 : i32} { + "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.A"(%arg2) : (tensor) -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:0", _xla_replica_id = 0 : i64 +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingRaggedTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:0", _xla_replica_id = 0 : i64 +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:0", _xla_replica_id = 0 : i64 +// CHECK: tf_executor.island wraps "tf.A" +// CHECK-NOT: _xla_replica_id +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:1", _xla_replica_id = 1 : i64 +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingRaggedTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:1", _xla_replica_id = 1 : i64 +// CHECK: tf_executor.island wraps +// CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" +// CHECK-SAME: _parallel_execution_ids = "r0:1", _xla_replica_id = 1 : i64 +// CHECK: tf_executor.island wraps "tf.A" +// CHECK-NOT: _xla_replica_id +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: tf_executor.fetch + + +// Tests tf._TPUDeviceOrdinalPlaceholder ops are replaced with explicit device +// ordinal constant values given by logical_core. +// CHECK-LABEL: func @device_ordinals +func.func @device_ordinals() { + tf_executor.graph { + %0:1 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"], TPU_REPLICATED_CORE_1 = ["/job:worker/replica:0/task:0/device:TPU:3", "/job:worker/replica:0/task:0/device:TPU:4"]}} { + %1 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + %2 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 1} : () -> tensor + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: tf_executor.island wraps "tf.Const"() <{value = dense<1> : tensor}> {_parallel_execution_ids = "r0:0"} +// CHECK: tf_executor.island wraps "tf.Const"() <{value = dense<3> : tensor}> {_parallel_execution_ids = "r0:0"} +// CHECK: tf_executor.island wraps "tf.Const"() <{value = dense<2> : tensor}> {_parallel_execution_ids = "r0:1"} +// CHECK: tf_executor.island wraps "tf.Const"() <{value = dense<4> : tensor}> {_parallel_execution_ids = "r0:1"} + +// ----- +// Tests parallel_execute nested inside replicate +// CHECK-LABEL: func @nested_parallel_execute +func.func @nested_parallel_execute(%arg0: tensor, %arg1: tensor) { + %0:4 = tf_executor.graph { + %1:5 = tf_executor.island { + %2:4 = tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {n = 2 : i32} { + %3:2 = "tf_device.parallel_execute"() ({ + %6 = "tf_device.launch"() ({ + %4 = "tf.OpA"(%arg0) : (tensor) -> tensor + tf_device.return %4: tensor + }) {device = "/TPU:1"} : () -> (tensor) + tf_device.return %6: tensor + }, { + %4 = "tf_device.launch"() ({ + %5 = "tf.OpB"(%arg1) : (tensor) -> (tensor) + tf_device.return %5: tensor + }) {device = "/TPU:2"} : () -> (tensor) + tf_device.return %4 : tensor + }) : () -> (tensor, tensor) + tf_device.return %3#0, %3#1 : tensor, tensor + } + tf_executor.yield %2#0, %2#1, %2#2, %2#3 : tensor, tensor, tensor, tensor + } + tf_executor.fetch %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor + } + func.return +} + +// CHECK: tf_executor.island +// CHECK: tf_device.parallel_execute +// CHECK: tf_device.launch +// CHECK: <{device = "/TPU:1"}> +// CHECK: tf.OpA +// CHECK: tf_device.launch +// CHECK: <{device = "/TPU:2"}> +// CHECK: tf.OpB +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: tf_executor.island +// CHECK: tf_device.parallel_execute +// CHECK: tf_device.launch +// CHECK: <{device = "/TPU:1"}> +// CHECK: tf.OpA +// CHECK: tf_device.launch +// CHECK: <{device = "/TPU:2"}> +// CHECK: tf.OpB +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: tf_executor.fetch + +// ----- +// Tests parallel_group attr can merge correctly. +// CHECK-LABEL: func @merge_of_parallel_group_attr +func.func @merge_of_parallel_group_attr() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}, _parallel_execution_ids = "r4:5"} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "CORE_0"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf.opA" +// device = "/CPU:0" +// CHECK: _parallel_execution_ids = "r4:5,r0:0" +// CHECK: "tf.opA" +// device = "/GPU:1" +// CHECK: _parallel_execution_ids = "r4:5,r0:1" + +// ----- + +// Tests tf._TPUDeviceOrdinalPlaceholder cannot be updated when device ordinal +// is missing. + +func.func @missing_device_ordinals() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = tf_device.replicate {n = 2 : i32, devices = {TPU_REPLICATED_CORE_1 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + // expected-error@below {{requires device ordinal from device TPU_REPLICATED_CORE_0 to be present in 'tf.device.replicate' op}} + %2 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + tf_device.return %2 : tensor + } + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests tf._TPUDeviceOrdinalPlaceholder cannot be updated when device ordinal +// is missing. + +func.func @missing_devices() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = tf_device.replicate {n = 2 : i32} { + // expected-error@below {{devices attribute is not present}} + %2 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + tf_device.return %2 : tensor + } + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests devices are not remapped if device is not in replicate devices. +// CHECK-LABEL: func @no_override_device_new +func.func @no_override_device_new() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "/TPU:0"} : () -> () + tf_device.return + } + tf_executor.yield + } + %1 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "/TPU:1"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} +// CHECK: "tf.opA" +// device = "/TPU:0" +// CHECK: _parallel_execution_ids = "r0:0" +// CHECK: "tf.opA" +// device = "/TPU:0" +// CHECK: _parallel_execution_ids = "r0:1" +// CHECK: "tf.opA" +// device = "/TPU:1" +// CHECK: _parallel_execution_ids = "r1:0" +// CHECK: "tf.opA" +// device = "/TPU:1" +// CHECK: _parallel_execution_ids = "r1:1" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island_legacy.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island_legacy.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5caf579121873d1414714173c78b79a00ba69385 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island_legacy.mlir @@ -0,0 +1,283 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics %s -tf-replicate-to-island=legacy-graph-export=true | FileCheck %s + +// Tests per replica island has same control operands as island holding +// replicate. +// CHECK-LABEL: func @controls_per_replica +func.func @controls_per_replica() { + tf_executor.graph { + %1 = tf_executor.ControlTrigger {} + %2 = tf_executor.ControlTrigger {} + %3 = tf_executor.island(%1, %2) { + tf_device.replicate {n = 2 : i32} { + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: %[[CT_0:.*]] = tf_executor.ControlTrigger +// CHECK: %[[CT_1:.*]] = tf_executor.ControlTrigger +// CHECK: %{{.*}} = tf_executor.island(%[[CT_0]], %[[CT_1]]) +// CHECK: %{{.*}} = tf_executor.island(%[[CT_0]], %[[CT_1]]) + + +// Tests devices are not remapped if no devices were defined in replicate. +// CHECK-LABEL: func @no_devices +func.func @no_devices() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "CORE_0"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf_device.launch" +// CHECK: device = "CORE_0" +// CHECK: "tf.opA" +// CHECK: "tf_device.launch" +// CHECK: device = "CORE_0" +// CHECK: "tf.opA" + + +// Tests devices are not remapped if device is not in replicate devices. +// CHECK-LABEL: func @no_override_device +func.func @no_override_device() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "/TPU:2"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf_device.launch" +// CHECK: device = "/TPU:2" +// CHECK: "tf.opA" +// CHECK: "tf_device.launch" +// CHECK: device = "/TPU:2" +// CHECK: "tf.opA" + + +// Tests devices are remapped if device is in replicate devices. +// CHECK-LABEL: func @remap_device +func.func @remap_device() { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + "tf_device.launch"() ({ + "tf.opA"() : () -> () + tf_device.return + }) {device = "CORE_0"} : () -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: "tf_device.launch" +// CHECK: device = "/CPU:0" +// CHECK: "tf.opA" +// CHECK: "tf_device.launch" +// CHECK: device = "/GPU:1" +// CHECK: "tf.opA" + + +// Tests replicate with control dependency output has each expanded replica +// control pinned to a sink island. +// CHECK-LABEL: func @replicate_control +func.func @replicate_control() { + tf_executor.graph { + %1 = tf_executor.island { + tf_device.replicate {n = 2 : i32} { + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch %1 : !tf_executor.control + } + func.return +} + +// CHECK: %[[REPLICA_0:.*]] = tf_executor.island +// CHECK: %[[REPLICA_1:.*]] = tf_executor.island +// CHECK: %[[SINK:.*]] = tf_executor.island(%[[REPLICA_0]], %[[REPLICA_1]]) +// CHECK: tf_executor.fetch %[[SINK]] + + +// Tests unused replica are pinned to the graph fetch. +// CHECK-LABEL: func @unused_replica +func.func @unused_replica(%arg0: tensor) { + %0 = tf_executor.graph { + %1:3 = tf_executor.island { + %2:2 = tf_device.replicate([%arg0, %arg0] as %ri0: tensor) {n = 2 : i32} { + tf_device.return %ri0 : tensor + } + tf_executor.yield %2#0, %2#1 : tensor, tensor + } + tf_executor.fetch %1#1 : tensor + } + func.return +} + +// CHECK: {{%.*}}, [[REPLICA_0_CONTROL:%.*]] = tf_executor.island +// CHECK: [[REPLICA_1_OUTPUT:%.*]], {{%.*}} = tf_executor.island +// CHECK: tf_executor.fetch [[REPLICA_1_OUTPUT]], [[REPLICA_0_CONTROL]] + + +// Tests replicate results are remapped correctly. +// CHECK-LABEL: func @replicate_result +func.func @replicate_result(%arg0: tensor, %arg1: tensor) { + %0:4 = tf_executor.graph { + %1:5 = tf_executor.island { + %2:4 = tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {n = 2 : i32} { + %3 = "tf.opA"(%arg2) : (tensor) -> tensor + %4 = "tf.opB"(%arg2) : (tensor) -> tensor + tf_device.return %3, %4 : tensor, tensor + } + tf_executor.yield %2#0, %2#1, %2#2, %2#3 : tensor, tensor, tensor, tensor + } + tf_executor.fetch %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor + } + func.return +} + +// CHECK: %[[REPLICA_0:.*]]:2, %{{.*}} = tf_executor.island +// CHECK: %[[REPLICA_1:.*]]:2, %{{.*}} = tf_executor.island +// CHECK: tf_executor.fetch %[[REPLICA_0]]#0, %[[REPLICA_1]]#0, %[[REPLICA_0]]#1, %[[REPLICA_1]]#1 + + +// Tests replicate results are remapped correctly with packed inputs. +// CHECK-LABEL: func @replicate_with_packed_input +func.func @replicate_with_packed_input(%arg0: tensor, %arg1: tensor) { + %0:4 = tf_executor.graph { + %1:5 = tf_executor.island { + %2:4 = tf_device.replicate(%arg0 as %arg2: tensor, %arg1 as %arg3: tensor) + {n = 2 : i32, _packed_input_indices = [0, 1]} { + %3 = "tf.opA"(%arg2) : (tensor) -> tensor + %4 = "tf.opB"(%arg3) : (tensor) -> tensor + tf_device.return %3, %4 : tensor, tensor + } + tf_executor.yield %2#0, %2#1, %2#2, %2#3 : tensor, tensor, tensor, tensor + } + tf_executor.fetch %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor + } + func.return +} + +// CHECK: %[[REPLICA_0:.*]]:2, %{{.*}} = tf_executor.island +// CHECK: "tf.opA"(%arg0) +// CHECK: "tf.opB"(%arg1) +// CHECK: %[[REPLICA_1:.*]]:2, %{{.*}} = tf_executor.island +// CHECK: "tf.opA"(%arg0) +// CHECK: "tf.opB"(%arg1) +// CHECK: tf_executor.fetch %[[REPLICA_0]]#0, %[[REPLICA_1]]#0 + + +// Tests replica id is added correctly. +// CHECK-LABEL: func @replica_id_attr_added +func.func @replica_id_attr_added(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + %0 = tf_executor.island { + tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {n = 2 : i32} { + "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.EnqueueTPUEmbeddingArbitraryTensorBatch"(%arg2){table_ids = [1, 2]} : (tensor) -> () + "tf.A"(%arg2) : (tensor) -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK: tf_executor.island +// CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch" +// CHECK-SAME: _xla_replica_id = 0 +// CHECK: "tf.EnqueueTPUEmbeddingRaggedTensorBatch" +// CHECK-SAME: _xla_replica_id = 0 +// CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" +// CHECK-SAME: _xla_replica_id = 0 +// CHECK: "tf.A" +// CHECK-NOT: _xla_replica_id +// CHECK: tf_executor.island +// CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch" +// CHECK-SAME: _xla_replica_id = 1 +// CHECK: "tf.EnqueueTPUEmbeddingRaggedTensorBatch" +// CHECK-SAME: _xla_replica_id = 1 +// CHECK: "tf.EnqueueTPUEmbeddingArbitraryTensorBatch" +// CHECK-SAME: _xla_replica_id = 1 +// CHECK: "tf.A" +// CHECK-NOT: _xla_replica_id +// CHECK: tf_executor.fetch + + +// Tests tf._TPUDeviceOrdinalPlaceholder ops are replaced with explicit device +// ordinal constant values based on the first TPU core device id. +// CHECK-LABEL: func @device_ordinals +func.func @device_ordinals() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = tf_device.replicate {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + %2 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + tf_device.return %2 : tensor + } + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch + } + func.return +} + +// CHECK: tf_executor.island +// CHECK: [[CONST_0:%.+]] = "tf.Const" +// CHECK-SAME: value = dense<1> : tensor +// CHECK: tf_executor.yield [[CONST_0]] +// CHECK: tf_executor.island +// CHECK: [[CONST_1:%.+]] = "tf.Const" +// CHECK-SAME: value = dense<2> : tensor +// CHECK: tf_executor.yield [[CONST_1]] + +// ----- + +// Tests tf._TPUDeviceOrdinalPlaceholder cannot be updated when device ordinal +// is missing. + +func.func @missing_device_ordinals() { + tf_executor.graph { + %0:3 = tf_executor.island { + %1:2 = tf_device.replicate {n = 2 : i32, devices = {TPU_REPLICATED_CORE_1 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + // expected-error@below {{requires device ordinal from device TPU_REPLICATED_CORE_0 to be present in 'tf.device.replicate' op}} + %2 = "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + tf_device.return %2 : tensor + } + tf_executor.yield %1#0, %1#1 : tensor, tensor + } + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..02c9d4454ded2a24ef67fb5fd9e2db3afea7293d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir @@ -0,0 +1,507 @@ +// RUN: tf-opt -split-input-file -tf-test-resource-alias-analysis -verify-diagnostics %s | FileCheck %s + +// Test 2 resources that do not alias. + +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @non_aliasing_reads_writes +// expected-remark@below {{Region #0, Arg #0, ID 1 : 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 2}} +func.func @non_aliasing_reads_writes( + %arg0: !tf_res, + %arg1: !tf_res, + %arg2: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + // CHECK: tf_executor.island + %island:2 = tf_executor.island { + %read0 = "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %arg2) : (!tf_res, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (!tf_res) -> tensor<32xf32> + // expected-remark@below {{Result #0, ID 0 : 0}} + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read2 = "tf.ReadVariableOp"(%var_handle) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg1, %read0) : (!tf_res, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg0, %read2) : (!tf_res, tensor<32xf32>) -> () + %read3 = "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor<32xf32> + tf_executor.yield %read3 : tensor<32xf32> + } + tf_executor.fetch %island#0 : tensor<32xf32> + } + func.return %graph : tensor<32xf32> +} + +// ----- +// Tests aliasing of the two resource handles that refer to the same variable. + +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @aliasing_reads_writes +func.func @aliasing_reads_writes(%arg0: tensor<32xf32>) -> () { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 0, 1, 2}} + %vh1_id:2 = "tf.IdentityN"(%vh1, %arg0) : (!tf_res, tensor<32xf32>) -> (!tf_res, tensor<32xf32>) + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh1_id#0, %arg0) : (!tf_res, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + %read2 = "tf.ReadVariableOp"(%vh1) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh0, %read2) : (!tf_res, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%vh1_id#0, %read1) : (!tf_res, tensor<32xf32>) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// ----- +// Tests handling of Cast, Identity, and IdentityN + +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @aliasing_identities +func.func @aliasing_identities(%arg0: tensor<32xf32>) -> () { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2, 3, 4}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2, 3, 4}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 0, 1, 2, 3, 4}} + %vh1_id1 = "tf.Identity"(%vh1) : (!tf_res) -> !tf_res + // expected-remark@below {{Result #0, ID 3 : 0, 1, 2, 3, 4}} + %vh1_id2:2 = "tf.IdentityN"(%vh1, %arg0) : (!tf_res, tensor<32xf32>) -> (!tf_res, tensor<32xf32>) + // expected-remark@below {{Result #0, ID 4 : 0, 1, 2, 3, 4}} + %vh1_cast = "tf.Cast"(%vh1) : (!tf_res) -> (!tf_res) + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// ----- +// Test an unknown op that has a resource result is marked unknown + +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @unknown_resource_op +func.func @unknown_resource_op(%arg0: tensor<32xf32>) -> () { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %0 = "tf.UnknownVarHandleOp"() : () -> !tf_res +} + +// ----- +// Test aliasing through TPUReplicatedInput +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @aliasing_tpu_replicated_input +func.func @aliasing_tpu_replicated_input(%arg0: tensor<32xf32>) -> () { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 2}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 1, 2}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v1"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 0, 1, 2}} + %replicated = "tf.TPUReplicatedInput"(%vh0, %vh1) : (!tf_res, !tf_res) -> (!tf_res) + "tf.AssignVariableOp"(%vh0, %arg0) : (!tf_res, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%replicated) : (!tf_res) -> tensor<32xf32> + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// ----- +// Test aliasing through IfOp + +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @if_op_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 2, 3, 5}} +func.func @if_op_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 3, 5}} + // expected-remark@below {{Result #2, ID 3 : 0, 1, 2, 3, 5}} + %if:3 = "tf.If"(%read0, %arg1, %vh0) { + then_branch = @if_then, else_branch = @if_else, is_stateless = true + } : (tensor<32xf32>, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// expected-remark@below {{Region #0, Arg #0, ID 2 : 0, 1, 2}} +// expected-remark@below {{Region #0, Arg #1, ID 3 : 0, 3}} +func.func @if_then(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + func.return %u0, %id0, %id0 : !tf_res, !tf_res, !tf_res +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 0, 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 2}} +func.func @if_else(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + func.return %id0, %id0, %arg1 : !tf_res, !tf_res, !tf_res +} + +// ----- +// Test aliasing through CaseOp + +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @case_op_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 2, 3, 5}} +func.func @case_op_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 3, 5}} + // expected-remark@below {{Result #2, ID 3 : 0, 1, 2, 3, 5}} + %if:3 = "tf.Case"(%read0, %arg1, %vh0) { + branches = [@case_branch0, @case_branch1, @case_branch2], + is_stateless = true + } : (tensor, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// expected-remark@below {{Region #0, Arg #0, ID 2 : 0, 1, 2}} +// expected-remark@below {{Region #0, Arg #1, ID 3 : 0, 3}} +func.func @case_branch0(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + func.return %u0, %id0, %id0 : !tf_res, !tf_res, !tf_res +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 0, 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 2}} +func.func @case_branch1(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + func.return %id0, %id0, %arg1 : !tf_res, !tf_res, !tf_res +} + +// expected-remark@below {{Region #0, Arg #0, ID 0 : 0}} +// expected-remark@below {{Region #0, Arg #1, ID 1 : 1}} +func.func @case_branch2(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + func.return %arg0, %arg0, %arg1 : !tf_res, !tf_res, !tf_res +} + +// ----- +// Test aliasing through WhileOp +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @while_op_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 2, 3, 5}} +// expected-remark@below {{Region #0, Arg #2, ID 6 : 1, 2, 3, 6}} +func.func @while_op_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 3, 5, 6}} + // expected-remark@below {{Result #2, ID 3 : 1, 2, 3, 5, 6}} + %w:3 = "tf.While"(%arg0, %arg1, %arg2) { + body = @while_body, cond = @while_cond, is_stateless = false + } : (!tf_res, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// CHECK-LABEL: func @while_body +// Return 0 : new unknown resource +// Return 1 : arg2 +// Return 2 : arg1 +// expected-remark@below {{Region #0, Arg #0, ID 1 : 0, 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 0, 2}} +// expected-remark@below {{Region #0, Arg #2, ID 3 : 0, 3}} +func.func @while_body(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + func.return %u0, %arg2, %arg1 : !tf_res, !tf_res, !tf_res +} + +// CHECK-LABEL: func @while_cond +// expected-remark@below {{Region #0, Arg #0, ID 0 : 0}} +// expected-remark@below {{Region #0, Arg #1, ID 1 : 1}} +// expected-remark@below {{Region #0, Arg #2, ID 2 : 2}} +func.func @while_cond(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) -> tensor { + %0 = arith.constant dense : tensor + func.return %0 : tensor +} + +// ----- +// Test alias propagation through calls. +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @aliasing_through_calls +func.func @aliasing_through_calls(%arg0: tensor<32xf32>) -> () { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2, 3}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2, 3}} + %vh1 = "tf.Identity"(%vh0) : (!tf_res) -> (!tf_res) + // expected-remark@below {{Result #0, ID 2 : Unknown}} + // expected-remark@below {{Result #1, ID 3 : 0, 1, 2, 3}} + %c:2 = func.call @passthru(%vh1) : (!tf_res) -> (!tf_res, !tf_res) + func.return +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 1}} +func.func @passthru(%arg0: !tf_res) -> (!tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vx = "tf.VarHandleOp"() {container = "cf", shared_name = "vx"} : () -> !tf_res + func.return %vx, %arg0 : !tf_res, !tf_res +} + +// ----- +// Test aliasing through IfRegion + +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @if_region_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 7 : 1, 4, 6, 7}} +// expected-remark@below {{Region #0, Arg #1, ID 8 : 1, 2, 4, 5, 6, 8}} +func.func @if_region_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 3, 4, 5}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor + // expected-remark@below {{Result #0, ID 4 : Unknown}} + // expected-remark@below {{Result #1, ID 5 : 0, 1, 2, 3, 4, 5, 6, 8}} + // expected-remark@below {{Result #2, ID 6 : 1, 2, 4, 5, 6, 7, 8}} + %if:3 = "tf.IfRegion"(%read0) ({ + // expected-remark@below {{Result #0, ID 1 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 1, 2, 4, 5, 6, 8}} + %id0 = "tf.Identity"(%arg1) : (!tf_res) -> !tf_res + "tf.Yield"(%u0, %id0, %id0) : (!tf_res, !tf_res, !tf_res) -> () + }, { + // expected-remark@below {{Result #0, ID 3 : 0, 1, 3, 4, 5}} + %id0 = "tf.Identity"(%vh0) : (!tf_res) -> !tf_res + "tf.Yield"(%id0, %id0, %arg0) : (!tf_res, !tf_res, !tf_res) -> () + }) {is_stateless = true} : (tensor) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// ----- +// Test aliasing through CaseRegion + +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @case_region_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 7 : 1, 4, 6, 7}} +// expected-remark@below {{Region #0, Arg #1, ID 8 : 1, 2, 4, 5, 6, 8}} +func.func @case_region_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 3, 4, 5}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor + // expected-remark@below {{Result #0, ID 4 : Unknown}} + // expected-remark@below {{Result #1, ID 5 : 0, 1, 2, 3, 4, 5, 6, 8}} + // expected-remark@below {{Result #2, ID 6 : 1, 2, 4, 5, 6, 7, 8}} + %if:3 = "tf.CaseRegion"(%read0) ({ + // expected-remark@below {{Result #0, ID 1 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 1, 2, 4, 5, 6, 8}} + %id0 = "tf.Identity"(%arg1) : (!tf_res) -> !tf_res + "tf.Yield"(%u0, %id0, %id0) : (!tf_res, !tf_res, !tf_res) -> () + }, { + // expected-remark@below {{Result #0, ID 3 : 0, 1, 3, 4, 5}} + %id0 = "tf.Identity"(%vh0) : (!tf_res) -> !tf_res + "tf.Yield"(%id0, %id0, %arg0) : (!tf_res, !tf_res, !tf_res) -> () + }, { + "tf.Yield"(%vh0, %arg1, %arg1) : (!tf_res, !tf_res, !tf_res) -> () + }) {is_stateless = true} : (tensor) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// ----- +// Test aliasing through WhileRegion +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @while_region_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 11 : 1, 8, 11}} +// expected-remark@below {{Region #0, Arg #1, ID 12 : 1, 8, 9, 10, 12}} +// expected-remark@below {{Region #0, Arg #2, ID 13 : 1, 8, 9, 10, 13}} +func.func @while_region_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 8}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 8 : Unknown}} + // expected-remark@below {{Result #1, ID 9 : 1, 8, 9, 10, 12, 13}} + // expected-remark@below {{Result #2, ID 10 : 1, 8, 9, 10, 12, 13}} + // expected-remark@below {{Region #0, Arg #0, ID 2 : 1, 2, 8}} + // expected-remark@below {{Region #0, Arg #1, ID 3 : 1, 3, 8}} + // expected-remark@below {{Region #0, Arg #2, ID 4 : 1, 4, 8}} + // expected-remark@below {{Region #1, Arg #0, ID 5 : 1, 5, 8}} + // expected-remark@below {{Region #1, Arg #1, ID 6 : 1, 6, 8}} + // expected-remark@below {{Region #1, Arg #2, ID 7 : 1, 7, 8}} + %w:3 = "tf.WhileRegion"(%arg0, %arg1, %arg2) ({ + ^bb0(%carg0: !tf_res, %carg1: !tf_res, %carg2: !tf_res): + %0 = arith.constant dense : tensor + "tf.Yield"(%0) : (tensor) -> () + },{ + ^bb0(%barg0: !tf_res, %barg1: !tf_res, %barg2: !tf_res): + // expected-remark@below {{Result #0, ID 1 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + "tf.Yield"(%u0, %barg2, %barg1) : (!tf_res, !tf_res, !tf_res) -> () + }) {is_stateless = false} : (!tf_res, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + func.return +} + +// ----- +// Test aliasing through calls +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @aliasing_through_calls +func.func @aliasing_through_calls(%arg0: tensor<32xf32>) -> () { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 0, 1, 2}} + %c:2 = func.call @passthru(%vh0) : (!tf_res) -> (!tf_res, !tf_res) + func.return +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 1}} +func.func @passthru(%arg0: !tf_res) -> (!tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + func.return %vh0, %arg0 : !tf_res, !tf_res +} + +// ----- +// Test aliasing through tf_device.launch +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @aliasing_through_launch +func.func @aliasing_through_launch(%arg0: tensor<32xf32>) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %vh = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> !tf_res + + // expected-remark@below {{Result #0, ID 1 : 0, 1}} + %launch = "tf_device.launch"() ({ + tf_device.return %vh : !tf_res + }) {device = ""} : () -> !tf_res + func.return +} + +// ----- +// Test aliasing through tf_device.cluster +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @aliasing_through_cluster +func.func @aliasing_through_cluster(%arg0: tensor<32xf32>) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %vh = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> !tf_res + + // expected-remark@below {{Result #0, ID 1 : 0, 1}} + %cluster = "tf_device.cluster"() ({ + tf_device.return %vh : !tf_res + }) : () -> !tf_res + func.return +} + +// ----- + +// Tests that ops with trait `TF_UniqueResourceAllocation` are not aliasing. + +// CHECK-LABEL: func @unique_resource_allocation +func.func @unique_resource_allocation(%arg0: tensor) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %stack_handle1 = "tf.StackV2"(%arg0) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // expected-remark@below {{Result #0, ID 1 : 1}} + %stack_handle2 = "tf.StackV2"(%arg0) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + func.return +} + +// ----- + +!tf_res = tensor<*x!tf_type.resource>> + +// Tests that ops with different known resource types get different resource IDs +// assigned, even if resource instances are unknown. +func.func @known_different_resource_types_unknown_instances(%arg0: tensor) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %iter_handle = "tf.IteratorV2"() {container = "c", shared_name = "v0", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 1}} + %seed_handle = "tf.DummySeedGenerator"() : () -> !tf_res + func.return +} + +// ----- + +!tf_res = tensor<*x!tf_type.resource>> + +// Tests that ops with same known resource type get same resource ID assigned +// (not unknown ID) if resource instances are unknown. +func.func @known_same_resource_types_unknown_instances(%arg0: tensor) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %iter_handle1 = "tf.IteratorV2"() {container = "c", shared_name = "v0", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1}} + %iter_handle2 = "tf.IteratorV2"() {container = "c", shared_name = "v1", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + func.return +} + +// ----- + +!tf_res = tensor<*x!tf_type.resource>> + +// Tests that an allocated resource is correctly propagated to island and graph +// results. +func.func @allocated_resource_propagation_island_graph() { + // expected-remark@below {{Result #0, ID 2 : 0, 1, 2}} + %graph = tf_executor.graph { + // CHECK: tf_executor.island + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %island:2 = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2}} + %iter_handle = "tf.IteratorV2"() {container = "c", shared_name = "v0", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + tf_executor.yield %iter_handle : !tf_res + } + tf_executor.fetch %island#0 : !tf_res + } + func.return +} + +// ----- + +!tf_res = tensor<*x!tf_type.resource>> + +// Tests that aliasing and non-aliasing values are correctly identified through +// multiple islands (`%iter_handle1`, `%iter_handle2`, `%island1#0` and +// `%island3#0` all point to the same resource here). +func.func @multiple_islands() { + %graph = tf_executor.graph { + // CHECK: tf_executor.island + // expected-remark@below {{Result #0, ID 2 : 0, 2, 3, 4}} + %island1:2 = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 2, 3, 4}} + %iter_handle1 = "tf.IteratorV2"() {container = "c", shared_name = "v0", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 1}} + %seed_handle = "tf.DummySeedGenerator"() : () -> !tf_res + tf_executor.yield %iter_handle1 : !tf_res + } + %island2:2 = tf_executor.island { + %1 = "tf.IteratorGetNext"(%island1#0) : (!tf_res) -> tensor + tf_executor.yield %1 : tensor + } + // expected-remark@below {{Result #0, ID 4 : 0, 2, 3, 4}} + %island3:2 = tf_executor.island { + // expected-remark@below {{Result #0, ID 3 : 0, 2, 3, 4}} + %iter_handle2 = "tf.IteratorV2"() {container = "c", shared_name = "v0", output_shapes = [#tf_type.shape<>], output_types = [!tf_res]} : () -> !tf_res + tf_executor.yield %iter_handle2 : !tf_res + } + tf_executor.fetch %island2#0 : tensor + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d0db403dfe9638474fffaff68bb85765ed82ffb9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir @@ -0,0 +1,540 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -tf-resource-device-inference %s | FileCheck %s + +!tf_res = tensor<*x!tf_type.resource>> + +// Tests that the pass can correctly propagate device attributes inside the same +// function. + +// CHECK-LABEL: func @propagate_in_function +func.func @propagate_in_function( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: !tf_res {tf.device = "/TPU:1"}) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/CPU:0"} + : () -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/CPU:0"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) + -> !tf_res + %read = "tf.ReadVariableOp"(%id2) : (!tf_res) -> tensor<32xf32> + %id3 = "tf.Identity"(%read) : (tensor<32xf32>) -> tensor<32xf32> + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// ----- +!tf_res = tensor<*x!tf_type.resource>> + +// Tesets that the pass can propagate through tf.If's branches. + +// CHECK-LABEL: func @propagate_if_op +func.func @propagate_if_op( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.If" + "tf.If"(%arg1, %id0, %var_handle) { + then_branch = @if_then, + else_branch = @if_else, + is_stateless = false} + : (tensor, !tf_res, + !tf_res) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @if_then +func.func @if_then( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @if_else +func.func @if_else( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + + +// ----- +!tf_res = tensor<*x!tf_type.resource>> + +// Tesets that the pass can propagate through tf.While's branches. +// CHECK-LABEL: func @propagate_while_op +func.func @propagate_while_op( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.While" + "tf.While"(%arg1, %id0, %var_handle) { + body = @while_body, + cond = @while_cond, is_stateless = false} + : (tensor, !tf_res, + !tf_res) -> + (tensor, !tf_res, + !tf_res) + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @while_body +func.func @while_body( + %arg0: tensor, + %arg1: !tf_res, + %arg2: !tf_res) -> + (tensor, !tf_res, + !tf_res) { + %graph:3 = tf_executor.graph { + // CHECK: tf_executor.island + %island:4 = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg2) : (!tf_res) + -> !tf_res + tf_executor.yield %arg0, %id0, %id1 + : tensor, !tf_res, + !tf_res + } + tf_executor.fetch %island#0, %island#1, %island#2 + : tensor, !tf_res, + !tf_res + } + func.return %graph#0, %graph#1, %graph#2 + : tensor, !tf_res, + !tf_res +} + +// CHECK-LABEL: func @while_cond +func.func @while_cond( + %arg0: tensor, + %arg1: !tf_res, + %arg2: !tf_res) -> tensor<32xf32> { + %graph = tf_executor.graph { + // CHECK: tf_executor.island + %island:2 = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + %read = "tf.ReadVariableOp"(%id0) + : (!tf_res) -> tensor<32xf32> + tf_executor.yield %read : tensor<32xf32> + } + tf_executor.fetch %island#0 : tensor<32xf32> + } + func.return %graph : tensor<32xf32> +} + +// ----- +!tf_res = tensor<*x!tf_type.resource>> + +// Tesets that the pass reports error on conflicting assignments from multiple +// callers. + +func.func @error_on_conflict_multiple_callers( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + %island = tf_executor.island { + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + "tf.If"(%arg1, %id0, %var_handle) { + then_branch = @if_then_and_else, + else_branch = @if_then_and_else, is_stateless = false} + : (tensor, !tf_res, + !tf_res) -> () + "tf.If"(%arg1, %var_handle, %id0) { + // expected-error@above {{Conflicting device assignment for resource}} + then_branch = @if_then_and_else, + else_branch = @if_then_and_else, + is_stateless = false} + : (tensor, !tf_res, + !tf_res) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +func.func @if_then_and_else( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + %island = tf_executor.island { + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// ----- + +// Test that the pass can propagate through calls +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @test_function +// CHECK-SAME: {tf.device = "/TPU:0"} +func.func @test_function(%arg0: !tf_res) { + // CHECK: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + %read = "tf.ReadVariableOp"(%id0) : (!tf_res) -> tensor<32xf32> + %cst = arith.constant dense<3.0> : tensor<32xf32> + %add = "tf.AddV2"(%read, %cst) : (tensor<32xf32>, tensor<32xf32>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %add) : (!tf_res, tensor<32xf32>) -> () + func.return +} + +// CHECK-LABEL: func @propagate_through_calls +func.func @propagate_through_calls( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: !tf_res {tf.device = "/TPU:1"}) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/CPU:0"} + : () -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/CPU:0"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) + -> !tf_res + %read = "tf.ReadVariableOp"(%id2) : (!tf_res) -> tensor<32xf32> + %id3 = "tf.Identity"(%read) : (tensor<32xf32>) -> tensor<32xf32> + func.call @test_function(%id1) : (!tf_res) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// Test propagation through IfRegion (with non-inlined calls) +// CHECK-LABEL: func @propagate_if_region +func.func @propagate_if_region( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.IfRegion" + "tf.IfRegion"(%arg1) ({ + func.call @ifregion_then(%id0, %var_handle) : (!tf_res, !tf_res) -> () + "tf.Yield"() : () -> () + }, { + func.call @ifregion_else(%id0, %var_handle) : (!tf_res, !tf_res) -> () + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @ifregion_then +// CHECK-SAME: (%arg0: {{.+}} {tf.device = "/TPU:0"}, %arg1: {{.+}} {tf.device = "/TPU:1"} +func.func @ifregion_then( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @ifregion_else +// CHECK-SAME: (%arg0: {{.+}} {tf.device = "/TPU:0"}, %arg1: {{.+}} {tf.device = "/TPU:1"} +func.func @ifregion_else( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// Test progagation through IfRegion (inlined calls) +// CHECK-LABEL: func @propagate_if_region_inlined +func.func @propagate_if_region_inlined( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.IfRegion" + "tf.IfRegion"(%arg1) ({ + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + "tf.Yield"() : () -> () + }, { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// Test propagation through WhileRegion (inlined calls) +// CHECK-LABEL: func @propagate_while_region_inlined +func.func @propagate_while_region_inlined( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} : () -> !tf_res + // CHECK-NEXT: "tf.WhileRegion" + "tf.WhileRegion"(%arg1, %id0, %var_handle) ({ + ^bb0(%carg0: tensor, %carg1: !tf_res, %carg2: !tf_res): + // CHECK: ^bb + // CHECK: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %cid0 = "tf.Identity"(%carg1) : (!tf_res) -> !tf_res loc("cid0") + %read = "tf.ReadVariableOp"(%cid0) : (!tf_res) -> tensor<32xf32> + %cst = arith.constant dense<3.0> : tensor<32xf32> + %cmp = "tf.Less"(%read, %cst) : (tensor<32xf32>, tensor<32xf32>) -> tensor<32xi1> + %dims = arith.constant dense<0> : tensor<1xi32> + %reduce = "tf.All"(%cmp, %dims) {keep_dims = false} : (tensor<32xi1>, tensor<1xi32>) -> tensor + "tf.Yield"(%reduce) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: !tf_res, %barg2: !tf_res): + // CHECK: ^bb + // CHECK: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %bid0 = "tf.Identity"(%barg1) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%barg2) : (!tf_res) -> !tf_res + "tf.Yield"(%barg0, %bid0, %id1) : (tensor, !tf_res,!tf_res) -> () + }){is_stateless = false} + : (tensor, !tf_res, !tf_res) -> (tensor, !tf_res, !tf_res) + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// Test propagation through WhileRegion (non-inlined calls) +// CHECK-LABEL: func @propagate_while_region +func.func @propagate_while_region( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} : () -> !tf_res + // CHECK-NEXT: "tf.WhileRegion" + "tf.WhileRegion"(%arg1, %id0, %var_handle) ({ + ^bb0(%carg0: tensor, %carg1: !tf_res, %carg2: !tf_res): + %cond = func.call @whileregion_cond(%carg0, %carg1, %carg2) : (tensor, !tf_res, !tf_res) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: !tf_res, %barg2: !tf_res): + %new_values:3 = func.call @whileregion_body(%barg0, %barg1, %barg2) : (tensor, !tf_res,!tf_res) -> (tensor, !tf_res,!tf_res) + "tf.Yield"(%new_values#0, %new_values#1, %new_values#2) : (tensor, !tf_res,!tf_res) -> () + }){is_stateless = false} + : (tensor, !tf_res, !tf_res) -> (tensor, !tf_res, !tf_res) + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @whileregion_body +func.func @whileregion_body(%arg0: tensor, %arg1: !tf_res, %arg2: !tf_res) -> (tensor, !tf_res, !tf_res) { + %graph:3 = tf_executor.graph { + // CHECK: tf_executor.island + %island:4 = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg1) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg2) : (!tf_res) -> !tf_res + tf_executor.yield %arg0, %id0, %id1 : tensor, !tf_res, !tf_res + } + tf_executor.fetch %island#0, %island#1, %island#2 : tensor, !tf_res, !tf_res + } + func.return %graph#0, %graph#1, %graph#2: tensor, !tf_res, !tf_res +} + +// CHECK-LABEL: func @whileregion_cond +func.func @whileregion_cond(%arg0: tensor, %arg1: !tf_res, %arg2: !tf_res) -> tensor { + %graph = tf_executor.graph { + // CHECK: tf_executor.island + %island:2 = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg1) : (!tf_res) -> !tf_res + %read = "tf.ReadVariableOp"(%id0) : (!tf_res) -> tensor<32xf32> + %cst = arith.constant dense<3.0> : tensor<32xf32> + %cmp = "tf.Less"(%read, %cst) : (tensor<32xf32>, tensor<32xf32>) -> tensor<32xi1> + %dims = arith.constant dense<0> : tensor<1xi32> + %reduce = "tf.All"(%cmp, %dims) {keep_dims = false} : (tensor<32xi1>, tensor<1xi32>) -> tensor + tf_executor.yield %reduce : tensor + } + tf_executor.fetch %island#0 : tensor + } + func.return %graph : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_analyzer.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_analyzer.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fa54a6f5a2e4824d7cdd7fef470ed6e1182af9d9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_analyzer.mlir @@ -0,0 +1,57 @@ +// RUN: tf-opt -verify-diagnostics -allow-unregistered-dialect \ +// RUN: -split-input-file -tf-resource-analyzer-test %s \ +// RUN: | FileCheck %s + +// TODO(b/269548549): Add tests to cover more patterns. + +// Test that VarHandleOp is not marked as "potentially written" if it is not +// assigned inside the function called by "tf.BatchFunction". + +module { +// CHECK-LABEL: @serving_default + func.func @serving_default() -> (tensor<*xi32>) { + // expected-remark@below {{device: "", container: "", shared_name: "var_0", is_potentially_written: false}} + %0 = "tf.VarHandleOp"() {shared_name = "var_0"} : () -> tensor>> + %1 = "tf.BatchFunction"(%0) { + f = @called_by_batch_func, + operandSegmentSizes = array, + batch_timeout_micros = 1000, + max_batch_size = 8, + num_batch_threads = 2 + } : (tensor>>) -> tensor<*xi32> + return %1 : tensor<*xi32> + } + + func.func private @called_by_batch_func(%arg: tensor<*x!tf_type.resource>) -> tensor { + %0 = "tf.ReadVariableOp"(%arg) : (tensor<*x!tf_type.resource>) -> tensor + return %0 : tensor + } +} + +// ----- + +// Test that VarHandleOp is marked as "potentially written" if it is +// assigned inside the function called by "tf.BatchFunction". + +module { +// CHECK-LABEL: @serving_default + func.func @serving_default() -> (tensor<*xi32>) { + // expected-remark@below {{device: "", container: "", shared_name: "var_0", is_potentially_written: true}} + %0 = "tf.VarHandleOp"() {shared_name = "var_0"} : () -> tensor>> + %1 = "tf.BatchFunction"(%0) { + f = @called_by_batch_func_assign, + operandSegmentSizes = array, + batch_timeout_micros = 1000, + max_batch_size = 8, + num_batch_threads = 2 + } : (tensor>>) -> tensor<*xi32> + return %1 : tensor<*xi32> + } + + func.func private @called_by_batch_func_assign(%arg: tensor<*x!tf_type.resource>) -> tensor { + %0 = "tf.Const"() {value = dense<4> : tensor<2xi32>} : () -> tensor<2xi32> + "tf.AssignVariableOp"(%arg, %0) : (tensor<*x!tf_type.resource>, tensor<2xi32>) -> () + %1 = "tf.ReadVariableOp"(%arg) : (tensor<*x!tf_type.resource>) -> tensor + return %1 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_inlining.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_inlining.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8071de3fbf7c5fa26375434b8342a1e5960bb341 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_inlining.mlir @@ -0,0 +1,25 @@ +// RUN: tf-opt -tf-shape-inference -inline='default-pipeline=''' %s | FileCheck %s --dump-input=always +// RUN: tf-opt -tf-standard-pipeline=enable-inliner %s | FileCheck %s --dump-input=always + +// Tests function with argument has no resource subtype but caller operand has a +// resource subtype, and after shape inference, function argument is refined and +// no `tf.Cast` ops are generated. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 384 : i32}} { + // CHECK-LABEL: func @main + func.func @main() -> tensor { + // CHECK-NEXT: %[[VAR:.*]] = "tf.VarHandleOp" + // CHECK-NEXT: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK-NEXT: return %[[READ_VAR]] + // CHECK-NOT: "tf.Cast" + %0 = "tf.VarHandleOp"() {_class = ["loc:@Variable"], allowed_devices = [], container = "", device = "", shared_name = "Variable"} : () -> tensor>> + %1 = "tf.StatefulPartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor>>) -> tensor + func.return %1 : tensor + } + + // CHECK-NOT: func private @callee + func.func private @callee(%arg0: tensor) -> tensor<*xf32> attributes {tf.signature.is_stateful} { + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_op_lifting.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_op_lifting.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e61665c1ff716a9f6afa1bf6e3b05d28d34a77ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/resource_op_lifting.mlir @@ -0,0 +1,1447 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-resource-op-lifting | env FILECHECK_OPTS="" FileCheck %s + +// Tests that resource load operations are hoisted. + +// CHECK-LABEL: func @only_resource_load +func.func @only_resource_load() -> tensor<*xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + // CHECK: "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) + // CHECK: tf_device.return %[[COMPUTE_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> tensor<*xi32> + + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + func.return %1 : tensor<*xi32> +} + +// ----- + +// Tests that resource store operations are hoisted. + +// CHECK-LABEL: func @only_resource_store +func.func @only_resource_store() -> tensor<*xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[CLUSTER_RES:[0-9]*]]:2 = "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"() + // CHECK: tf_device.return %[[COMPUTE_RES]], %[[COMPUTE_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> (tensor<*xi32>, tensor<*xi32>) + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[CLUSTER_RES]]#1) + + %1 = "tf_device.cluster"() ({ + %2 = "tf.SomeComputation"() : () -> (tensor<*xi32>) + "tf.AssignVariableOp"(%0, %2) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + tf_device.return %2 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + func.return %1 : tensor<*xi32> +} + +// ----- + +// Tests that resource ops with both load and store are hoisted. + +// CHECK-LABEL: func @same_resource_load_and_store +func.func @same_resource_load_and_store() -> tensor<*xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + // CHECK: %[[CLUSTER_RES:[0-9]*]]:2 = "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) + // CHECK: tf_device.return %[[COMPUTE_RES]], %[[COMPUTE_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> (tensor<*xi32>, tensor<*xi32>) + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[CLUSTER_RES]]#1) + + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%0, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + func.return %1 : tensor<*xi32> +} + +// ----- + +// Tests that resource ops with both load and store are hoisted +// but input to load and output from store have mixed defined/undefined shapes. + +// CHECK-LABEL: func @same_resource_load_and_store_cast +func.func @same_resource_load_and_store_cast() -> tensor<1xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + // CHECK: %[[CLUSTER_RES:[0-9]*]]:2 = "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) + // CHECK: %[[CAST_RES:[0-9]*]] = "tf.Cast"(%[[COMPUTE_RES]]) + // CHECK: tf_device.return %[[CAST_RES]], %[[COMPUTE_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> (tensor<1xi32>, tensor<*xi32>) + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[CLUSTER_RES]]#1) + + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<1xi32> + %3 = "tf.SomeComputation"(%2) : (tensor<1xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%0, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + %4 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<1xi32> + tf_device.return %4 : tensor<1xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<1xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + func.return %1 : tensor<1xi32> +} + +// ----- + +// Tests that anonymous internal resource operations are eliminated. + +// CHECK-LABEL: func @anonymous_internal_resource +func.func @anonymous_internal_resource() -> tensor<*xi32> { + + // CHECK: %[[COMPUTE1_RES:[0-9]*]] = "tf.SomeComputation1"() + %0 = "tf.SomeComputation1"() : () -> (tensor<*xi32>) + + // CHECK: %[[CLUSTER_RES:[0-9]*]] = "tf_device.cluster" + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[COMPUTE2_RES:[0-9]*]] = "tf.SomeComputation2"(%[[COMPUTE1_RES]]) + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: tf_device.return %[[COMPUTE2_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> tensor<*xi32> + + %1 = "tf_device.cluster"() ( { + %1 = "tf.VarHandleOp"() {shared_name = "cd2c89b7-88b7-44c8-ad83-06c2a9158347"} : () -> tensor<*x!tf_type.resource>> + "tf.AssignVariableOp"(%1, %0) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + %2 = "tf.ReadVariableOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation2"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%1, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + %4 = "tf.ReadVariableOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + tf_device.return %4 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]] + return %1 : tensor<*xi32> +} + +// ----- + +// Tests that anonymous internal resource operations (including DestroyResourceOp) are eliminated. + +// CHECK-LABEL: func @anonymous_internal_resource_with_destroy +func.func @anonymous_internal_resource_with_destroy() -> tensor<*xi32> { + + // CHECK: %[[COMPUTE1_RES:[0-9]*]] = "tf.SomeComputation1"() + %0 = "tf.SomeComputation1"() : () -> (tensor<*xi32>) + + // CHECK: %[[CLUSTER_RES:[0-9]*]] = "tf_device.cluster" + // CHECK-NOT: "tf.VarHandleOp" + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: %[[COMPUTE2_RES:[0-9]*]] = "tf.SomeComputation2"(%[[COMPUTE1_RES]]) + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK-NOT: "tf.DestroyResourceOp" + // CHECK: tf_device.return %[[COMPUTE2_RES]] + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> tensor<*xi32> + + %1 = "tf_device.cluster"() ( { + %1 = "tf.VarHandleOp"() {shared_name = "cd2c89b7-88b7-44c8-ad83-06c2a9158347"} : () -> tensor<*x!tf_type.resource>> + "tf.AssignVariableOp"(%1, %0) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + %2 = "tf.ReadVariableOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation2"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%1, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + %4 = "tf.ReadVariableOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + "tf.DestroyResourceOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> () + tf_device.return %4 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]] + return %1 : tensor<*xi32> +} + +// ----- + +// Tests that named internal resource operations are not hoisted. + +// CHECK-LABEL: func @named_internal_resource +func.func @named_internal_resource() -> tensor<*xi32> { + + // CHECK: %[[CLUSTER_RES:[0-9]*]] = "tf_device.cluster" + %0 = "tf_device.cluster"() ( { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + %2 = "tf.ReadVariableOp"(%1) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[COMPUTE_RES]]) + "tf.AssignVariableOp"(%1, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + + // CHECK: tf_device.return %[[COMPUTE_RES]] + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]] + return %0 : tensor<*xi32> +} + +// ----- + +// Tests that pass lifts resource reads/writes from a loop, and removed unused +// resources. + +// CHECK-LABEL: func @cluster_with_loop +func.func @cluster_with_loop() -> () { + // CHECK: %[[COUNT:.*]] = "tf.Const"() <{value = dense<10> : tensor}> + %0 = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %unused = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + "tf_device.cluster"() ({ + // CHECK: %[[WHILE:.*]]:2 = "tf.While"(%[[COUNT]], %[[READ]]) + %2:3 = "tf.While"(%0, %1, %unused) + {body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + // CHECK: tf_device.return %[[WHILE]]#1 : tensor + tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} : () -> tensor + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK: "tf.AssignVariableOp"(%[[VH]], %[[CLUSTER]]) + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor, %[[BARG1:.*]]: tensor) +func.func @while_body(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor<*x!tf_type.resource>>) + -> (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) { + %read0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: %[[ADD0:.*]] = "tf.AddV2"(%[[BARG1]], %[[BARG1]]) + %add0 = "tf.AddV2"(%read0, %read0) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%arg1, %add0) : (tensor<*x!tf_type.resource>>, tensor) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: %[[ADD1:.*]] = "tf.AddV2"(%[[ADD0]], %[[ADD0]]) + %add1 = "tf.AddV2"(%read1, %read1) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%arg1, %add1) : (tensor<*x!tf_type.resource>>, tensor) -> () + // CHECK-NEXT: %[[DELTA:.*]] = "tf.Const"() <{value = dense<-1> : tensor}> + %constant = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + // CHECK-NEXT: %[[ADD2:.*]] = "tf.AddV2"(%[[BARG0]], %[[DELTA]]) + %add2 = "tf.AddV2"(%arg0, %constant) : (tensor, tensor) -> tensor + // CHECK-NEXT: return %[[ADD2]], %[[ADD1]] + %id = "tf.Identity"(%arg2) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + func.return %add2, %arg1, %id : tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>> +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor, %[[CARG1:.*]]: tensor) +func.func @while_cond(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor<*x!tf_type.resource>>) -> tensor { + // CHECK-NEXT: return %[[CARG0]] + func.return %arg0 : tensor +} + +// ----- + +// Tests that pass lifts resource reads from loop condition. + +// CHECK-LABEL: func @cluster_with_loop +func.func @cluster_with_loop() -> () { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + "tf_device.cluster"() ({ + // CHECK: %[[WHILE:.*]] = "tf.While"(%[[READ]]) + %1 = "tf.While"(%0) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + // CHECK: tf_device.return %[[WHILE]] : tensor + tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} : () -> tensor + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK: "tf.AssignVariableOp"(%[[VH]], %[[CLUSTER]]) + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor) +func.func @while_body(%arg0: tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + %constant = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]] + func.return %arg0 : tensor<*x!tf_type.resource>> +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor) +func.func @while_cond(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %id = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: return %[[CARG0]] + func.return %read : tensor +} + +// ----- + +// Tests that pass lifts read-only resource reads from loop, but does not add +// assign after the loop. + +// CHECK-LABEL: func @cluster_with_loop +func.func @cluster_with_loop() -> () { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: "tf_device.cluster"() + "tf_device.cluster"() ({ + // CHECK: %[[WHILE:.*]] = "tf.While"(%[[READ]]) + %1 = "tf.While"(%0) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + // CHECK: tf_device.return + tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} : () -> () + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK-NOT: "tf.AssignVariableOp" + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor) +func.func @while_body(%arg0: tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: return %[[BARG0]] + func.return %arg0 : tensor<*x!tf_type.resource>> +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor) +func.func @while_cond(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %read = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: return %[[CARG0]] + func.return %read : tensor +} + +// ----- + +// Tests that pass lifts resource reads from nested loops. + +// CHECK-LABEL: func @cluster_with_nested_loop +func.func @cluster_with_nested_loop() -> () { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[VH_UNUSED:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + "tf_device.cluster"() ({ + // CHECK: %[[WHILE:.*]] = "tf.While"(%[[READ]]) + %2:2 = "tf.While"(%0, %1) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + // CHECK: tf_device.return %[[WHILE]] : tensor + tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} : () -> tensor + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK: "tf.AssignVariableOp"(%[[VH]], %[[CLUSTER]]) + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor) +func.func @while_body(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) { + // CHECK: %[[WHILE:.*]] = "tf.While"(%[[BARG0]]) + %0:2 = "tf.While"(%arg0, %arg1) { + body = @while_body1, cond = @while_cond1, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + // CHECK-NEXT: return %[[WHILE]] + func.return %0#0, %0#1 : tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>> +} +// CHECK: func @while_cond(%arg0: tensor) +func.func @while_cond(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) -> tensor { + %id = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: return %[[CARG0]] + func.return %read : tensor +} +// CHECK: func @while_body1(%[[BARG0:.*]]: tensor) +func.func @while_body1(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + %constant = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]] + func.return %arg0, %arg1 : tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>> +} +// CHECK: func @while_cond1(%arg0: tensor) +func.func @while_cond1(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) -> tensor { + %id = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: return %[[CARG0]] + func.return %read : tensor +} + +// ----- + +// Tests that pass reports error on non-aliasing while input/output resources. + +func.func @cluster_with_loop() -> () { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + "tf_device.cluster"() ({ + // expected-error@+1 {{result #0 not tied to function argument for branch @while_body}} + %1 = "tf.While"(%0) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) + tf_device.return + }) {cluster_attr = "cluster_attr"} : () -> () + func.return +} +func.func @while_body(%arg0: tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + func.return %0 : tensor<*x!tf_type.resource>> +} +func.func @while_cond(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %read = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + func.return %read : tensor +} + +// ----- + +// Tests that the pass reports error on non-aliasing WhileRegion input/output +// resources. It cannot lift resource ops from such WhileRegion ops and should +// fail with a helpful error message. + +func.func @fail_non_aliasing_resource_input_output() -> () { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + "tf_device.cluster"() ({ + // expected-error@+1 {{Result #0 is not tied to arg #0 of the body}} + %1 = "tf.WhileRegion"(%0) ({ + ^bb0(%carg0:tensor<*x!tf_type.resource>>): + %cond = "tf.SomeOp"() : () -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%carg0:tensor<*x!tf_type.resource>>): + %body = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + "tf.Yield"(%body) : (tensor<*x!tf_type.resource>>) -> () + }) { is_stateless = false } + : (tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) + tf_device.return + }) {cluster_attr = "cluster_attr"} : () -> () + func.return +} + +// ----- + +// Tests that pass reports error on unsupported ops in loop cond. + +func.func @cluster_with_loop() -> () { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + "tf_device.cluster"() ({ + // expected-error@+1 {{found resource write in loop condition.}} + %1 = "tf.While"(%0) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) + tf_device.return + }) {cluster_attr = "cluster_attr"} : () -> () + func.return +} +func.func @while_body(%arg0: tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>>) { + %constant = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return %arg0 : tensor<*x!tf_type.resource>> +} +func.func @while_cond(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %read = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + %constant = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return %read : tensor +} + +// ----- + +// CHECK: func @cluster_with_case(%[[ARG0:.*]]: tensor) -> tensor<4xf32> +func.func @cluster_with_case(%arg0: tensor) -> tensor<4xf32> { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK-DAG: %[[READ0:.*]] = "tf.ReadVariableOp"(%[[VH0]]) + // CHECK-DAG: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + %2 = "tf_device.cluster"() ({ + // CHECK: %[[CASE:.*]]:2 = "tf.Case"(%[[ARG0]], %[[READ0]], %[[READ1]]) + %3:2 = "tf.Case"(%arg0, %0, %1) {branches = [@branch_0, @branch_1, @branch_2], is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) + // CHECK-NEXT: %[[ADD:.*]] = "tf.AddV2"(%[[CASE]]#1, %[[CASE]]#0) + %4 = "tf.ReadVariableOp"(%3#0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %5 = "tf.AddV2"(%4, %3#1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + // CHECK-NEXT: tf_device.return %[[ADD]], %[[CASE]]#1 + tf_device.return %5 : tensor<4xf32> + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor<4xf32>, tensor<4xf32>) + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor<4xf32> +} +// CHECK: func @branch_0(%[[TARG0:.*]]: tensor<4xf32>, %[[TARG1:.*]]: tensor<4xf32>) +func.func @branch_0(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + // CHECK-NEXT: return %[[CONST]], %[[CONST]] + func.return %arg0, %constant : tensor<*x!tf_type.resource>>, tensor<4xf32> +} +// CHECK: func @branch_1(%[[EARG0:.*]]: tensor<4xf32>, %[[EARG1:.*]]: tensor<4xf32>) +func.func @branch_1(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) { + %id = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.AssignVariableOp"(%arg0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + // CHECK-NEXT: return %[[EARG1]], %[[EARG1]] + func.return %arg0, %read : tensor<*x!tf_type.resource>>, tensor<4xf32> +} +// CHECK: func @branch_2(%[[EARG0:.*]]: tensor<4xf32>, %[[EARG1:.*]]: tensor<4xf32>) +func.func @branch_2(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) { + %id = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.AssignVariableOp"(%arg0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + // CHECK-NEXT: return %[[EARG1]], %[[EARG1]] + func.return %arg0, %read : tensor<*x!tf_type.resource>>, tensor<4xf32> +} + +// ----- + +// Tests that pass lifts resource reads from if branches. + +// CHECK: func @cluster_with_if(%[[ARG0:.*]]: tensor) -> tensor<4xf32> +func.func @cluster_with_if(%arg0: tensor) -> tensor<4xf32> { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK-DAG: %[[READ0:.*]] = "tf.ReadVariableOp"(%[[VH0]]) + // CHECK-DAG: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + %2 = "tf_device.cluster"() ({ + // CHECK: %[[IF:.*]]:2 = "tf.If"(%[[ARG0]], %[[READ0]], %[[READ1]]) + %3:2 = "tf.If"(%arg0, %0, %1) {then_branch = @if_then, else_branch = @if_else, + is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) + // CHECK-NEXT: %[[ADD:.*]] = "tf.AddV2"(%[[IF]]#1, %[[IF]]#0) + %4 = "tf.ReadVariableOp"(%3#0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %5 = "tf.AddV2"(%4, %3#1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + // CHECK-NEXT: tf_device.return %[[ADD]], %[[IF]]#1 + tf_device.return %5 : tensor<4xf32> + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor<4xf32>, tensor<4xf32>) + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor<4xf32> +} +// CHECK: func @if_then(%[[TARG0:.*]]: tensor<4xf32>, %[[TARG1:.*]]: tensor<4xf32>) +func.func @if_then(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + // CHECK-NEXT: return %[[CONST]], %[[CONST]] + func.return %arg0, %constant : tensor<*x!tf_type.resource>>, tensor<4xf32> +} +// CHECK: func @if_else(%[[EARG0:.*]]: tensor<4xf32>, %[[EARG1:.*]]: tensor<4xf32>) +func.func @if_else(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) { + %id = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.AssignVariableOp"(%arg0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + // CHECK-NEXT: return %[[EARG1]], %[[EARG1]] + func.return %arg0, %read : tensor<*x!tf_type.resource>>, tensor<4xf32> +} + +// ----- + +// Tests that pass lifts resource reads from nested if ops. + +// CHECK: func @cluster_with_nested_if(%[[ARG0:.*]]: tensor) -> tensor +func.func @cluster_with_nested_if(%arg0: tensor) -> tensor { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK-DAG: %[[READ0:.*]] = "tf.ReadVariableOp"(%[[VH0]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + %2 = "tf_device.cluster"() ({ + // CHECK: %[[IF:.*]] = "tf.If"(%[[ARG0]], %[[READ0]]) + %3 = "tf.If"(%arg0, %0, %1) {then_branch = @if_then, else_branch = @if_else, + is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + // CHECK-NEXT: %[[ADD:.*]] = "tf.AddV2"(%[[IF]], %[[IF]]) + %4 = "tf.ReadVariableOp"(%3) : (tensor<*x!tf_type.resource>>) -> tensor + %5 = "tf.AddV2"(%4, %4) : (tensor, tensor) -> tensor + // CHECK-NEXT: tf_device.return %[[ADD]], %[[IF]] + tf_device.return %5 : tensor + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor, tensor) + }) {cluster_attr = "cluster_attr"} : () -> tensor + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor +} +// CHECK: func @if_then(%[[TARG0:.*]]: tensor) +func.func @if_then(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: %[[IIF:.*]] = "tf.If"(%[[TARG0]], %[[TARG0]]) + %read = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + %3 = "tf.If"(%read, %arg0) {then_branch = @inner_if_then, else_branch = @inner_if_else, + is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + // CHECK-NEXT: return %[[IIF]] + func.return %3 : tensor<*x!tf_type.resource>> +} +// CHECK: func @if_else(%[[EARG0:.*]]: tensor) +func.func @if_else(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: return %[[EARG0]] + func.return %arg0 : tensor<*x!tf_type.resource>> +} +// CHECK: func @inner_if_then(%[[ITARG0:.*]]: tensor) +func.func @inner_if_then(%arg0: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + %constant = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %constant) : (tensor<*x!tf_type.resource>>, tensor) -> () + // CHECK-NEXT: return %[[CONST]] + func.return %arg0 : tensor<*x!tf_type.resource>> +} +// CHECK: func @inner_if_else(%[[IEARG0:.*]]: tensor) +func.func @inner_if_else(%arg0: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: return %[[IEARG0]] + func.return %arg0 : tensor<*x!tf_type.resource>> +} + +// ----- + +// Tests that the pass reports error for ambiguous resource aliasing. + +func.func @cluster_with_if(%arg0: tensor) -> tensor<4xf32> { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + // expected-error @+1 {{result #0 is not tied to the same argument across all branches}} + %3 = "tf.If"(%arg0, %0, %1) {then_branch = @if_then, else_branch = @if_else, + is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + %4 = "tf.ReadVariableOp"(%3) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + tf_device.return %4 : tensor<4xf32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + func.return %2 : tensor<4xf32> +} +func.func @if_then(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + func.return %arg0 : tensor<*x!tf_type.resource>> +} +func.func @if_else(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + func.return %arg1 : tensor<*x!tf_type.resource>> +} + +// ----- + +// Tests that the pass reports error if output does not alias input. + +func.func @cluster_with_if(%arg0: tensor) -> tensor<4xf32> { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + // expected-error @+1 {{result #0 not tied to function argument for branch @if_then}} + %3 = "tf.If"(%arg0, %0, %1) {then_branch = @if_then, else_branch = @if_else, + is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) + %4 = "tf.ReadVariableOp"(%3) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + tf_device.return %4 : tensor<4xf32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + func.return %2 : tensor<4xf32> +} +func.func @if_then(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + %0 = "tf.foo"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + func.return %0 : tensor<*x!tf_type.resource>> +} +func.func @if_else(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) + -> (tensor<*x!tf_type.resource>>) { + %0 = "tf.bar"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + func.return %0 : tensor<*x!tf_type.resource>> +} + +// ----- + +// Tests that the pass lifts resources on two partitioned call ops sharing the +// same callee. The lifting should clone the callee then modify the clone. + +// CHECK-LABEL: @cluster_with_partitioned_call +func.func @cluster_with_partitioned_call() -> tensor { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[CONST:.*]] = "tf.Const"() + %1 = "tf.Const"() {value = dense<10.0> : tensor} : () -> tensor + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + %2 = "tf_device.cluster"() ({ + // CHECK: %[[PC0:.*]] = "tf.PartitionedCall"(%[[CONST]], %[[READ]], %[[CONST]]) + // CHECK-SAME: f = @callee_resource_lifted + %3 = "tf.PartitionedCall"(%1, %0, %1) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor<*x!tf_type.resource>>, tensor) -> tensor + // CHECK: %[[PC1:.*]] = "tf.PartitionedCall"(%[[CONST]], %[[READ]], %[[CONST]]) + // CHECK-SAME: f = @callee_resource_lifted + %4 = "tf.PartitionedCall"(%1, %0, %1) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor<*x!tf_type.resource>>, tensor) -> tensor + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[PC0]], %[[PC1]]) + %5 = "tf.AddV2"(%3, %4) : (tensor, tensor) -> tensor + // CHECK: tf_device.return %[[ADD]] : tensor + tf_device.return %5 : tensor + }) {cluster_attr = "cluster_attr"} : () -> tensor + func.return %2 : tensor +} +// CHECK: @callee(%[[OA0:.*]]: tensor, %[[OA1:.*]]: tensor<*x!tf_type.resource>>, %[[OA2:.*]]: tensor) -> tensor +func.func @callee(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor) -> tensor { + // CHECK: "tf.ReadVariableOp"(%[[OA1]]) + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor + %1 = "tf.AddV2"(%0, %arg0) : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%1, %arg2) : (tensor, tensor) -> tensor + func.return %2 : tensor +} +// CHECK: func private @callee_resource_lifted(%[[A0:.*]]: tensor, %[[A1:.*]]: tensor, %[[A2:.*]]: tensor) -> tensor +// CHECK-NEXT: %[[ADD0:.*]] = "tf.AddV2"(%[[A1]], %[[A0]]) +// CHECK-NEXT: %[[ADD1:.*]] = "tf.AddV2"(%[[ADD0]], %[[A2]]) +// CHECK-NEXT: return %[[ADD1]] + + +// ----- + +// Tests that the pass lifts resources on two stateful partitioned call ops +// sharing the same callee. The lifting should clone the callee then modify the +// clone. + +// CHECK-LABEL: @cluster_with_stateful_partitioned_call +func.func @cluster_with_stateful_partitioned_call() -> () { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + // CHECK: %[[CONST:.*]] = "tf.Const"() + %2 = "tf.Const"() {value = dense<10.0> : tensor} : () -> tensor + // CHECK-DAG: %[[READ0:.*]] = "tf.ReadVariableOp"(%[[VH0]]) + // CHECK-DAG: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + "tf_device.cluster"() ({ + // CHECK: %[[PC0:.*]] = "tf.StatefulPartitionedCall"(%[[READ0]], %[[READ1]], %[[CONST]]) + // CHECK-SAME: f = @callee_resource_lifted + %3 = "tf.StatefulPartitionedCall"(%0, %1, %2) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor) -> tensor<*x!tf_type.resource>> + // CHECK: %[[PC1:.*]] = "tf.StatefulPartitionedCall"(%[[PC0]], %[[READ1]], %[[CONST]]) + // CHECK-SAME: f = @callee_resource_lifted + %4 = "tf.StatefulPartitionedCall"(%3, %1, %2) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor) -> tensor<*x!tf_type.resource>> + // CHECK: tf_device.return %[[PC1]] : tensor + tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} : () -> tensor + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]) + func.return +} +// CHECK: @callee(%[[OA0:.*]]: tensor<*x!tf_type.resource>>, %[[OA1:.*]]: tensor<*x!tf_type.resource>>, %[[OA2:.*]]: tensor) -> tensor<*x!tf_type.resource>> +func.func @callee(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor) -> tensor<*x!tf_type.resource>> { + // CHECK: "tf.ReadVariableOp"(%[[OA1]]) + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor + %1 = "tf.AddV2"(%0, %arg2) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%arg0, %1) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return %arg0 : tensor<*x!tf_type.resource>> +} +// CHECK: func private @callee_resource_lifted(%[[A0:.*]]: tensor, %[[A1:.*]]: tensor, %[[A2:.*]]: tensor) -> tensor +// CHECK-NEXT: %[[ADD:.*]] = "tf.AddV2"(%[[A1]], %[[A2]]) +// CHECK-NEXT: return %[[ADD]] + + +// ----- + +// Tests that the pass reports error on called function that has resource output +// which doesn't alias an input. + +func.func @cluster_with_stateful_partitioned_call() -> () { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf.Const"() {value = dense<10.0> : tensor} : () -> tensor + "tf_device.cluster"() ({ + %3 = "tf.StatefulPartitionedCall"(%0, %1, %2) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor) -> tensor<*x!tf_type.resource>> + %4 = "tf.StatefulPartitionedCall"(%3, %1, %2) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor) -> tensor<*x!tf_type.resource>> + tf_device.return + }) {cluster_attr = "cluster_attr"} : () -> () + func.return +} +// expected-error @+1 {{unsupported function call: resource return value does not alias an input.}} +func.func @callee(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor) -> tensor<*x!tf_type.resource>> { + %0 = "tf._Unknown_"() : () -> tensor<*x!tf_type.resource>> + func.return %0 : tensor<*x!tf_type.resource>> +} + +// ----- + +// Tests call op where it's result is the result of a tf.ReadVariableOp. + +// CHECK-LABEL: func @call_with_forwarded_read_only_result +// CHECK-SAME: (%[[RESOURCE_ARG0:.*]]: tensor<*x!tf_type.resource>>) +func.func @call_with_forwarded_read_only_result(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[RESOURCE_ARG0]]) + %0 = "tf_device.cluster"() ({ + // CHECK: %[[CALL:.*]] = "tf.StatefulPartitionedCall"(%[[READ]]) + %1 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor<*x!tf_type.resource>>) -> tensor + // CHECK-NEXT: tf_device.return %[[CALL]] + tf_device.return %1 : tensor + }) {} : () -> tensor + func.return +} + +func.func @callee(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor + %1 = "tf.Identity"(%0) {device = ""} : (tensor) -> tensor + func.return %1 : tensor +} + +// CHECK: func private @callee_resource_lifted(%[[A0:.*]]: tensor) -> tensor +// CHECK-NEXT: return %[[A0]] + +// ----- + +// Test that the pass can lift resources out of IfRegion +// CHECK: func @cluster_with_ifregion(%[[ARG0:.*]]: tensor) -> tensor<4xf32> +func.func @cluster_with_ifregion(%arg0: tensor) -> tensor<4xf32> { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + // CHECK: %[[IF:.*]]:2 = "tf.IfRegion"(%[[ARG0]]) + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + %3:2 = "tf.IfRegion"(%arg0) ({ + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + // CHECK-NEXT: "tf.Yield"(%[[CONST]], %[[CONST]]) + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }, { + // CHECK: "tf.Yield"(%[[READ1]], %[[READ1]]) + %id = "tf.Identity"(%1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }) {is_stateless = false} : (tensor) -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[IF]]#1, %[[IF]]#0) + // CHECK-NEXT: tf_device.return %[[ADD]], %[[IF]]#1 + %4 = "tf.ReadVariableOp"(%3#0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %5 = "tf.AddV2"(%4, %3#1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + tf_device.return %5 : tensor<4xf32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor<4xf32>, tensor<4xf32>) + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor<4xf32> +} + +// Test that the pass can lift resources out of CaseRegion +// CHECK: func @cluster_with_caseregion(%[[ARG0:.*]]: tensor) -> tensor<4xf32> +func.func @cluster_with_caseregion(%arg0: tensor) -> tensor<4xf32> { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + // CHECK: %[[CASE:.*]]:2 = "tf.CaseRegion"(%[[ARG0]]) + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + %3:2 = "tf.CaseRegion"(%arg0) ({ + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + // CHECK-NEXT: "tf.Yield"(%[[CONST]], %[[CONST]]) + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }, { + // CHECK: "tf.Yield"(%[[READ1]], %[[READ1]]) + %id = "tf.Identity"(%1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }, { + // CHECK: %[[CONST1:.*]] = "tf.Const" + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[READ1]], %[[CONST1]]) + // CHECK: "tf.Yield"(%[[READ1]], %[[SUB]]) + %id = "tf.Identity"(%1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %constant = "tf.Const"() {value = dense<1.0> : tensor<4xf32>} : () -> tensor<4xf32> + %sub = "tf.Sub"(%read, %constant) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %sub) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }) {is_stateless = false} : (tensor) -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[CASE]]#1, %[[CASE]]#0) + // CHECK-NEXT: tf_device.return %[[ADD]], %[[CASE]]#1 + %4 = "tf.ReadVariableOp"(%3#0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %5 = "tf.AddV2"(%4, %3#1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + tf_device.return %5 : tensor<4xf32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor<4xf32>, tensor<4xf32>) + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor<4xf32> +} + +// ----- + +// Test that the pass can lift resources out of WhileRegion + +!tf_ref = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @cluster_with_whileregion +func.func @cluster_with_whileregion() -> () { + // CHECK: %[[COUNT:.*]] = "tf.Const"() <{value = dense<10> : tensor}> + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + // CHECK: %[[WHILE:.*]]:2 = "tf.WhileRegion"(%[[COUNT]], %[[READ]]) + %0 = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> !tf_ref + %pass_through = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> !tf_ref + %unused = "tf.VarHandleOp"() {container = "c", shared_name = "v3"} : () -> !tf_ref + "tf_device.cluster"() ({ + %2:4 = "tf.WhileRegion"(%0, %1, %pass_through, %unused) ({ + // CHECK: (%[[CARG0:.+]]: tensor, %[[CARG1:.+]]: tensor): + // CHECK: %[[CAST:.+]] = "tf.Cast"(%[[CARG1]]) + // CHECK: "tf.Less"(%[[CARG0]], %[[CAST]]) + // CHECK: "tf.Yield" + ^bb0(%carg0: tensor, %carg1: !tf_ref, %carg2: !tf_ref, %carg3: !tf_ref): + %read0 = "tf.ReadVariableOp"(%carg1) : (!tf_ref) -> tensor + %cast = "tf.Cast"(%read0) : (tensor) -> tensor + %cond = "tf.Less"(%carg0, %cast) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + // CHECK: (%[[BARG0:.+]]: tensor, %[[BARG1:.+]]: tensor): + // CHECK: %[[ADD0:.*]] = "tf.AddV2"(%[[BARG1]], %[[BARG1]]) + // CHECK-NEXT: %[[ADD1:.*]] = "tf.AddV2"(%[[ADD0]], %[[ADD0]]) + // CHECK-NEXT: %[[DELTA:.*]] = "tf.Const"() <{value = dense<-1> : tensor}> + // CHECK-NEXT: %[[ADD2:.*]] = "tf.AddV2"(%[[BARG0]], %[[DELTA]]) + // CHECK-NEXT: "tf.Yield"(%[[ADD2]], %[[ADD1]]) + ^bb1(%barg0: tensor, %barg1: !tf_ref, %barg2: !tf_ref, %barg3: !tf_ref): + %read0 = "tf.ReadVariableOp"(%barg1) : (!tf_ref) -> tensor + %add0 = "tf.AddV2"(%read0, %read0) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%barg1, %add0) : (!tf_ref, tensor) -> () + %read1 = "tf.ReadVariableOp"(%barg1) : (!tf_ref) -> tensor + %add1 = "tf.AddV2"(%read1, %read1) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%barg1, %add1) : (!tf_ref, tensor) -> () + %constant = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %add2 = "tf.AddV2"(%barg0, %constant) : (tensor, tensor) -> tensor + %id = "tf.Identity"(%barg3) : (!tf_ref) -> !tf_ref + "tf.Yield"(%add2, %barg1, %pass_through, %id) : (tensor, !tf_ref, !tf_ref, !tf_ref) -> () + }) {device = "", is_stateless = false} + : (tensor, !tf_ref, !tf_ref, !tf_ref) + -> (tensor, !tf_ref, !tf_ref, !tf_ref) + tf_device.return + }) {cluster_attr = "cluster_attr"} : () -> () + // CHECK: tf_device.return %[[WHILE]]#1 : tensor + // CHECK: {cluster_attr = "cluster_attr"} : () -> tensor + // CHECK: "tf.AssignVariableOp"(%[[VH]], %[[CLUSTER]]) + // CHECK: return + func.return +} + +// ----- + +// Test that the pass can lift out recursively (If with another if it its body) +// CHECK: func @cluster_with_if_within_if(%[[ARG0:.*]]: tensor, %[[ARG1:.*]]: tensor) -> tensor<4xf32> +func.func @cluster_with_if_within_if(%arg0: tensor, %arg1: tensor) -> tensor<4xf32> { + // CHECK: %[[VH0:.*]] = "tf.VarHandleOp"() + // CHECK: %[[VH1:.*]] = "tf.VarHandleOp"() + // CHECK: %[[READ0:.*]] = "tf.ReadVariableOp"(%[[VH0]]) + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[VH1]]) + // CHECK: %[[CLUSTER:.*]]:2 = "tf_device.cluster"() + // CHECK: %[[IF:.*]]:2 = "tf.IfRegion"(%[[ARG0]]) + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "v2"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + %3:2 = "tf.IfRegion"(%arg0) ({ + // CHECK-NEXT: %[[CONST:.*]] = "tf.Const"() + // CHECK-NEXT: "tf.Yield"(%[[CONST]], %[[CONST]]) + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }, { + // CHECK: %[[IF1:.*]] = "tf.IfRegion" + // CHECK: "tf.Yield"(%[[READ1]]) + // CHECK: "tf.Yield"(%[[READ0]]) + // CHECK: "tf.Yield"(%[[IF1]], %[[IF1]]) + %id = "tf.Identity"(%1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %read = "tf.IfRegion"(%arg1) ({ + %read_then = "tf.ReadVariableOp"(%id) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.Yield"(%read_then) : (tensor<4xf32>) -> () + }, { + %read_else = "tf.ReadVariableOp"(%0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + "tf.Yield"(%read_else) : (tensor<4xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"(%0, %read) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + }) {is_stateless = false} : (tensor) -> (tensor<*x!tf_type.resource>>, tensor<4xf32>) + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[IF]]#1, %[[IF]]#0) + // CHECK-NEXT: tf_device.return %[[ADD]], %[[IF]]#1 + %4 = "tf.ReadVariableOp"(%3#0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %5 = "tf.AddV2"(%4, %3#1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + tf_device.return %5 : tensor<4xf32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xf32> + // CHECK: {cluster_attr = "cluster_attr"} : () -> (tensor<4xf32>, tensor<4xf32>) + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]#1) + // CHECK: return %[[CLUSTER]]#0 + func.return %2 : tensor<4xf32> +} + +// ----- + +// IfRegion with store in just one branch + +// CHECK: func @if_region_with_store_in_then(%[[ARG0:.*]]: tensor) +func.func @if_region_with_store_in_then(%arg0: tensor) { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VH]]) + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + // CHECK: %[[IF:.*]] = "tf.IfRegion"(%[[ARG0]]) + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + "tf_device.cluster"() ({ + "tf.IfRegion"(%arg0) ({ + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<0.000000e+00> + // CHECK: "tf.Yield"(%[[CONST]]) + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"() : () -> () + }, { + // CHECK: "tf.Yield"(%[[READ]]) + "tf.Yield"() : () -> () + }) { is_stateless = true} : (tensor) -> () + tf_device.return + }) { cluster_attr = "cluster_attr" } : () -> () + // CHECK: tf_device.return %[[IF]] + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]) + func.return +} + +// ----- + +// IfRegion with store in both branches + +// CHECK: func @if_region_with_store_in_both(%[[ARG0:.*]]: tensor) +func.func @if_region_with_store_in_both(%arg0: tensor) { + // CHECK: %[[VH:.*]] = "tf.VarHandleOp"() + // CHECK: %[[CLUSTER:.*]] = "tf_device.cluster"() + // CHECK: %[[IF:.*]] = "tf.IfRegion"(%[[ARG0]]) + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + "tf_device.cluster"() ({ + "tf.IfRegion"(%arg0) ({ + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<0.000000e+00> + // CHECK: "tf.Yield"(%[[CONST]]) + %constant = "tf.Const"() {value = dense<0.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"() : () -> () + }, { + // CHECK: %[[CONST:.*]] = "tf.Const"() <{value = dense<1.000000e+00> + // CHECK: "tf.Yield"(%[[CONST]]) + %constant = "tf.Const"() {value = dense<1.0> : tensor<4xf32>} : () -> tensor<4xf32> + "tf.AssignVariableOp"(%0, %constant) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + "tf.Yield"() : () -> () + }) { is_stateless = true} : (tensor) -> () + tf_device.return + }) { cluster_attr = "cluster_attr" } : () -> () + // CHECK: tf_device.return %[[IF]] + // CHECK: "tf.AssignVariableOp"(%[[VH0]], %[[CLUSTER]]) + func.return +} + + +// Make sure unsupported resources are handled correctly. If a resource is used +// in an unsupported op, resource op lifting should skip lifting that resource. +// So for the below test, the IR should stay unchanged. +// CHECK-LABEL: func @test_unsupported_resource_op +func.func @test_unsupported_resource_op() -> tensor<*xi32> { + // CHECK: "tf.VarHandleOp" + // CHECK: "tf_device.cluster"() ({ + // CHECK: "tf.ReadVariableOp" + // CHECK: "tf.SomeResourceOperation" + // CHECK: "tf.SomeComputation" + // CHECK: tf_device.return + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK: return + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + "tf.SomeResourceOperation"(%0) : (tensor<*x!tf_type.resource>>) -> () + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + func.return %1 : tensor<*xi32> +} + +// Test unsupported use of resource ops in functional control flow. In the test +// below, arg0 has an unsupported use whereas arg1 does not. So we expect arg0 +// to not be lifted and arg1 to be lifted. +// CHECK-LABEL: func @test_unsupported_resource_op_in_if +func.func @test_unsupported_resource_op_in_if(%arg0: tensor) -> tensor<*xi32> { + // CHECK: [[VH0:%.*]] = "tf.VarHandleOp"() <{container = "c", shared_name = "v"}> + // CHECK: [[VH1:%.*]] = "tf.VarHandleOp"() <{container = "d", shared_name = "w"}> + // CHECK-NOT: "tf.ReadVariableOp"([[VH0]]) + // CHECK: [[READ1:%.*]] = "tf.ReadVariableOp"([[VH1]]) + // CHECK-NOT: "tf.ReadVariableOp"([[VH0]]) + // CHECK: "tf_device.cluster"() ({ + // CHECK: "tf.If"({{%.*}}, [[VH0]], [[READ1]]) + // CHECK-SAME: else_branch = @else_fn, is_stateless = true, then_branch = @then_fn + // CHECK: tf_device.return + // CHECK: return + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "d", shared_name = "w"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf_device.cluster"() ({ + %3 = "tf.If"(%arg0, %0, %1) + { else_branch = @else_fn, then_branch = @then_fn, is_stateless = true} + : (tensor, tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) -> tensor<*xi32> + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + func.return %2 : tensor<*xi32> +} + +// CHECK-LABEL: func @else_fn +// CHECK-SAME: (%{{.*}}: tensor<*x!tf_type.resource>, %{{.*}}: tensor<*xi32>) +func.func @else_fn(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>) -> tensor<*xi32> { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %2 = "tf.Add"(%0, %1) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + func.return %2 : tensor<*xi32> +} + +// CHECK-LABEL: func @then_fn +// CHECK-SAME: (%{{.*}}: tensor<*x!tf_type.resource>, %{{.*}}: tensor<*xi32>) +func.func @then_fn(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<*x!tf_type.resource>) -> tensor<*xi32> { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %2 = "tf.Add"(%0, %1) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + "tf.UnsupportedResourceOp"(%arg0) : (tensor<*x!tf_type.resource>) -> () + func.return %2 : tensor<*xi32> +} + +// Test type refinement. If the resource has a single subtype, check that that +// type gets used when hoisting the read. None of the result types will change. +// CHECK-LABEL: func @type_refinement_use_subtype +func.func @type_refinement_use_subtype() -> tensor<*xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + // CHECK-SAME: -> tensor<4xi32> + // CHECK: %[[CLUSTER_RES:[0-9]*]]:2 = "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) : (tensor<4xi32>) -> tensor<*xi32> + // CHECK: tf_device.return %[[COMPUTE_RES]], %[[COMPUTE_RES]] + // CHECK-SAME: tensor<*xi32>, tensor<*xi32> + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> (tensor<*xi32>, tensor<*xi32>) + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[CLUSTER_RES]]#1) + + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%0, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + // CHECK-SAME: tensor<*xi32> + func.return %1 : tensor<*xi32> +} + +// If multiple types are used across reads and writes, check that the read uses +// the most refined type. The first ReadVariable should refine the type from +// *xi32 to ?xi32 and the assign should refine it further to 4xi32. +// CHECK-LABEL: func @type_refinement_use_refined_type +func.func @type_refinement_use_refined_type() -> tensor<4xi32> { + + // CHECK: %[[RES_HANDLE:[0-9]*]] = "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + + // CHECK: %[[RES_READ_VAL:[0-9]*]] = "tf.ReadVariableOp"(%[[RES_HANDLE]]) + // CHECK-SAME: -> tensor<4xi32> + // CHECK: %[[CLUSTER_RES:[0-9]*]]:2 = "tf_device.cluster" + // CHECK: %[[COMPUTE_RES:[0-9]*]] = "tf.SomeComputation"(%[[RES_READ_VAL]]) : (tensor<4xi32>) -> tensor<4xi32> + // CHECK: tf_device.return %[[COMPUTE_RES]], %[[COMPUTE_RES]] + // CHECK-SAME: tensor<4xi32>, tensor<4xi32> + // CHECK: {cluster_attr = "cluster_attr"} + // CHECK-SAME: () -> (tensor<4xi32>, tensor<4xi32>) + // CHECK: "tf.AssignVariableOp"(%[[RES_HANDLE]], %[[CLUSTER_RES]]#1) + + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {dtype = i32} : (tensor<*x!tf_type.resource>>) -> tensor + %3 = "tf.SomeComputation"(%2) : (tensor) -> (tensor<4xi32>) + "tf.AssignVariableOp"(%0, %3) {dtype = i32} : (tensor<*x!tf_type.resource>>, tensor<4xi32>) -> () + tf_device.return %3 : tensor<4xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<4xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + // CHECK-SAME: tensor<4xi32> + func.return %1 : tensor<4xi32> +} + +// ----- + +!tf_res = tensor<*x!tf_type.resource>> + +// Test all tf.VarIsInitializedOp's are set to true. +// CHECK-LABEL: func @tpu_computation +func.func @tpu_computation(%arg0: !tf_res, %arg1: tensor, %arg2: tensor) { + %0 = "tf_device.cluster"() ({ + %1 = "tf.Case"(%arg2, %arg0) {branches = [@case_branch], is_stateless = false} : (tensor, !tf_res) -> tensor + + // CHECK: "tf.CaseRegion" + %2 = "tf.CaseRegion"(%arg2) ({ + // CHECK-NEXT: [[CASE_REGION_BRANCH:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %3 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: "tf.Yield"([[CASE_REGION_BRANCH]]) + "tf.Yield"(%3) : (tensor) -> () + }) {is_stateless = false} : (tensor) -> tensor + + %4 = "tf.If"(%arg1, %arg0) {then_branch = @if_then, else_branch = @if_else, is_stateless = false} : (tensor, !tf_res) -> tensor + + // CHECK: "tf.IfRegion" + %5 = "tf.IfRegion"(%arg1) ({ + // CHECK-NEXT: [[IF_REGION_THEN:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %6 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: "tf.Yield"([[IF_REGION_THEN]]) + "tf.Yield"(%6) : (tensor) -> () + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: [[IF_REGION_ELSE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %7 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: "tf.Yield"([[IF_REGION_ELSE]]) + "tf.Yield"(%7) : (tensor) -> () + }) {is_stateless = false} : (tensor) -> tensor + + %8:2 = "tf.While"(%arg0, %arg1) {body = @while_body, cond = @while_cond, is_stateless = false} : (!tf_res, tensor) -> (!tf_res, tensor) + + // CHECK: "tf.WhileRegion" + %9 = "tf.WhileRegion"(%arg1) ({ + // CHECK-NEXT: ^{{.+}}({{.+}}: tensor): + ^cond(%carg0: tensor): + // CHECK-NEXT: [[WHILE_REGION_COND:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %10 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: "tf.Yield"([[WHILE_REGION_COND]]) + "tf.Yield"(%10) : (tensor) -> () + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: ^{{.+}}({{.+}}: tensor): + ^body(%barg0: tensor): + // CHECK-NEXT: [[WHILE_REGION_BODY:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %11 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: "tf.Yield"([[WHILE_REGION_BODY]]) + "tf.Yield"(%11) : (tensor) -> () + }) {is_stateless = false} : (tensor) -> tensor + + %12 = "tf.StatefulPartitionedCall"(%arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} : (!tf_res) -> tensor + + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %13 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + + // CHECK: tf_device.return [[TRUE]] : + tf_device.return %13 : tensor + }) : () -> tensor + func.return +} + +// CHECK-LABEL: func @case_branch +func.func @case_branch(%arg0: !tf_res) -> tensor { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %0 : tensor +} + +// CHECK-LABEL: func @if_then +func.func @if_then(%arg0: !tf_res) -> tensor { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %0 : tensor +} + +// CHECK-LABEL: func @if_else +func.func @if_else(%arg0: !tf_res) -> tensor { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %0 : tensor +} + +// CHECK-LABEL: func @while_cond +// CHECK-SAME: ({{.+}}: tensor) +func.func @while_cond(%arg0: !tf_res, %arg1: tensor) -> tensor { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %0 : tensor +} + +// CHECK-LABEL: func @while_body +// CHECK-SAME: ({{.+}}: tensor) +func.func @while_body(%arg0: !tf_res, %arg1: tensor) -> (!tf_res, tensor) { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %arg0, %0 : !tf_res, tensor +} + +// CHECK-LABEL: func @callee +func.func @callee(%arg0: !tf_res) -> tensor { + // CHECK: [[TRUE:%.+]] = "tf.Const" + // CHECK-SAME: value = dense : tensor + %0 = "tf.VarIsInitializedOp"(%arg0) : (!tf_res) -> tensor + // CHECK-NEXT: return [[TRUE]] : + func.return %0 : tensor +} + +// ----- + +// Tests passthrough tf.Cast ops are removed. + +!tf_res = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @tpu_computation +func.func @tpu_computation(%arg0: !tf_res) { + "tf_device.cluster"() ({ + %0 = "tf.While"(%arg0) {body = @while_body, cond = @while_cond, is_stateless = false} : (!tf_res) -> !tf_res + %1 = "tf.WhileRegion"(%arg0) ({ + ^cond(%carg0: !tf_res): + %2 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + ^body(%barg0: !tf_res): + // CHECK-NOT: tf.Cast + %2 = "tf.Cast"(%barg0) : (!tf_res) -> !tf_res + "tf.Yield"(%2) : (!tf_res) -> () + }) {is_stateless = false} : (!tf_res) -> !tf_res + tf_device.return + }) {} : () -> () + func.return +} + +func.func @while_cond(%arg0: !tf_res) -> tensor { + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @while_body +func.func @while_body(%arg0: !tf_res) -> !tf_res { + // CHECK-NOT: tf.Cast + %0 = "tf.Cast"(%arg0) : (!tf_res) -> !tf_res + func.return %0 : !tf_res +} + +// ----- + +// Tests passthrough tf.Cast ops are removed. + +!tf_res_static = tensor>> +!tf_res_dynamic = tensor<*x!tf_type.resource>> + +// CHECK-LABEL: func @tpu_computation +func.func @tpu_computation(%arg0: !tf_res_static) { + "tf_device.cluster"() ({ + %0 = "tf.While"(%arg0) {body = @while_body, cond = @while_cond, is_stateless = false} : (!tf_res_static) -> !tf_res_dynamic + %1 = "tf.WhileRegion"(%arg0) ({ + ^cond(%carg0: !tf_res_static): + %2 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + ^body(%barg0: !tf_res_static): + // CHECK-NOT: tf.Cast + %2 = "tf.Cast"(%barg0) : (!tf_res_static) -> !tf_res_dynamic + "tf.Yield"(%2) : (!tf_res_dynamic) -> () + }) {is_stateless = false} : (!tf_res_static) -> !tf_res_dynamic + tf_device.return + }) {} : () -> () + func.return +} + +func.func @while_cond(%arg0: !tf_res_static) -> tensor { + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @while_body +func.func @while_body(%arg0: !tf_res_static) -> !tf_res_dynamic { + // CHECK-NOT: tf.Cast + %0 = "tf.Cast"(%arg0) : (!tf_res_static) -> !tf_res_dynamic + func.return %0 : !tf_res_dynamic +} + +// ----- + +// Tests that a resource ops device are removed while hoisting. + +// CHECK-LABEL: func @same_resource_load_and_store +// CHECK-NOT: device = +// CHECK: return +func.func @same_resource_load_and_store() -> tensor<*xi32> { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf_device.cluster"() ({ + %2 = "tf.ReadVariableOp"(%0) {device = "/device:TPU_REPLICATED_CORE:0"} : (tensor<*x!tf_type.resource>>) -> tensor<*xi32> + %3 = "tf.SomeComputation"(%2) : (tensor<*xi32>) -> (tensor<*xi32>) + "tf.AssignVariableOp"(%0, %3) {device = "/device:TPU_REPLICATED_CORE:0"} : (tensor<*x!tf_type.resource>>, tensor<*xi32>) -> () + tf_device.return %3 : tensor<*xi32> + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + + // CHECK: return %[[CLUSTER_RES]]#0 + func.return %1 : tensor<*xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/rewrite_tpu_embedding_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/rewrite_tpu_embedding_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b2da38f3d3d8841963ec099e041f403260f059c6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/rewrite_tpu_embedding_ops.mlir @@ -0,0 +1,75 @@ +// RUN: tf-opt -tf-rewrite-tpu-embedding-ops %s | FileCheck %s + +// CHECK-LABEL: func @recv_tpu_embedding_activations +func.func @recv_tpu_embedding_activations() -> (tensor<512x256xf32>) { + // CHECK: %[[DATA:.*]] = "tf.XlaRecvTPUEmbeddingDeduplicationData"() <{config = {{.*}}}> : () -> tensor + // CHECK: %[[RESULT:.*]] = "tf.XlaRecvTPUEmbeddingActivations"(%[[DATA]]) <{config = {{.*}}}> : (tensor) -> tensor<512x256xf32> + // CHECK: return %[[RESULT]] + // CHECK-NOT: tf.RecvTPUEmbeddingActivations + // CHECK-NOT: tf.SendTPUEmbeddingGradients + + %0 = "tf.RecvTPUEmbeddingActivations"() {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02"} : () -> tensor<512x256xf32> + func.return %0 : tensor<512x256xf32> +} + +// CHECK-LABEL: func @send_tpu_embedding_gradients +func.func @send_tpu_embedding_gradients(%arg0: tensor<512x256xf32>) -> () { + // CHECK: %[[DATA:.*]] = "tf.XlaRecvTPUEmbeddingDeduplicationData"() <{config = {{.*}}}> : () -> tensor + // CHECK: "tf.XlaSendTPUEmbeddingGradients"(%arg0, %[[DATA]]) <{config = {{.*}}, operandSegmentSizes = array}> : (tensor<512x256xf32>, tensor) -> () + // CHECK-NOT: tf.SendTPUEmbeddingGradients + // CHECK-NOT: tf.RecvTPUEmbeddingActivations + + "tf.SendTPUEmbeddingGradients"(%arg0) {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02", operandSegmentSizes = array} : (tensor<512x256xf32>) -> () + func.return +} + +// CHECK-LABEL: func @recv_send_ops +func.func @recv_send_ops() -> () { + // CHECK: %[[DATA:.*]] = "tf.XlaRecvTPUEmbeddingDeduplicationData"() + // CHECK: %[[ACTIVATIONS:.*]] = "tf.XlaRecvTPUEmbeddingActivations"(%[[DATA]]) + // CHECK: "tf.XlaSendTPUEmbeddingGradients"(%[[ACTIVATIONS]], %[[DATA]]) + + %0 = "tf.RecvTPUEmbeddingActivations"() {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02"} : () -> tensor<512x256xf32> + "tf.SendTPUEmbeddingGradients"(%0) {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02", operandSegmentSizes = array} : (tensor<512x256xf32>) -> () + func.return +} + +// CHECK-LABEL: func @no_embedding_ops +func.func @no_embedding_ops(%arg0: tensor<2x2xf32>) -> (tensor<2x2xf32>) { + // CHECK: tf.Add + %0 = "tf.Add"(%arg0, %arg0) : (tensor<2x2xf32>, tensor<2x2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// CHECK-LABEL: func @nested_embedding_op +func.func @nested_embedding_op(%arg0: tensor, %arg1: tensor<512x256xf32>) -> (tensor<512x256xf32>) { + %1 = "tf.IfRegion"(%arg0) ({ + // CHECK: "tf.XlaRecvTPUEmbeddingDeduplicationData" + // CHECK: "tf.XlaRecvTPUEmbeddingActivations" + // CHECK-NOT: tf.RecvTPUEmbeddingActivations + %0 = "tf.RecvTPUEmbeddingActivations"() {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02"} : () -> tensor<512x256xf32> + "tf.Yield"(%0) : (tensor<512x256xf32>) -> () + }, { + "tf.Yield"(%arg1) : (tensor<512x256xf32>) -> () + }) { is_stateless = true}: (tensor) -> tensor<512x256xf32> + func.return %1 : tensor<512x256xf32> +} + +// CHECK-LABEL: func @doubly_nested_embedding_op +func.func @doubly_nested_embedding_op(%arg0: tensor, %arg1: tensor, %arg2: tensor<512x256xf32>) -> (tensor<512x256xf32>) { + %2 = "tf.IfRegion"(%arg0) ({ + %1 = "tf.IfRegion"(%arg1) ({ + // CHECK: "tf.XlaRecvTPUEmbeddingDeduplicationData" + // CHECK: "tf.XlaRecvTPUEmbeddingActivations" + // CHECK-NOT: tf.RecvTPUEmbeddingActivations + %0 = "tf.RecvTPUEmbeddingActivations"() {config = "\0A%\0A\0Dwatches_table\10\F5\03\18\80\02 \01*\0C\1A\00j\05\0D\00\00\80?\88\01\01\10\02\18\80\04 \01(\02"} : () -> tensor<512x256xf32> + "tf.Yield"(%0) : (tensor<512x256xf32>) -> () + }, { + "tf.Yield"(%arg2) : (tensor<512x256xf32>) -> () + }) { is_stateless = true}: (tensor) -> tensor<512x256xf32> + "tf.Yield"(%1) : (tensor<512x256xf32>) -> () + }, { + "tf.Yield"(%arg2) : (tensor<512x256xf32>) -> () + }) { is_stateless = true}: (tensor) -> tensor<512x256xf32> + func.return %2 : tensor<512x256xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3cd9cf9d824bfb8d4724dc70134317f2dc889ebd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference.mlir @@ -0,0 +1,2236 @@ +// RUN: tf-opt %s -tf-shape-inference -verify-diagnostics | FileCheck %s + +!tf_variant = tensor>> + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 130 : i32}} { + // CHECK-LABEL: func @main(%arg0: tensor<1xi32>, %arg1: tensor<1xi32>) -> tensor<1xi32> + func.func @main(%arg0: tensor<1xi32>, %arg1: tensor<1xi32>) -> tensor<*xi32> { + // CHECK: %[[RESULT:.*]] = "tf.AddV2" + // CHECK-SAME: (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK: return %[[RESULT]] : tensor<1xi32> + %0 = "tf.Cast"(%arg0) : (tensor<1xi32>) -> tensor<*xi32> + %1 = "tf.Cast"(%arg1) : (tensor<1xi32>) -> tensor<*xi32> + %2 = "tf.AddV2"(%0, %1) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + func.return %2 : tensor<*xi32> + } + + // CHECK-LABEL: func @simple_chain + func.func @simple_chain(%arg0: tensor<1xf32>) -> tensor<*xf32> { + // CHECK: %[[MUL:.*]] = "tf.Mul"{{.*}} (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // CHECK: %[[ADD:.*]] = "tf.Add"(%[[MUL]], %[[MUL]]) : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // CHECK: return %[[ADD]] : tensor<1xf32> + %0 = "tf.Mul"(%arg0, %arg0) : (tensor<1xf32>, tensor<1xf32>) -> tensor<*xf32> + %1 = "tf.Add"(%0, %0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> + } + + // CHECK-LABEL: func @simple_chain_with_broadcast + func.func @simple_chain_with_broadcast(%arg0: tensor<1xf32>, %arg1: tensor<10xf32>) -> tensor { + // CHECK: %[[MUL:.*]] = "tf.Mul"{{.*}} (tensor<1xf32>, tensor<10xf32>) -> tensor<10xf32> + // CHECK: %[[ADD:.*]] = "tf.Add"(%[[MUL]], %[[MUL]]) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32> + // CHECK: %[[CAST:.*]] = "tf.Cast"(%[[ADD]]) {{.*}} : (tensor<10xf32>) -> tensor<*xf32> + // CHECK: %[[TENSOR_CAST:.*]] = tensor.cast %[[CAST]] : tensor<*xf32> to tensor + // CHECK: return %[[TENSOR_CAST]] : tensor + %0 = "tf.Mul"(%arg0, %arg1) : (tensor<1xf32>, tensor<10xf32>) -> tensor<*xf32> + %1 = "tf.Add"(%0, %0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %2 = tensor.cast %1 : tensor<*xf32> to tensor + func.return %2 : tensor + } + + // CHECK-LABEL: func @unknown_op + func.func @unknown_op(%arg0: tensor<1xf32>) -> tensor<*xf32> { + // CHECK: %[[MUL:.*]] = "tf.Mul"{{.*}} (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // CHECK: %[[UNKNOWN:.*]] = "tf.Unknown"(%[[MUL]], %[[MUL]]) : (tensor<1xf32>, tensor<1xf32>) -> tensor<*xf32> + // CHECK: return %[[UNKNOWN]] : tensor<*xf32> + %0 = "tf.Mul"(%arg0, %arg0) : (tensor<1xf32>, tensor<1xf32>) -> tensor<*xf32> + %1 = "tf.Unknown"(%0, %0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> + } + + // CHECK-LABEL: func @multiple_blocks_one_return(%arg0: tensor) -> tensor + func.func @multiple_blocks_one_return(%arg0: tensor) -> tensor<*xf32> { + cf.br ^bb1 + ^bb1: + // CHECK: %[[IDENTITY:.*]] = "tf.Identity"(%arg0) : (tensor) -> tensor + // CHECK: return %[[IDENTITY]] : tensor + %ret = "tf.Identity"(%arg0) : (tensor) -> tensor<*xf32> + func.return %ret : tensor<*xf32> + } + + + // Tests the case where an inference opportunity relies on folding. + + // CHECK-LABEL: func @simple_folding + func.func @simple_folding(%arg0: tensor<1x1x1x1xi32>, %arg1: tensor<1x1x1x1xf32>) -> tensor { + // CHECK: %[[SHAPE:.*]] = "tf.Shape" + // CHECK: %[[CONV:.*]] = "tf.Conv2DBackpropInput"(%[[SHAPE]] + // CHECK-SAME: (tensor<4xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>) -> tensor<1x1x1x1xf32> + // CHECK: return %[[CONV]] : tensor<1x1x1x1xf32> + %0 = "tf.Shape"(%arg0) : (tensor<1x1x1x1xi32>) -> tensor<4xi32> + %1 = "tf.Conv2DBackpropInput"(%0, %arg1, %arg1) { + padding = "VALID", strides = [1, 1, 1, 1] + } : (tensor<4xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>) -> tensor + func.return %1 : tensor + } + + // Tests where tf.Const's value needs to be refined. + + func.func @const_refine() -> tensor<*xi32> { + %0 = "tf.Const"() {value = dense<[3, 2]> : tensor<2xi32>} : () -> tensor<*xi32> + // CHECK: "tf.Const" + // CHECK-SAME: -> tensor<2xi32> + func.return %0 : tensor<*xi32> + } + + // Tests the case where an op's shape function returns non-fully-defined shapes. + + // CHECK-LABEL: func @op_non_fully_defined_shape_fn + func.func @op_non_fully_defined_shape_fn(%arg0: tensor<*xi32>, %arg1: tensor<0xi32>) -> tensor { + // CHECK: tf.BroadcastGradientArgs + // CHECK-SAME: (tensor<*xi32>, tensor<0xi32>) -> (tensor, tensor) + %2:2 = "tf.BroadcastGradientArgs"(%arg0, %arg1) {T = "tfdtype$DT_INT32", name = "BroadcastGradientArgs"} : (tensor<*xi32>, tensor<0xi32>) -> (tensor, tensor) + func.return %2#0 : tensor + } + + // CHECK-LABEL: func @shape_from_const_input + func.func @shape_from_const_input(%arg0: tensor<3x3x32x64xf32>, %arg1: tensor<200x24x24x64xf32>) -> tensor { + %0 = "tf.Const"() {value = dense<[200, 26, 26, 32]> : tensor<4xi32>} : () -> tensor<4xi32> + // CHECK: tf.Conv2DBackpropInput + // CHECK-SAME: (tensor<4xi32>, tensor<3x3x32x64xf32>, tensor<200x24x24x64xf32>) -> tensor<200x26x26x32xf32> + %1 = "tf.Conv2DBackpropInput"(%0, %arg0, %arg1) {data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true} : (tensor<4xi32>, tensor<3x3x32x64xf32>, tensor<200x24x24x64xf32>) -> tensor + func.return %1 : tensor + } + + // CHECK-LABEL: func @shape_from_if_to_branch_functions_to_results + // CHECK-SAME: (%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @shape_from_if_to_branch_functions_to_results(%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<*xf32> { + %0 = "tf.If"(%arg0, %arg1) {Tcond = i1, Tin = ["tfdtype$DT_FLOAT"], Tout = ["tfdtype$DT_FLOAT"], else_branch = @if_else_branch, is_stateless = true, name = "if", then_branch = @if_then_branch} : (tensor, tensor<1x2x3xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @if_then_branch + // CHECK-SAME: (%arg0: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @if_then_branch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: return + // CHECK-SAME: tensor<1x2x3xf32> + func.return %arg0 : tensor<*xf32> + } + + // CHECK-LABEL: func @if_else_branch + // CHECK-SAME: (%arg0: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @if_else_branch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Identity"(%arg0) : (tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + %0 = "tf.Identity"(%arg0) : (tensor<*xf32>) -> (tensor<*xf32>) + // CHECK: return + // CHECK-SAME: tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + + // Verify shape propagation from function arg -> if region body -> if region output -> function return type + // CHECK-LABEL: shape_from_if_to_region_bodies_to_output + // CHECK-SAME: -> tensor<1x2x3xf32> + func.func @shape_from_if_to_region_bodies_to_output(%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<*xf32> { + %unshaped = "tf.Cast"(%arg1) : (tensor<1x2x3xf32>) -> tensor<*xf32> + // CHECK: <{is_stateless = true}> + %0 = "tf.IfRegion"(%arg0) <{is_stateless = true}> ({ + // CHECK: "tf.Add"{{.+}}(tensor<1x2x3xf32>, tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + // CHECK: "tf.Yield"{{.+}}(tensor<1x2x3xf32>) -> () + %1 = "tf.Add"(%unshaped, %unshaped) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + // CHECK: "tf.Sub"{{.+}}(tensor<1x2x3xf32>, tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + // CHECK: "tf.Yield"{{.+}}(tensor<1x2x3xf32>) -> () + %2 = "tf.Sub"(%unshaped, %unshaped) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + // CHECK: (tensor) -> tensor<1x2x3xf32> + }) : (tensor) -> tensor<*xf32> + // CHECK: return {{.*}} : tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @shape_from_case_to_branch_functions_to_results + // CHECK-SAME: (%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @shape_from_case_to_branch_functions_to_results(%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<*xf32> { + %0 = "tf.Case"(%arg0, %arg1) {branches = [@case_branch0, @case_branch1], is_stateless = true} : (tensor, tensor<1x2x3xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @case_branch0 + // CHECK-SAME: (%arg0: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @case_branch0(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: return + // CHECK-SAME: tensor<1x2x3xf32> + func.return %arg0 : tensor<*xf32> + } + + // CHECK-LABEL: func @case_branch1 + // CHECK-SAME: (%arg0: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @case_branch1(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Identity"(%arg0) : (tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + %0 = "tf.Identity"(%arg0) : (tensor<*xf32>) -> (tensor<*xf32>) + // CHECK: return + // CHECK-SAME: tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: shape_from_case_to_region_bodies_to_output + // CHECK-SAME: -> tensor<1x2x3xf32> + func.func @shape_from_case_to_region_bodies_to_output(%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<*xf32> { + %unshaped = "tf.Cast"(%arg1) : (tensor<1x2x3xf32>) -> tensor<*xf32> + // CHECK: <{is_stateless = true}> + %0 = "tf.CaseRegion"(%arg0) <{is_stateless = true}> ({ + // CHECK: "tf.Add"{{.+}}(tensor<1x2x3xf32>, tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + // CHECK: "tf.Yield"{{.+}}(tensor<1x2x3xf32>) -> () + %1 = "tf.Add"(%unshaped, %unshaped) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + // CHECK: "tf.Sub"{{.+}}(tensor<1x2x3xf32>, tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + // CHECK: "tf.Yield"{{.+}}(tensor<1x2x3xf32>) -> () + %2 = "tf.Sub"(%unshaped, %unshaped) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%2) : (tensor<*xf32>) -> () + // CHECK: (tensor) -> tensor<1x2x3xf32> + }) : (tensor) -> tensor<*xf32> + // CHECK: return {{.*}} : tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @shape_from_while_to_cond_body_functions + func.func @shape_from_while_to_cond_body_functions(%arg0: tensor<4xf32>, %arg1: tensor>>, %arg2: tensor>>) -> tensor<4xf32> { + // CHECK: "tf.While" + // CHECK-SAME: (tensor<4xf32>, tensor>>, tensor>>) -> (tensor<4xf32>, tensor>>, tensor>>) + %0:3 = "tf.While"(%arg0, %arg1, %arg2) {cond = @while_cond_func, body = @while_body_func, is_stateless = true} : (tensor<4xf32>, tensor>>, tensor>>) -> (tensor<4xf32>, tensor<*x!tf_type.resource>, tensor>>) + func.return %0#0 : tensor<4xf32> + } + + // CHECK-LABEL: func @while_cond_func + // CHECK-SAME: (%arg0: tensor<4xf32>, %arg1: tensor>>, %arg2: tensor>>) -> tensor + func.func @while_cond_func(%arg0: tensor<*xf32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor>>) -> tensor { + %0 = "tf.Const"() {value = dense<[1.000000e-04,2.000000e-04,3.000000e-04,4.000000e-04]> : tensor<4xf32>} : () -> tensor<4xf32> + %1 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Equal + // CHECK-SAME: (tensor<4xf32>, tensor<4xf32>) -> tensor<*xi1> + // TODO(ycao): Investigate why result type of tf.Equal is not inferred. + %2 = "tf.Equal"(%0, %arg0) : (tensor<4xf32>, tensor<*xf32>) -> tensor<*xi1> + %3 = "tf.Any"(%2, %1) : (tensor<*xi1>, tensor) -> (tensor) + func.return %3 : tensor + } + + // CHECK-LABEL: func @while_body_func + func.func @while_body_func(%arg0: tensor<*xf32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor>>) -> (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor>>) { + %0 = "tf.Const"() {value = dense<1.000000e-04> : tensor} : () -> tensor + // CHECK: tf.AddV2 + // CHECK-SAME: (tensor<4xf32>, tensor) -> tensor<4xf32> + %1 = "tf.AddV2"(%arg0, %0) : (tensor<*xf32>, tensor) -> tensor<*xf32> + // CHECK: "tf.Identity" + // CHECK-SAME: (tensor>>) -> tensor>> + %2 = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>) -> tensor<*x!tf_type.resource> + // CHECK: "tf.TPUReplicatedInput" + // CHECK-SAME: (tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%2) : (tensor<*x!tf_type.resource>) -> tensor<*x!tf_type.resource> + // CHECK: "tf.ReadVariableOp" + // CHECK-SAME: (tensor>>) -> tensor<4xf32> + %read = "tf.ReadVariableOp"(%ri) : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + // CHECK: "tf.ReadVariableOp" + // CHECK-SAME: (tensor>>) -> tensor<*xf32> + %read1 = "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor<*xf32> + // CHECK: return + // CHECK-SAME: tensor<4xf32> + // CHECK-SAME: tensor>> + func.return %1, %arg1, %arg2 : tensor<*xf32>, tensor<*x!tf_type.resource>, tensor>> + } + + // Verify shape propagation from function arg -> while region cond/body -> while region output -> function return type + // CHECK-LABEL: func @shape_from_while_operands_to_cond_body_to_while_results + // CHECK-SAME: -> tensor<1x2x3xf32> + func.func @shape_from_while_operands_to_cond_body_to_while_results(%arg0: tensor, %arg1: tensor<1x2x3xf32>) -> tensor<*xf32> { + %unshaped = "tf.Cast"(%arg1) : (tensor<1x2x3xf32>) -> tensor<*xf32> + // CHECK: "tf.WhileRegion" + // CHECK: <{is_stateless = true}> + %0:2 = "tf.WhileRegion"(%arg0, %unshaped) <{is_stateless = true}> ({ + // CHECK: {{.*}}({{.+}}: tensor, {{.+}}: tensor<1x2x3xf32>): + ^bb0(%carg0: tensor, %carg1: tensor<*xf32>): + %limit = arith.constant dense<5> : tensor + %cond = "tf.NotEqual"(%carg0, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + // CHECK: {{.*}}({{.+}}: tensor, {{.+}}: tensor<1x2x3xf32>): + ^bb0(%barg0: tensor, %barg1: tensor<*xf32>): + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg0, %one) : (tensor, tensor) -> tensor + // CHECK: "tf.Neg"({{.+}}) : (tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + %neg = "tf.Neg"(%barg1) : (tensor<*xf32>) -> tensor<*xf32> + // CHECK: "tf.Yield"{{.+}}, {{.+}}) : (tensor, tensor<1x2x3xf32>) -> () + "tf.Yield"(%sub, %neg) : (tensor, tensor<*xf32>) -> () + // CHECK: (tensor, tensor<1x2x3xf32>) -> (tensor, tensor<1x2x3xf32>) + }) : (tensor, tensor<*xf32>) -> (tensor, tensor<*xf32>) + // CHECK: return {{.+}}#1 : tensor<1x2x3xf32> + func.return %0#1 : tensor<*xf32> + } + + // CHECK-LABEL: func @shape_from_case_to_branch_functions( + // CHECK-SAME: %[[ARG_0:.*]]: tensor, + // CHECK-SAME: %[[ARG_1:.*]]: tensor>> + func.func @shape_from_case_to_branch_functions(%arg0: tensor, %arg1: tensor>>) -> tensor<1x2x3xf32> { + // CHECK: %[[CASE:.*]] = "tf.Case"(%[[ARG_0]], %[[ARG_1]]) + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch_0, @branch_1], is_stateless = false} : (tensor, tensor>>) -> tensor<1x2x3xf32> + // CHECK: return %[[CASE]] : tensor<1x2x3xf32> + func.return %0 : tensor<1x2x3xf32> + } + // CHECK-LABEL: func @branch_0 + // CHECK-SAME: %[[ARG_0:.*]]: tensor>>) -> tensor<1x2x3xf32> + func.func @branch_0(%arg0: tensor) -> tensor<*xf32> { + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) : (tensor>>) -> tensor<1x2x3xf32> + %0 = "tf.ReadVariableOp"(%arg0) : (tensor) -> (tensor<*xf32>) + // CHECK: return %[[READ]] : tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + // CHECK-LABEL: func @branch_1 + // CHECK-SAME: %[[ARG_0:.*]]: tensor>>) -> tensor<1x2x3xf32> + func.func @branch_1(%arg0: tensor) -> tensor<*xf32> { + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) : (tensor>>) -> tensor<1x2x3xf32> + %0 = "tf.ReadVariableOp"(%arg0) : (tensor) -> (tensor<*xf32>) + // CHECK: return %[[READ]] : tensor<1x2x3xf32> + func.return %0 : tensor<*xf32> + } + + func.func @partitioned_call(%arg0: tensor) -> tensor<*xi32> { + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @partitioned_call_func} : (tensor) -> (tensor<*xi32>) + func.return %0 : tensor<*xi32> + } + + // CHECK-LABEL: func @partitioned_call_func + // CHECK-SAME: (%arg0: tensor) -> tensor + func.func @partitioned_call_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + // CHECK: return + // CHECK-SAME: tensor + func.return %arg0 : tensor<*xi32> + } + + // CHECK-LABEL: func @invalid_function_reused_by_control_flows + func.func @invalid_function_reused_by_control_flows(%arg0: tensor, %arg1: tensor<1x2x3xf32>, %arg2: tensor<3xf32>) -> (tensor<1x2x3xf32>, tensor<3xf32>) { + %0 = "tf.If"(%arg0, %arg1) {Tcond = i1, Tin = ["tfdtype$DT_FLOAT"], Tout = ["tfdtype$DT_FLOAT"], _xla_propagate_compile_time_consts = true, device = "", else_branch = @reused_if_else_branch, is_stateless = true, name = "if", then_branch = @reused_if_then_branch} : (tensor, tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + %1 = "tf.If"(%arg0, %arg2) {Tcond = i1, Tin = ["tfdtype$DT_FLOAT"], Tout = ["tfdtype$DT_FLOAT"], _xla_propagate_compile_time_consts = true, device = "", else_branch = @reused_if_else_branch, is_stateless = true, name = "if", then_branch = @reused_if_then_branch} : (tensor, tensor<3xf32>) -> tensor<3xf32> + func.return %0, %1 : tensor<1x2x3xf32>, tensor<3xf32> + } + + // CHECK-LABEL: func @reused_if_then_branch + // CHECK-SAME: (%arg0: tensor<*xf32>) -> tensor<*xf32> + // expected-warning @+1 {{expected control flow function @reused_if_then_branch to have exactly 1 use}} + func.func @reused_if_then_branch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: return + // CHECK-SAME: tensor<*xf32> + func.return %arg0 : tensor<*xf32> + } + + // CHECK-LABEL: func @reused_if_else_branch + // CHECK-SAME: (%arg0: tensor<*xf32>) -> tensor<*xf32> + // expected-warning @+1 {{expected control flow function @reused_if_else_branch to have exactly 1 use}} + func.func @reused_if_else_branch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Identity"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> + %0 = "tf.Identity"(%arg0) : (tensor<*xf32>) -> (tensor<*xf32>) + // CHECK: return + // CHECK-SAME: tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @with_graph_and_islands + // CHECK-SAME: %[[ARG_0:.*]]: tensor>> + // CHECK-SAME: -> tensor<4xf32> + func.func @with_graph_and_islands(%arg0: tensor>>) -> tensor<*xf32> { + %graph = tf_executor.graph { + %island:2 = tf_executor.island { + // CHECK: %[[ID_0:.*]] = "tf.IdentityN"(%[[ARG_0]]) + %id0 = "tf.IdentityN"(%arg0) + : (tensor>>) -> tensor>> + // CHECK-NEXT: %[[READ_0:.*]] = "tf.ReadVariableOp"(%[[ID_0]]) + // CHECK-SAME: (tensor>>) -> tensor<4xf32> + %read = "tf.ReadVariableOp"(%id0) : (tensor>>) -> tensor<*xf32> + // CHECK-NEXT: tf_executor.yield %[[READ_0]] : tensor<4xf32> + tf_executor.yield %read : tensor<*xf32> + } + // CHECK: tf_executor.fetch + // CHECK-SAME: tensor<4xf32> + tf_executor.fetch %island#0 : tensor<*xf32> + } + // CHECK: return + // CHECK-SAME: tensor<4xf32> + func.return %graph : tensor<*xf32> + } + + // CHECK-LABEL: func @next_iteration_user + func.func @next_iteration_user(%arg0: tensor<32x?x256x4xf32>) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf_executor.NextIteration.Source + // CHECK-SAME: : tensor<32x?x4xf32> + %1:3 = tf_executor.NextIteration.Source : tensor + %out, %c_out = tf_executor.island { + %dims = "tf.Const"() {value = dense<[32, -1, 4]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: "tf.Reshape" + // CHECK-SAME: -> tensor<32x?x4xf32> + %reshape = "tf.Reshape"(%arg0, %dims) : (tensor<32x?x256x4xf32>, tensor<3xi32>) -> tensor + // CHECK: tf_executor.yield + // CHECK-SAME: : tensor<32x?x4xf32> + tf_executor.yield %reshape : tensor + } + // CHECK: tf_executor.NextIteration.Sink + // CHECK-SAME: : tensor<32x?x4xf32> + tf_executor.NextIteration.Sink[%1#1] %out : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor + } + + // Check that supported tf_executor ops can receive data from ops on which + // shape inference has inferred the result types, without throwing any errors. + // CHECK-LABEL: func @supported_tf_executor_users + func.func @supported_tf_executor_users(%arg0: tensor<32x?x256x4xf32>, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> tensor { + %0 = tf_executor.graph { + %island:3 = tf_executor.island { + %dims = "tf.Const"() {value = dense<[32, -1, 4]> : tensor<3xi32>} : () -> tensor<3xi32> + %reshape = "tf.Reshape"(%arg0, %dims) : (tensor<32x?x256x4xf32>, tensor<3xi32>) -> tensor + %cast = "tf.Cast"(%arg2) : (tensor) -> tensor<*xi1> + tf_executor.yield %reshape, %cast : tensor, tensor<*xi1> + } + // CHECK: tf_executor.Merge + // CHECK-SAME: : (tensor<32x?x4xf32>, tensor) -> + // CHECK: tf_executor.Switch + // CHECK-SAME: : (tensor<32x?x4xf32>, tensor) -> + // CHECK: tf_executor._SwitchN + // CHECK-SAME: : tensor + // CHECK: tf_executor.Enter + // CHECK-SAME: : (tensor<32x?x4xf32>) -> + // CHECK: tf_executor.Exit + // CHECK-SAME: : tensor + // CHECK: tf_executor.LoopCond + // CHECK-SAME: tensor + %merge:3 = "tf_executor.Merge"(%island#0, %arg1) : (tensor, tensor) -> (tensor, tensor, !tf_executor.control) + %switch:3 = "tf_executor.Switch"(%island#0, %arg2) : (tensor, tensor) -> (tensor, tensor, !tf_executor.control) + %switchn:3 = "tf_executor._SwitchN"(%island#0, %arg3) {num_outs = 2} : (tensor, tensor) -> (tensor, tensor, !tf_executor.control) + %enter:2 = "tf_executor.Enter"(%island#0) { frame_name = "frame"} : (tensor) -> (tensor, !tf_executor.control) + %exit:2 = "tf_executor.Exit"(%island#0) : (tensor) -> (tensor, !tf_executor.control) + %loop_cond:2 = "tf_executor.LoopCond" (%island#1) : (tensor<*xi1>) -> (tensor<*xi1>, !tf_executor.control) + tf_executor.fetch %enter#0 : tensor + } + func.return %0 : tensor + } + + // Tests that tensor.cast result shapes are refined. + // CHECK-LABEL: func @tensor_cast_refine_return + func.func @tensor_cast_refine_return(%arg0: tensor<4xi32>) -> (tensor<*xi32>) { + // CHECK-NOT: tensor.cast + %0 = tensor.cast %arg0 : tensor<4xi32> to tensor<*xi32> + func.return %0 : tensor<*xi32> + } + + // CHECK-LABEL: func @tensor_cast_refine_user + func.func @tensor_cast_refine_user(%arg0: tensor<1x?xf32>, %arg1: tensor) -> tensor<*xf32> { + // CHECK: tensor.cast %{{.*}} : tensor<1x?xf32> to tensor<1x?xf32> + %0 = tensor.cast %arg0 : tensor<1x?xf32> to tensor<*xf32> + // CHECK: tensor.cast %{{.*}} : tensor to tensor + %1 = tensor.cast %arg1 : tensor to tensor<*xf32> + // CHECK: tf.Add + // CHECK-SAME: (tensor<1x?xf32>, tensor) -> tensor + %2 = "tf.AddV2"(%0, %1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> + } + + // CHECK-LABEL: func @cast_variant_same_shape + func.func @cast_variant_same_shape(%arg0: tensor>>) -> tensor { + // CHECK: Cast + // CHECK-SAME: (tensor>>) -> tensor>> + %0 = "tf.Cast"(%arg0) {Truncate = false} : (tensor>>) -> tensor + %1 = "tf.Identity"(%0) : (tensor) -> tensor + func.return %1 : tensor + } + + // CHECK-LABEL: func @cast_variant_to_unranked + func.func @cast_variant_to_unranked(%arg0: tensor>>) -> tensor<*x!tf_type.variant> { + // CHECK: Cast + // CHECK-SAME: (tensor>>) -> tensor>> + %0 = "tf.Cast"(%arg0) {Truncate = false} : (tensor>>) -> tensor<*x!tf_type.variant> + %1 = "tf.Identity"(%0) : (tensor<*x!tf_type.variant>) -> tensor<*x!tf_type.variant> + func.return %1 : tensor<*x!tf_type.variant> + } + + // CHECK-LABEL: func @while_variant + // CHECK-SAME: -> tensor>> + func.func @while_variant(%arg0: tensor>>) -> tensor { + // CHECK: tf.While + // CHECK-SAME: -> tensor>> + %0 = "tf.While"(%arg0) {cond = @variant_cond_func, body = @variant_body_func, is_stateless = true} : (tensor>>) -> tensor + // CHECK: tf.ZerosLike + // CHECK-SAME: -> tensor>> + %1 = "tf.ZerosLike"(%0) : (tensor) -> tensor + // CHECK: tf.Identity + // CHECK-SAME: -> tensor>> + %2 = "tf.Identity"(%1) : (tensor) -> tensor + func.return %2 : tensor + } + // CHECK-LABEL: func @variant_cond_func + func.func @variant_cond_func(%arg0: tensor>>) -> tensor { + %0 = "tf._SomeOp"() : () -> tensor + func.return %0 : tensor + } + // CHECK-LABEL: func @variant_body_func + func.func @variant_body_func(%arg0: tensor>>) -> tensor>> { + func.return %arg0 : tensor>> + } + + // CHECK-LABEL: zeros_like_constrained_result_shape + func.func @zeros_like_constrained_result_shape(%arg0: tensor) -> tensor { + // CHECK: %0 = "tf.ZerosLike"(%arg0) : (tensor) -> tensor + // CHECK: return %0 : tensor + %0 = "tf.ZerosLike"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } + + // Test propagation from called functions to the call site. + // CHECK-LABEL: func @stateful_partitioned_call( + // CHECK-SAME: -> tensor<20xi32> + func.func @stateful_partitioned_call(%arg0: tensor<20xi32>, %arg1: tensor) -> tensor<*xi32> { + // CHECK: tf.PartitionedCall + // CHECK-SAME: (tensor<20xi32>) -> tensor<20xi32> + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @partitioned_called_func} : (tensor<20xi32>) -> tensor<*xi32> + // CHECK: tf.StatefulPartitionedCall + // CHECK-SAME: (tensor<20xi32>) -> tensor<20xi32> + %1 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @stateful_partitioned_call_func} : (tensor<20xi32>) -> tensor<*xi32> + // CHECK: tf.TPUPartitionedCall + // CHECK-SAME: (tensor<20xi32>, tensor) -> tensor<20xi32> + %2 = "tf.TPUPartitionedCall"(%arg0, %arg1) {autotuner_thresh = 0 : i64, f = @tpu_partitioned_call_func} : (tensor<20xi32>, tensor) -> tensor<*xi32> + func.return %0 : tensor<*xi32> + } + func.func @partitioned_called_func(%arg0: tensor) -> (tensor) { + func.return %arg0 : tensor + } + func.func @stateful_partitioned_call_func(%arg0: tensor) -> (tensor) { + func.return %arg0 : tensor + } + func.func @tpu_partitioned_call_func(%arg0: tensor) -> (tensor) { + func.return %arg0 : tensor + } + + // Test propagation involving const values across caller and callee. + func.func @partitioned_call_const(%arg0 : tensor<6xf32>) -> tensor<*xf32> { + %0 = "tf.Const"() {value = dense<[3, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %1 = "tf.PartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @partitioned_call_func_const} : (tensor<2xi32>) -> (tensor<2xi32>) + // CHECK: "tf.Reshape" + // CHECK-SAME: tensor<3x2xf32> + %2 = "tf.Reshape"(%arg0, %1) : (tensor<6xf32>, tensor<2xi32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> + } + + // CHECK-LABEL: func @partitioned_call_func_const + func.func @partitioned_call_func_const(%arg0: tensor<2xi32>) -> tensor<2xi32> { + func.return %arg0 : tensor<2xi32> + } + + // Test iteratively updating call site if a func.call is used. + // CHECK-LABEL: func @call_partitioned_call2( + // CHECK-SAME: -> tensor<1xi32> + func.func @call_partitioned_call2() -> tensor<*xi32> { + // CHECK: () -> tensor<1xi32> + %0 = func.call @partitioned_called_func2() : () -> tensor<*xi32> + func.return %0 : tensor<*xi32> + } + // CHECK-LABEL: func @partitioned_called_func2( + // CHECK-SAME: -> tensor<1xi32> + func.func @partitioned_called_func2() -> (tensor<*xi32>) { + %0 = "tf.Const"() {value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + %1 = tensor.cast %0 : tensor<1xi32> to tensor<*xi32> + func.return %1 : tensor<*xi32> + } + + // CHECK-LABEL: func @tensor_list_refine + func.func @tensor_list_refine() { + tf_executor.graph { + %control = tf_executor.island { + %0 = "tf.Const"() {device = "", value = dense<2> : tensor<2xi32>} : () -> tensor<2xi32> + %1 = "tf.Const"() {device = "", value = dense<3> : tensor} : () -> tensor + // CHECK: TensorListReserve{{.*}}-> tensor>> + %2 = "tf.TensorListReserve"(%0, %1) {device = ""} : (tensor<2xi32>, tensor) -> tensor>> + // CHECK: TensorListReserve{{.*}}-> tensor>> + %3 = "tf.TensorListReserve"(%0, %1) {device = ""} : (tensor<2xi32>, tensor) -> tensor>> + %4 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %5 = "tf.Const"() {device = "", value = dense<[[1.000000e+00, 2.000000e+00], [3.000000e+00, 4.000000e+00]]> : tensor<2x2xf32>} : () -> tensor<2x2xf32> + // CHECK: tf.TensorListSetItem{{.*}}: (tensor>>, tensor, tensor<2x2xf32>) -> tensor>> + %6 = "tf.TensorListSetItem"(%3, %4, %5) {device = ""} : (tensor>>, tensor, tensor<2x2xf32>)-> tensor<*x!tf_type.variant> + %7 = "tf.Const"() {device = "", value = dense<-1> : tensor} : () -> tensor + %8 = "tf.StopGradient"(%6) : (tensor<*x!tf_type.variant>) -> tensor<*x!tf_type.variant> + // CHECK: tf.TensorListStack{{.*}}: (tensor>>, tensor) -> tensor + %9 = "tf.TensorListStack"(%8, %7) {device = "", num_elements = -1 : i64} : (tensor<*x!tf_type.variant>, tensor) -> tensor<*xf32> + tf_executor.yield + } + tf_executor.fetch + } + func.return + } + + // CHECK-LABEL: single_mutation_same_element_shape + func.func @single_mutation_same_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %tl_2 = "tf.TensorListPushBack"(%tl_1, %elem) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %stack = "tf.TensorListStack"(%tl_2, %elem_shape) {num_elements = -1 : i64} : (tensor>>, tensor<2xi32>) -> tensor<*xf32> + func.return + } + + // CHECK-LABEL: single_mutation_multiple_element_shape + func.func @single_mutation_multiple_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<8x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %tl_2 = "tf.TensorListPushBack"(%tl_1, %elem_1) : (tensor>>, tensor<8x1xf32>) -> tensor>> + func.return + } + + // CHECK-LABEL: single_mutation_dynamic_element_shape + func.func @single_mutation_dynamic_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + %elem_1 = "tf._SomeOtherOp"() : () -> tensor + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %tl_2 = "tf.TensorListPushBack"(%tl_1, %elem_1) : (tensor>>, tensor) -> tensor>> + func.return + } + + // CHECK-LABEL: multiple_mutation_same_element_shape + func.func @multiple_mutation_same_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %resize = "tf.Const"() {value = dense<20> : tensor} : () -> tensor + %tl_2 = "tf.TensorListResize"(%tl_0, %resize) : (tensor>>, tensor) -> tensor>> + func.return + } + + // CHECK-LABEL: multiple_mutation_multiple_element_shape + func.func @multiple_mutation_multiple_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %tl_2 = "tf.TensorListPushBack"(%tl_1, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %zero = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<8x1xf32> + %tl_3 = "tf.TensorListSetItem"(%tl_1, %zero, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + func.return + } + + // CHECK-LABEL: multiple_mutation_dynamic_element_shape + func.func @multiple_mutation_dynamic_element_shape() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %tl_2 = "tf.TensorListPushBack"(%tl_1, %elem_0) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %zero = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %elem_1 = "tf._SomeOtherOp"() : () -> tensor + %tl_3 = "tf.TensorListSetItem"(%tl_1, %zero, %elem_1) : (tensor>>, tensor, tensor) -> tensor>> + func.return + } + + // CHECK-LABEL: infer_subtype_from_scatter + func.func @infer_subtype_from_scatter() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %tensors = "tf._SomeOp"() : () -> tensor<3x16x1xf32> + %indices = "tf.Const"() {value = dense<[2, 5, 9]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: TensorListReserve + // CHECK-SAME: tensor>> + %tl_0 = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListScatterIntoExistingList"(%tl_0, %tensors, %indices) : (tensor>>, tensor<3x16x1xf32>, tensor<3xi32>) -> tensor>> + func.return + } + + // CHECK-LABEL: tensorlist_from_tensor + func.func @tensorlist_from_tensor() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %tensors = "tf._SomeOp"() : () -> tensor<10x16x1xf32> + // CHECK: TensorListFromTensor + // CHECK-SAME: tensor>> + %tl = "tf.TensorListFromTensor"(%tensors, %elem_shape) : (tensor<10x16x1xf32>, tensor<2xi32>) -> tensor>> + func.return + } + + // CHECK-LABEL: infer_subtype_from_write_while + func.func @infer_subtype_from_write_while() { + %zero = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK-SAME: tensor>> + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:2 = "tf.WhileRegion"(%zero, %tl) ({ + ^bb0(%barg0: tensor, %barg1: tensor>>): // no predeceessors + %cond = "tf.Less"(%barg0, %size) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: tensor>>): // no predeceessors + %index = "tf.AddV2"(%barg0, %one) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor<16x1xf32> + %tl_loop = "tf.TensorListSetItem"(%barg1, %index, %elem) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + "tf.Yield"(%index, %tl_loop) : (tensor, tensor>>) -> () + }) {is_stateless = false} : (tensor, tensor>>) -> (tensor, tensor>>) + func.return + } + + // CHECK-LABEL: tensor_list_if_region_yield_multiple_elem_shape + func.func @tensor_list_if_region_yield_multiple_elem_shape(%arg0: tensor) -> () { + %zero = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK-SAME: tensor>> + %tl_0 = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %elem_0 = "tf.SomeOp"() : () -> tensor<16x1xf32> + %elem_1 = "tf.SomeOp"() : () -> tensor<16x1xf32> + %elem_2 = "tf.SomeOp"() : () -> tensor<8x1xf32> + %tl_1 = "tf.IfRegion"(%arg0) ({ + %tl_true = "tf.TensorListSetItem"(%tl_0, %zero, %elem_0) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + "tf.Yield"(%tl_true) : (tensor>>) -> () + }, { + %tl_false = "tf.TensorListSetItem"(%tl_0, %zero, %elem_1) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + "tf.Yield"(%tl_false) : (tensor>>) -> () + }) {is_stateless = false} : (tensor) -> tensor>> + %tl_2 = "tf.TensorListSetItem"(%tl_1, %one, %elem_2) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + func.return + } + + // CHECK-LABEL: while_result_multiple_element_shape + func.func @while_result_multiple_element_shape() { + %zero = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: TensorListReserve + // CHECK-SAME: tensor>> + %tl = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %while:2 = "tf.WhileRegion"(%zero, %tl) ({ + ^bb0(%barg0: tensor, %barg1: tensor>>): // no predeceessors + %cond = "tf.Less"(%barg0, %size) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: tensor>>): // no predeceessors + %index = "tf.AddV2"(%barg0, %one) : (tensor, tensor) -> tensor + %elem_0 = "tf._SomeOp"() : () -> tensor<16x1xf32> + %tl_loop = "tf.TensorListSetItem"(%barg1, %index, %elem_0) : (tensor>>, tensor, tensor<16x1xf32>) -> tensor>> + "tf.Yield"(%index, %tl_loop) : (tensor, tensor>>) -> () + }) {is_stateless = false} : (tensor, tensor>>) -> (tensor, tensor>>) + %elem_1 = "tf._SomeOtherOp"() : () -> tensor<8x1xf32> + %tl_set_item = "tf.TensorListSetItem"(%while#1, %one, %elem_1) : (tensor>>, tensor, tensor<8x1xf32>) -> tensor>> + func.return + } + + // CHECK-LABEL: do_not_refine_tensorlist_with_unknown_user + func.func @do_not_refine_tensorlist_with_unknown_user() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem) : (tensor>>, tensor<16x1xf32>) -> tensor>> + "tf.UnknownTensorListUser"(%tl_1) : (tensor>>) -> () + func.return + } + + // CHECK-LABEL: replace_tensor_list_element_shape + func.func @replace_tensor_list_element_shape() { + // CHECK: %[[ELEMENT_SHAPE:.*]] = "tf.Const"() <{value = dense<[-1, 1]> : tensor<2xi32>}> + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<16x1xf32> + // CHECK: EmptyTensorList + // CHECK-SAME: tensor>> + %tl_0 = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + %tl_1 = "tf.TensorListPushBack"(%tl_0, %elem) : (tensor>>, tensor<16x1xf32>) -> tensor>> + %shape_32 = "tf.TensorListElementShape"(%tl_1) : (tensor>>) -> tensor + %shape_64 = "tf.TensorListElementShape"(%tl_1) : (tensor>>) -> tensor + // CHECK: %[[CAST:.*]] = "tf.Cast"(%[[ELEMENT_SHAPE]]){{.*}}: (tensor<2xi32>) -> tensor<2xi64> + // CHECK: "tf._SomeOtherOp"(%[[ELEMENT_SHAPE]], %[[CAST]]) + "tf._SomeOtherOp"(%shape_32, %shape_64) : (tensor, tensor) -> () + func.return + } + + // CHECK-LABEL: refine_pop_back_results_from_operands + func.func @refine_pop_back_results_from_operands(%arg0: tensor>>, %arg1: tensor<1xi32>) -> (tensor, tensor<*xi32>) { + %0, %1 = "tf.TensorListPopBack"(%arg0, %arg1) : (tensor>>, tensor<1xi32>) -> (tensor, tensor<*xi32>) + // CHECK: %output_handle, %tensor = "tf.TensorListPopBack"(%arg0, %arg1) : (tensor>>, tensor<1xi32>) -> (tensor>>, tensor<2xi32>) + // CHECK: return %output_handle, %tensor : tensor>>, tensor<2xi32> + func.return %0, %1 : tensor, tensor<*xi32> + } + + // CHECK-LABEL: do_not_unrefine_fully_defined_subtypes + func.func @do_not_unrefine_fully_defined_subtypes() { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %size = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + // CHECK: "tf.TensorListReserve" + // CHECK-SAME: tensor>> + %tl_0 = "tf.TensorListReserve"(%elem_shape, %size) : (tensor<2xi32>, tensor) -> tensor>> + func.return + } + + // CHECK-LABEL: dont_update_for_ref + func.func @dont_update_for_ref() -> () { + // CHECK: () -> tensor<4x!tf_type.f32ref> + %11 = "tf.VariableV2"() {container = "", device = "", shape = #tf_type.shape<4>, shared_name = ""} : () -> tensor<4x!tf_type.f32ref> + // CHECK: (tensor<4x!tf_type.f32ref>) -> tensor<4xf32> + %12 = "tf.Identity"(%11) {device = ""} : (tensor<4x!tf_type.f32ref>) -> tensor<4xf32> + // CHECK: (tensor<4xf32>) -> tensor<4xf32> + %13 = "tf.Neg"(%12) {device = ""} : (tensor<4xf32>) -> tensor<4xf32> + func.return + } + + // CHECK-LABEL: operand_as_shape + func.func @operand_as_shape(%18: tensor, %39: tensor<1x4x4x32xf32>) -> () { + %cst_5 = arith.constant dense<512> : tensor + %19 = "tf.Pack"(%18, %cst_5) {N = 2 : i64, T = i32, axis = 0 : i64, device = ""} : (tensor, tensor) -> tensor<2xi32> + // CHECK: -> tensor<1x512xf32> + %40 = "tf.Reshape"(%39, %19) {T = f32, Tshape = i32, device = ""} : (tensor<1x4x4x32xf32>, tensor<2xi32>) -> tensor + func.return + } + + // CHECK-LABEL: func @shape_partial_eval + func.func @shape_partial_eval(%arg0: tensor<1x?x7xf32>, %arg1: tensor<3x7xf32>) -> tensor<*xf32> { + %0 = "tf.Shape"(%arg0) : (tensor<1x?x7xf32>) -> tensor<3xi32> + + // CHECK: tf.Reshape + // CHECK-SAME: tensor<1x3x7xf32> + %1 = "tf.Reshape"(%arg1, %0) : (tensor<3x7xf32>, tensor<3xi32>) -> tensor<*xf32> + return %1 : tensor<*xf32> + } + + // CHECK-LABEL: func @gather_concat_reshape + func.func @gather_concat_reshape(%arg0: tensor, %arg1: tensor) -> tensor<*xf32> { + %0 = "tf.Shape"(%arg0) : (tensor) -> tensor<3xi32> + + %indices = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %axis = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor + %1 = "tf.GatherV2"(%0, %indices, %axis) {batch_dims = 0 : i64, device = ""} : (tensor<3xi32>, tensor<2xi32>, tensor) -> tensor<2xi32> + + %last_dim = "tf.Const"() {value = dense<7> : tensor<1xi32>} : () -> tensor<1xi32> + %2 = "tf.ConcatV2"(%1, %last_dim, %axis) {device = ""} : (tensor<2xi32>, tensor<1xi32>, tensor) -> tensor<3xi32> + + + // CHECK: tf.Reshape + // CHECK-SAME: tensor + %3 = "tf.Reshape"(%arg1, %2) : (tensor, tensor<3xi32>) -> tensor<*xf32> + return %3 : tensor<*xf32> + } + + // CHECK-LABEL: const_fold + func.func @const_fold() -> () { + // CHECK: tf.Const + // CHECK-SAME: () -> tensor<4xi32> + %0 = "tf.Const"() {value = dense<[200, 26, 26, 32]> : tensor<4xi32>} : () -> tensor<*xi32> + // CHECK: tf.Const + // CHECK-SAME: () -> tensor<4xi32> + %1 = "tf.Const"() {value = dense<[200, 26, 26, 32]> : tensor<4xi32>} : () -> tensor<*xi32> + // CHECK: tf.Add + // CHECK-SAME: (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + %2 = "tf.Add"(%0, %1) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + func.return + } + + // CHECK-LABEL: cast_at_end(%arg0: + // CHECK-SAME: tensor<16x194x199x4xui8>, tensor<16x194x199x4xi8>, tensor<*xi8>, tensor + func.func @cast_at_end(%arg0: tensor<16x194x199x4xf32>, %arg1: tensor<16x194x199x4xi8>, %arg2: tensor<*xf32>) -> (tensor<*xui8>, tensor<*xi8>, tensor<*xi8>, tensor) { + %27 = "tf.Cast"(%arg0) {Truncate = false, device = ""} : (tensor<16x194x199x4xf32>) -> tensor<*xui8> + %28 = "tf.Cast"(%arg0) {Truncate = false, device = ""} : (tensor<16x194x199x4xf32>) -> tensor<*xi8> + %29 = "tf.Cast"(%arg2) {Truncate = false, device = ""} : (tensor<*xf32>) -> tensor<*xi8> + // CHECK: %[[CAST_RESULT_2:.*]] = "tf.Cast"(%arg0) + // CHECK-SAME: (tensor<16x194x199x4xf32>) -> tensor<*xi8> + // CHECK: %[[CAST_RESULT_3:.*]] = "tf.Cast"(%arg2) + // CHECK-SAME: (tensor<*xf32>) -> tensor<*xi8> + // CHECK: %[[TENSOR_CAST:.*]] = tensor.cast %[[CAST_RESULT_2]] : tensor<*xi8> to tensor + %2 = tensor.cast %28 : tensor<*xi8> to tensor + // CHECK: %[[CAST_RESULT_0:.*]] = "tf.Cast"(%arg0) + // CHECK-SAME: (tensor<16x194x199x4xf32>) -> tensor<16x194x199x4xui8> + // CHECK: %[[CAST_RESULT_1:.*]] = "tf.Cast"(%arg0) + // CHECK-SAME: (tensor<16x194x199x4xf32>) -> tensor<16x194x199x4xi8> + // CHECK: return %[[CAST_RESULT_0]], %[[CAST_RESULT_1]], %[[CAST_RESULT_3]], %[[TENSOR_CAST]] + func.return %27, %28, %29, %2 : tensor<*xui8>, tensor<*xi8>, tensor<*xi8>, tensor + } + + // CHECK-LABEL: infer_device_launch + func.func @infer_device_launch(%arg0: tensor<1x8x2xi32>) -> (tensor<*xf32>, tensor<*xf32>) { + %0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %1 = "tf_device.launch"() ({ + %2 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<1x8x2xi32>) -> tensor<1x8x2xf32> + tf_device.return %2 : tensor<1x8x2xf32> + // CHECK: () -> tensor<1x8x2xf32> + }) {device = "/device:CPU:0"} : () -> tensor<*xf32> + // CHECK: "tf.Cast"(%{{.*}}) <{Truncate = false}> : (tensor<1x8x2xf32>) -> tensor<*xf32> + // CHECK: (tensor, tensor<1x8x2xf32>) -> (tensor<1x8x1xf32>, tensor<1x8x1xf32>) + %3:2 = "tf.Split"(%0, %1) {device = ""} : (tensor, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + %4 = tensor.cast %1 : tensor<*xf32> to tensor + func.return %3#0, %3#1 : tensor<*xf32>, tensor<*xf32> + } + + // CHECK-LABEL: infer_device_cluster + func.func @infer_device_cluster(%arg0: tensor<1x8x2xi32>) -> (tensor<*xf32>, tensor<*xf32>) { + %0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %1 = "tf_device.cluster"() ({ + %2 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<1x8x2xi32>) -> tensor<1x8x2xf32> + tf_device.return %2 : tensor<1x8x2xf32> + // CHECK: () -> tensor<1x8x2xf32> + }) : () -> tensor<*xf32> + // CHECK: "tf.Cast"(%{{.*}}) <{Truncate = false}> : (tensor<1x8x2xf32>) -> tensor<*xf32> + // CHECK: (tensor, tensor<1x8x2xf32>) -> (tensor<1x8x1xf32>, tensor<1x8x1xf32>) + %3:2 = "tf.Split"(%0, %1) {device = ""} : (tensor, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + %4 = tensor.cast %1 : tensor<*xf32> to tensor + func.return %3#0, %3#1 : tensor<*xf32>, tensor<*xf32> + } + + // CHECK-LABEL: func @tensor_cast_dont_infer + func.func @tensor_cast_dont_infer(%arg0: tensor) -> tensor<1xi32> { + // CHECK: %[[RESULT:.*]] = tensor.cast + // CHECK-SAME: tensor to tensor<1xi32> + // CHECK: return %[[RESULT]] : tensor<1xi32> + %2 = tensor.cast %arg0 : tensor to tensor<1xi32> + func.return %2 : tensor<1xi32> + } + + // CHECK-LABEL: func @tensor_cast_partial_infer + func.func @tensor_cast_partial_infer(%arg0: tensor) -> tensor<10x?xi32> { + // CHECK: return {{.*}} : tensor<10x10xi32> + %2 = tensor.cast %arg0 : tensor to tensor<10x?xi32> + func.return %2 : tensor<10x?xi32> + } + + // CHECK-LABEL: operand_pack_unranked + // Verify fix: this only verifies that shape inference runs and completes on + // this input, rather than refining any shapes. + func.func @operand_pack_unranked(%arg0: tensor<*xf32>) -> () { + // CHECK: tf.Pack + %outputs_0 = "tf.Pack"(%arg0) {axis = 0 : i64, device = ""} : (tensor<*xf32>) -> tensor<*xf32> + %outputs_2 = "tf.TensorSliceDataset"(%outputs_0) {device = "", output_shapes = [#tf_type.shape<>], metadata=""} : (tensor<*xf32>) -> tensor + func.return + } + + // Test resource result subtypes are propagated to call op results. + // CHECK-LABEL: func @pcall_resource_result + func.func @pcall_resource_result(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: "tf.StatefulPartitionedCall" + // CHECK-SAME: (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_resource_result_func} : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource> + func.return + } + func.func @pcall_resource_result_func(%arg0: tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> { + func.return %arg0 : tensor<*x!tf_type.resource>> + } + + // Check that the fold for tf.Size does not crash with unranked output type. + // CHECK-LABEL: func @unranked_tf_size + func.func @unranked_tf_size() -> tensor<*xi32> { + %0 = "tf.Const"() {value = dense<[-1, 26]> : tensor<2xi32>} : () -> tensor<2xi32> + %add = "tf.AddV2"(%0, %0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<*xi32> + // CHECK: "tf.Size" + // CHECK-SAME: (tensor<2xi32>) -> tensor + %size = "tf.Size"(%add) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + func.return %size : tensor<*xi32> + } + + // Test no tf.Cast ops are inserted when refining tf_executor.graph results. + // CHECK-LABEL: func @call_in_graph({{%.+}}: tensor) -> tensor + func.func @call_in_graph(%arg0: tensor) -> tensor<*xi32> { + // CHECK-NOT: tf.Cast + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @call_in_graph_func} : (tensor) -> tensor<*xi32> + tf_executor.fetch %1#0 : tensor<*xi32> + } + func.return %0 : tensor<*xi32> + } + + // Test fetch and yield are diectly assigned to island and graph ops results. + // CHECK-LABEL: func @call_in_graph_func({{%.+}}: tensor) -> tensor + func.func @call_in_graph_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + // CHECK-NOT: tf.Cast + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.Identity"(%arg0) : (tensor<*xi32>) -> tensor<*xi32> + tf_executor.fetch %1#0 : tensor<*xi32> + } + func.return %0 : tensor<*xi32> + } + + // CHECK-LABEL: func @call_in_graph_1 + func.func @call_in_graph_1(%arg0: tensor, %arg1: tensor<5x5x1x32xbf16>) -> tensor<*xbf16> { + // CHECK: tf_executor.fetch %outputs : tensor + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.PartitionedCall"(%arg0, %arg1) { + config = "", config_proto = "", executor_type = "", f = @call_in_graph_func_1} : (tensor, tensor<5x5x1x32xbf16>) -> tensor<*xbf16> + tf_executor.fetch %1#0 : tensor<*xbf16> + } + func.return %0 : tensor<*xbf16> + } + + // CHECK-LABEL: func @call_in_graph_func_1 + func.func @call_in_graph_func_1(%arg0: tensor, %arg1: tensor<5x5x1x32xbf16>) -> tensor { + // CHECK: tf_executor.fetch %outputs : tensor + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.Conv2D"(%arg0, %arg1) {data_format = "NHWC", device = "", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true}: (tensor, tensor<5x5x1x32xbf16>) -> tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor + } + + // Test shape invariant While only propagates operand handle types into + // results and functions/regions. + // CHECK-LABEL: func @while_shape_invariant_propagate + // CHECK-SAME: ({{%.+}}: tensor<4xf32>, {{%.+}}: tensor>>, {{%.+}}: tensor>>, {{%.+}}: tensor<1xi32>) + // CHECK-SAME: -> (tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor, tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor) + func.func @while_shape_invariant_propagate(%arg0: tensor<4xf32>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor<1xi32>) -> (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor, tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor) { + // CHECK: "tf.While" + // CHECK-SAME: (tensor<4xf32>, tensor>>, tensor>>, tensor<1xi32>) + // CHECK-SAME: -> (tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor) + %0:4 = "tf.While"(%arg0, %arg1, %arg2, %arg3) {cond = @while_shape_invariant_cond_func_propagate, body = @while_shape_invariant_body_func_propagate, is_stateless = false, shape_invariant} : (tensor<4xf32>, tensor>>, tensor>>, tensor<1xi32>) -> (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor) + + // CHECK: "tf.WhileRegion" + // CHECK-SAME: shape_invariant + %1:4 = "tf.WhileRegion"(%arg0, %arg1, %arg2, %arg3) <{is_stateless = false, shape_invariant}> ({ + // CHECK-NEXT: ^{{.+}}({{%.+}}: tensor<*xf32>, {{%.+}}: tensor<*x!tf_type.resource>>, {{%.+}}: tensor>>, {{%.+}}: tensor): + ^cond(%carg0: tensor<*xf32>, %carg1: tensor<*x!tf_type.resource>, %carg2: tensor, %carg3: tensor): + %2 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + // CHECK: ^{{.+}}({{%.+}}: tensor<*xf32>, {{%.+}}: tensor<*x!tf_type.resource>>, {{%.+}}: tensor>>, {{%.+}}: tensor): + ^body(%barg0: tensor<*xf32>, %barg1: tensor<*x!tf_type.resource>, %barg2: tensor, %barg3: tensor): + %2 = "tf.SomeOp"(%barg3) : (tensor) -> tensor + // CHECK: "tf.Yield" + // CHECK-SAME: (tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor) -> () + "tf.Yield"(%barg0, %barg1, %barg2, %2) : (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor) -> () + // CHECK-NEXT: (tensor<4xf32>, tensor>>, tensor>>, tensor<1xi32>) + // CHECK-SAME: -> (tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor) + }) : (tensor<4xf32>, tensor>>, tensor>>, tensor<1xi32>) -> (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor) + + func.return %0#0, %0#1, %0#2, %0#3, %1#0, %1#1, %1#2, %1#3 : tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor, tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor + } + + // CHECK-LABEL: func @while_shape_invariant_cond_func_propagate + // CHECK-SAME: ({{%.+}}: tensor<*xf32>, {{%.+}}: tensor<*x!tf_type.resource>>, {{%.+}}: tensor>>, {{%.+}}: tensor) + // CHECK-SAME: -> tensor + func.func @while_shape_invariant_cond_func_propagate(%arg0: tensor<*xf32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor, %arg3: tensor) -> tensor { + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @while_shape_invariant_body_func_propagate + // CHECK-SAME: ({{%.+}}: tensor<*xf32>, {{%.+}}: tensor<*x!tf_type.resource>>, {{%.+}}: tensor>>, {{%.+}}: tensor) + // CHECK-SAME: -> (tensor<*xf32>, tensor<*x!tf_type.resource>>, tensor>>, tensor) + func.func @while_shape_invariant_body_func_propagate(%arg0: tensor<*xf32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor, %arg3: tensor) -> (tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor) { + %0 = "tf.SomeOp"(%arg3) : (tensor) -> tensor + func.return %arg0, %arg1, %arg2, %0 : tensor<*xf32>, tensor<*x!tf_type.resource>, tensor, tensor + } + + // Test shape invariant While replaces different dimensions with a dynamic + // dimension when creating a shape for refining cond and body. + // CHECK-LABEL: func @while_shape_invariant_different_dims + // CHECK-SAME: ({{%.+}}: tensor<1x2x3xf32>) + // CHECK-SAME: -> (tensor<1x8x3xf32>, tensor<1x8x3xf32>) + func.func @while_shape_invariant_different_dims(%arg0: tensor<1x2x3xf32>) -> (tensor<1x8x3xf32>, tensor<1x8x3xf32>) { + // CHECK: "tf.While" + // CHECK-SAME: (tensor<1x2x3xf32>) + // CHECK-SAME: -> tensor<1x8x3xf32> + %0 = "tf.While"(%arg0) {cond = @while_shape_invariant_cond_func_different_dims, body = @while_shape_invariant_body_func_different_dims, is_stateless = false, shape_invariant} : (tensor<1x2x3xf32>) -> tensor<1x8x3xf32> + + // CHECK: "tf.WhileRegion" + // CHECK-SAME: shape_invariant + %1 = "tf.WhileRegion"(%arg0) <{is_stateless = false, shape_invariant}> ({ + // CHECK-NEXT: ^{{.+}}({{%.+}}: tensor<1x?x3xf32>): + ^cond(%carg0: tensor<*xf32>): + %2 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + // CHECK: ^{{.+}}({{%.+}}: tensor<1x?x3xf32>): + ^body(%barg0: tensor<*xf32>): + %2 = "tf.Identity"(%barg0) : (tensor<*xf32>) -> tensor<*xf32> + // CHECK: "tf.Yield" + // CHECK-SAME: (tensor<1x?x3xf32>) -> () + "tf.Yield"(%2) : (tensor<*xf32>) -> () + // CHECK-NEXT: (tensor<1x2x3xf32>) + // CHECK-SAME: -> tensor<1x8x3xf32> + }) : (tensor<1x2x3xf32>) -> tensor<1x8x3xf32> + + func.return %0, %1 : tensor<1x8x3xf32>, tensor<1x8x3xf32> + } + + // CHECK-LABEL: func @while_shape_invariant_cond_func_different_dims + // CHECK-SAME: ({{%.+}}: tensor<1x?x3xf32>) + // CHECK-SAME: -> tensor + func.func @while_shape_invariant_cond_func_different_dims(%arg0: tensor<*xf32>) -> tensor { + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @while_shape_invariant_body_func_different_dims + // CHECK-SAME: ({{%.+}}: tensor<1x?x3xf32>) + // CHECK-SAME: -> tensor<1x?x3xf32> + func.func @while_shape_invariant_body_func_different_dims(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.Identity"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // Test shape invariant While can propagate handle type to result from body + // result. + // CHECK-LABEL: func @while_shape_invariant_body_result_propagate + // CHECK-SAME: ({{%.+}}: tensor<*x!tf_type.resource>>) + // CHECK-SAME: -> (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>) + func.func @while_shape_invariant_body_result_propagate(%arg0: tensor<*x!tf_type.resource>>) -> (tensor<*x!tf_type.resource>, tensor<*x!tf_type.resource>) { + // CHECK: "tf.While" + // CHECK-SAME: (tensor<*x!tf_type.resource>>) + // CHECK-SAME: -> tensor<*x!tf_type.resource>> + %0 = "tf.While"(%arg0) {cond = @while_shape_invariant_cond_func_body_result_propagate, body = @while_shape_invariant_body_func_body_result_propagate, is_stateless = false, shape_invariant} : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource> + + // CHECK: "tf.WhileRegion" + // CHECK-SAME: shape_invariant + %1 = "tf.WhileRegion"(%arg0) <{is_stateless = false, shape_invariant}> ({ + // CHECK-NEXT: ^{{.+}}({{%.+}}: tensor<*x!tf_type.resource>>): + ^cond(%carg0: tensor<*x!tf_type.resource>>): + %2 = "tf.Const"() {value = dense : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + // CHECK: ^{{.+}}({{%.+}}: tensor<*x!tf_type.resource>>): + ^body(%barg0: tensor<*x!tf_type.resource>>): + %2 = "tf.Identity"(%barg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + // CHECK: "tf.Yield" + // CHECK-SAME: (tensor<*x!tf_type.resource>>) -> () + "tf.Yield"(%2) : (tensor<*x!tf_type.resource>>) -> () + // CHECK-NEXT: (tensor<*x!tf_type.resource>>) + // CHECK-SAME: -> tensor<*x!tf_type.resource>> + }) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource> + + func.return %0, %1 : tensor<*x!tf_type.resource>, tensor<*x!tf_type.resource> + } + + // CHECK-LABEL: func @while_shape_invariant_cond_func_body_result_propagate + // CHECK-SAME: ({{%.+}}: tensor<*x!tf_type.resource>>) + // CHECK-SAME: -> tensor + func.func @while_shape_invariant_cond_func_body_result_propagate(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %0 = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @while_shape_invariant_body_func_body_result_propagate + // CHECK-SAME: ({{%.+}}: tensor<*x!tf_type.resource>>) + // CHECK-SAME: -> tensor<*x!tf_type.resource>> + func.func @while_shape_invariant_body_func_body_result_propagate(%arg0: tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> { + %0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + func.return %0 : tensor<*x!tf_type.resource>> + } + + // CHECK-LABEL: func @InferFromValueFolding + func.func @InferFromValueFolding(%arg0 : tensor) -> tensor<*xf32> { + %cst1 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %mul = "tf.Mul"(%arg0, %arg0) : (tensor, tensor) -> tensor + // Folding will infer that: Pow(%mul, 1.0) -> %mul + // However we don't have the actual value for the mul, but we can use the + // mul type! + // CHECK: tf.Pow + // CHECK-SAME: -> tensor + %pow = "tf.Pow"(%mul, %cst1) : (tensor, tensor) -> tensor<*xf32> + func.return %pow : tensor<*xf32> + } + + // Same as above, but don't infer when the type is "less" static. + // CHECK-LABEL: func @DontInferFromValueFolding + func.func @DontInferFromValueFolding(%arg0 : tensor<*xf32>) -> tensor { + %cst1 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %mul = "tf.Mul"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + // Folding will infer that: Pow(%mul, 1.0) -> %mul + // However we don't want to use the type of the mul as the type is less + // static, it'd be lossy. + // CHECK: tf.Pow + // CHECK-SAME: -> tensor + %pow = "tf.Pow"(%mul, %cst1) : (tensor<*xf32>, tensor) -> tensor + func.return %pow : tensor + } + + + // Test propagation of multiple callers into a function when all the callers + // have the same operand types. + + // CHECK-LABEL: func @multi_caller1 + func.func @multi_caller1(%arg0: tensor) -> tensor<*xi32> { + // CHECK: tf.PartitionedCall + // CHECK-SAME: -> tensor + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @multi_caller_callee} : (tensor) -> (tensor<*xi32>) + func.return %0 : tensor<*xi32> + } + // CHECK-LABEL: func @multi_caller2 + func.func @multi_caller2(%arg0: tensor) -> tensor<*xi32> { + // CHECK: tf.PartitionedCall + // CHECK-SAME: -> tensor + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @multi_caller_callee} : (tensor) -> (tensor<*xi32>) + func.return %0 : tensor<*xi32> + } + + // CHECK-LABEL: func private @multi_caller_callee + // CHECK-SAME: (%arg0: tensor) -> tensor + func.func private @multi_caller_callee(%arg0: tensor<*xi32>) -> tensor<*xi32> { + // CHECK: return + // CHECK-SAME: tensor + func.return %arg0 : tensor<*xi32> + } + + // Test conv2d inferReturnTypes can infer some information when input or + // filter does not have fully static shape. + + // CHECK-LABEL: func @conv2d_unranked_input_and_filter + func.func @conv2d_unranked_input_and_filter(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_unranked_filter + func.func @conv2d_unranked_filter(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor<256x?x?x?xf32> + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_unranked_filter_and_dynamic_batch + func.func @conv2d_unranked_filter_and_dynamic_batch(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_unranked_input + func.func @conv2d_unranked_input(%arg0: tensor<*xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<*xf32>, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_unranked_input_and_dynamic_channel + func.func @conv2d_unranked_input_and_dynamic_channel(%arg0: tensor<*xf32>, %arg1: tensor<3x3x3x?xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<*xf32>, tensor<3x3x3x?xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_dynamic_batch + func.func @conv2d_dynamic_batch(%arg0: tensor, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_dynamic_channel + func.func @conv2d_dynamic_channel(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x?xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor<256x32x32x?xf32> + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x?xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_fully_dynamic_spatial_dim + func.func @conv2d_fully_dynamic_spatial_dim(%arg0: tensor<256x?x?x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor<256x?x?x16xf32> + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x?x?x3xf32>, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_partially_dynamic_spatial_dim + func.func @conv2d_partially_dynamic_spatial_dim(%arg0: tensor<256x?x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor<256x?x32x16xf32> + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x?x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_dynamic_batch_and_partially_dynamic_spatial_dim + func.func @conv2d_dynamic_batch_and_partially_dynamic_spatial_dim(%arg0: tensor, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @conv2d_dynamic_batch_and_fully_dynamic_spatial_dim + func.func @conv2d_dynamic_batch_and_fully_dynamic_spatial_dim(%arg0: tensor, %arg1: tensor<3x3x3x16xf32>) -> tensor<*xf32> { + // CHECK: "tf.Conv2D" + // CHECK-SAME: -> tensor + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor, tensor<3x3x3x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: check_walking_identity + func.func @check_walking_identity(%arg0 : tensor<1x192x256x128xf32>) { + %0 = "tf.Const"() {value = dense<2> : tensor<2xi32>} : () -> tensor<2xi32> + %1 = "tf.Const"() {value = dense<2> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + %2 = "tf.Identity"(%1) {device = ""} : (tensor<2x2xi32>) -> tensor<2x2xi32> + // CHECK: SpaceToBatchND{{.*}}-> tensor<4x98x130x128xf32> + %3 = "tf.SpaceToBatchND"(%arg0, %0, %2) {device = ""} : (tensor<1x192x256x128xf32>, tensor<2xi32>, tensor<2x2xi32>) -> tensor<4x?x?x128xf32> + func.return + } + + // CHECK-LABEL: check_subtyperefinement + func.func @check_subtyperefinement(%arg0 : tensor<1x192x256x128xf32>, %arg1 : tensor, %arg2 : tensor) { + // CHECK: TensorListReserve + // CHECK-SAME: -> tensor>> + %0 = "tf.TensorListReserve"(%arg1, %arg1) {device = ""} : (tensor, tensor) -> tensor>> + %1 = "tf.TensorListSetItem"(%0, %arg1, %arg2) {device = ""} : (tensor>>, tensor, tensor) -> tensor>> + func.return + } + + func.func private @quant_fn(%arg0: tensor<*x!quant.uniform>) -> () { + func.return + } + + func.func @xla_call_module_parsing_error(%arg0: tensor) -> tensor<*xf32> { + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor<*xf32> + %1 = "tf.XlaCallModule"(%arg0, %0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], module = "invalid-stablehlo-module", platforms = [], version = 6 : i64} : (tensor, tensor<*xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> + } + + // CHECK-LABEL: func @xla_host_compute_mlir_empty_module + func.func @xla_host_compute_mlir_empty_module(%arg0: tensor<2xf32>) -> tensor<*xf32> { + // CHECK: "tf._XlaHostComputeMlir" + // CHECK-SAME: -> tensor<*xf32> + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", tpu_core = 0, host_mlir_module = ""} : (tensor<2xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @xla_host_compute_mlir_shape_inferred + func.func @xla_host_compute_mlir_shape_inferred(%arg0: tensor<2xf32>) -> tensor<*xf32> { + // CHECK: "tf._XlaHostComputeMlir" + // CHECK-SAME: -> tensor<2xf32> + // CHECK: return + // CHECK-SAME: tensor<2xf32> + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", tpu_core = 0, host_mlir_module = "module {\0A func.func @host_func(%arg0: tensor<*xf32>) -> tensor<*xf32> {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A return %0 : tensor<*xf32> \0A } \0A} \0A"} : (tensor<2xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + func.func @test_xla_select_and_scatter(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // CHECK: %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) <{scatter = @add_scatter, select = @ge_select}> : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor<4x5x1x1xbf16> + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @add_scatter, select = @ge_select} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK: func private @add_scatter(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NEXT: %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor, tensor) -> tensor + // CHECK-NEXT: return %0 : tensor + // CHECK-NEXT: } + // CHECK-NEXT: func private @ge_select(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NEXT: %0 = "tf.GreaterEqual"(%arg0, %arg1) {device = ""} : (tensor, tensor) -> tensor + // CHECK-NEXT: return %0 : tensor + // CHECK-NEXT: } + + func.func private @add_scatter(%arg0: tensor<*xbf16>, %arg1: tensor<*xbf16>) -> tensor<*xbf16> { + %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor<*xbf16>, tensor<*xbf16>) -> tensor<*xbf16> + func.return %0 : tensor<*xbf16> + } + + func.func private @ge_select(%arg0: tensor<*xbf16>, %arg1: tensor<*xbf16>) -> tensor<*xi1> { + %0 = "tf.GreaterEqual"(%arg0, %arg1) {device = ""} : (tensor<*xbf16>, tensor<*xbf16>) -> tensor<*xi1> + func.return %0 : tensor<*xi1> + } + + // Test that ref shapes are also refined but element type retained for + // passthrough ops. + // CHECK-LABEL: func @identity_ref + func.func @identity_ref(%arg0: tensor<312x500x!tf_type.f32ref>) { + // CHECK: (tensor<312x500x!tf_type.f32ref>) -> tensor<312x500xf32> + %15 = "tf.Identity"(%arg0) : (tensor<312x500x!tf_type.f32ref>) -> tensor<*xf32> + func.return + } + + // CHECK-LABEL: func @xla_reduce_window + func.func @xla_reduce_window(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // CHECK: 0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) <{computation = @sum_reducer3}> : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor<10xf32> + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @sum_reducer3} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @xla_gather + // CHECK-SAME: (%arg0: tensor<1x1x9xi32>, %arg1: tensor<1xi32>) -> tensor<1x1x8xi32> + func.func @xla_gather(%arg0: tensor<1x1x9xi32>, %arg1: tensor<1xi32>) -> tensor<*xi32> { + %cst = "tf.Const"() {value = dense<[1, 1, 8]> : tensor<3xi32>} : () -> tensor<3xi32> + %0 = "tf.XlaGather"(%arg0, %arg1, %cst) {dimension_numbers = "\0A\03\00\01\02\1A\01\02", indices_are_sorted = true} : (tensor<1x1x9xi32>, tensor<1xi32>, tensor<3xi32>) -> tensor<*xi32> + func.return %0 : tensor<*xi32> + } + + // CHECK-LABEL: func @xla_gather_with_fold + // CHECK-SAME: (%arg0: tensor<1x1x9xi32>, %arg1: tensor<1xi32>) -> tensor<1x1x8xi32> + func.func @xla_gather_with_fold(%arg0: tensor<1x1x9xi32>, %arg1: tensor<1xi32>) -> tensor<*xi32> { + %cst = "tf.Const"() {value = dense<[1, 1, 8]> : tensor<3xi32>} : () -> tensor<3xi32> + %slice_size = "tf.Identity"(%cst) : (tensor<3xi32>) -> tensor<3xi32> + %0 = "tf.XlaGather"(%arg0, %arg1, %slice_size) {dimension_numbers = "\0A\03\00\01\02\1A\01\02", indices_are_sorted = true} : (tensor<1x1x9xi32>, tensor<1xi32>, tensor<3xi32>) -> tensor<*xi32> + func.return %0 : tensor<*xi32> + } + + // CHECK: func private @sum_reducer3(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NEXT: %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor, tensor) -> tensor + // CHECK-NEXT: return %0 : tensor + // CHECK-NEXT: } + func.func private @sum_reducer3(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + + // CHECK-LABEL: @fill_with_shape_op + func.func @fill_with_shape_op(%arg0: tensor<3x2xi32>, %arg1: tensor) -> (tensor<*xi32>) { + // CHECK: %[[SHAPE:.*]] = "tf.Shape"(%{{.*}}) : (tensor<3x2xi32>) -> tensor<2xi32> + // CHECK: %[[FILL:.*]] = "tf.Fill"(%[[SHAPE]], %{{.*}}) : (tensor<2xi32>, tensor) -> tensor<3x2xi32> + // CHECK: return %[[FILL]] : tensor<3x2xi32> + %0 = "tf.Shape"(%arg0) : (tensor<3x2xi32>) -> tensor<2xi32> + %1 = "tf.Fill"(%0, %arg1) : (tensor<2xi32>, tensor) -> tensor<*xi32> + func.return %1 : tensor<*xi32> + } + + // Verifies that very large splat constants are not materialized as Tensors. + // CHECK-LABEL: @giant_tensor_input + func.func @giant_tensor_input() -> (tensor<*xf32>) { + %input = "tf.Const"() {value = dense<1.000000e+00> : tensor<1024x1024x1024x1024xf32>} : () -> tensor<1024x1024x1024x1024xf32> + %zero = "tf.Const"() {value = dense<0> : tensor<4xi32>} : () -> tensor<4xi32> + %one = "tf.Const"() {value = dense<1> : tensor<4xi32>} : () -> tensor<4xi32> + %0 = "tf.StridedSlice"(%input, %zero, %one, %one) {begin_mask = 15 : i64, device = "", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 0 : i64} : (tensor<1024x1024x1024x1024xf32>, tensor<4xi32>, tensor<4xi32>, tensor<4xi32>) -> tensor<*xf32> + + // CHECK: tensor<1x1x1x1xf32> + func.return %0 : tensor<*xf32> + } + + // Verifies handling of cases involving multiple iteration of feeding inputs. + // CHECK-LABEL: @const_input_required + func.func @const_input_required(%arg0: tensor<10xf64>) -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2,_arg3", outputs = "_retval0"}} { + %cst = "tf.Const"() {value = dense<6> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_0 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %0 = "tf.StridedSlice"(%arg0, %cst_0, %cst, %cst_0) {_XlaHasReferenceVars = false, begin_mask = 0 : i64, device = "/job:localhost/replica:0/task:0/device:XLA_CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 0 : i64} : (tensor<10xf64>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // CHECK-NOT: tensor + func.return %0 : tensor + } + + // CHECK-LABEL: set_dynamic_dimension_size_static_dim + func.func @set_dynamic_dimension_size_static_dim(%input: tensor<4x5xf32>, %size: tensor) -> tensor<*xf32> { + %dimension = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + + // CHECK: (tensor<4x5xf32>, tensor, tensor) -> tensor + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4x5xf32>, tensor, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: set_dynamic_dimension_size_dynamic_dim + func.func @set_dynamic_dimension_size_dynamic_dim(%input: tensor<4x5xf32>, %dimension: tensor, %size: tensor) -> tensor<*xf32> { + + // CHECK: (tensor<4x5xf32>, tensor, tensor) -> tensor + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4x5xf32>, tensor, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: set_dynamic_dimension_size_dynamic_input + func.func @set_dynamic_dimension_size_dynamic_input(%input: tensor<*xf32>, %size: tensor) -> tensor<*xf32> { + %dimension = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + + // CHECK: (tensor<*xf32>, tensor, tensor) -> tensor<*xf32> + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<*xf32>, tensor, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: passthrough_mapdataset + func.func @passthrough_mapdataset( + %arg0 : tensor, + %arg1 : tensor, + %arg2 : tensor, + %arg3 : tensor<10xi32>) { + %74 = "tf.ParallelMapDatasetV2"(%arg0, %arg1, %arg2, %arg2) { + Targuments = [!tf_type.resource, i64], deterministic = "default", + device = "/job:localhost/replica:0/task:0/device:CPU:0", + f = @__inference_Dataset_map_classfunctools.partial_95960, + f._tf_data_function = true, + output_shapes = [#tf_type.shape<64>, #tf_type.shape<69>, #tf_type.shape<73>, #tf_type.shape<12>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<64>, #tf_type.shape<64>, #tf_type.shape<>, #tf_type.shape<>], + output_types = [i32, i32, i32, i64, i32, i32, f32, i64, i32, i32, i32, f32], + preserve_cardinality = false, + use_inter_op_parallelism = true, + metadata = ""} : (tensor, tensor, tensor, tensor) -> tensor<*x!tf_type.variant> + // For this test case the first 12 inputs to the function f below will be + // from ParallelMapDataset, while the remaining one is a captured input. + %75 = "tf.MapDataset"(%74, %arg3) { + device = "/job:localhost/replica:0/task:0/device:CPU:0", + f = @__inference_Dataset_map_lambda_96150, + f._tf_data_function = true, + output_shapes = [#tf_type.shape<64>, #tf_type.shape<69>, #tf_type.shape<10>], + output_types = [i32, i32, i32], + preserve_cardinality = false, + use_inter_op_parallelism = true, + metadata = ""} : (tensor<*x!tf_type.variant>, tensor<10xi32>) -> tensor<*x!tf_type.variant> + func.return + } + + // CHECK: private @__inference_Dataset_map_lambda_96150(%arg0: tensor<64xi32> + // CHECK-SAME: %arg1: tensor<69xi32>, %arg2: tensor<73xi32>, %arg3: tensor<12xi64> + // CHECK-SAME: %arg8: tensor<64xi32>, %arg9: tensor<64xi32> + // CHECK-SAME: %arg12: tensor<10xi32>) + // CHECK-SAME: -> (tensor<64xi32>, tensor<69xi32>, tensor<10xi32>) + func.func private @__inference_Dataset_map_lambda_96150(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, %arg6: tensor, %arg7: tensor, %arg8: tensor, %arg9: tensor, %arg10: tensor, %arg11: tensor, %arg12: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.Identity"(%arg0) {device = ""} : (tensor) -> tensor<*xi32> + %1 = "tf.Identity"(%arg1) {device = ""} : (tensor) -> tensor<*xi32> + %2 = "tf.Identity"(%arg12) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + func.return %0, %1, %2 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32> + } + + // CHECK-LABEL: passthrough_takewhiledataset + func.func @passthrough_takewhiledataset(%arg0 : tensor, + %arg1 : tensor>>, %arg2 : tensor>>, + %arg3 : tensor>>, %arg4 : tensor>>, + %arg5 : tensor>>, %arg6 : tensor>>) { + %cst = arith.constant dense<30> : tensor + %cst_1 = arith.constant dense<-1> : tensor + %cst_2 = arith.constant dense<165000> : tensor + %0 = "tf.TakeDataset"(%arg0, %cst) {device = "", metadata = "", output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string]} : (tensor, tensor) -> tensor + %1 = "tf.TakeWhileDataset"(%0, %arg1, %arg2, %cst_1, %arg3, %arg4, %arg5, %arg6, %cst_2) {device = "", metadata = "", output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], predicate = @__inference_Dataset_take_while_predicate_7400, predicate._tf_data_function = true} : (tensor, tensor>>, tensor>>, tensor, tensor>>, tensor>>, tensor>>, tensor>>, tensor) -> tensor + func.return + } + + // CHECK: @__inference_Dataset_take_while_predicate_7400(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor, %arg4: tensor>>, %arg5: tensor>>, %arg6: tensor>>, %arg7: tensor>>, %arg8: tensor) -> tensor attributes {tf._tf_data_function = true, tf.signature.is_stateful} + func.func private @__inference_Dataset_take_while_predicate_7400(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<*x!tf_type.resource>, %arg3: tensor, %arg4: tensor<*x!tf_type.resource>, %arg5: tensor<*x!tf_type.resource>, %arg6: tensor<*x!tf_type.resource>, %arg7: tensor<*x!tf_type.resource>, %arg8: tensor) -> tensor attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %cst = arith.constant dense : tensor + func.return %cst : tensor + } + + // CHECK-LABEL: passthrough_reducedataset + func.func @passthrough_reducedataset( + %arg0 : tensor>>, + %arg1 : tensor<*x!tf_type.variant>, + %arg2 : tensor>>, + %arg3 : tensor>>, + %arg4 : tensor>>, + %arg5 : tensor>>, + %arg6 : tensor>>, + %arg7 : tensor>>, + %arg8 : tensor>>, + %arg9 : tensor>>, + %arg10 : tensor>>, + %arg11 : tensor>> + ) { + %cst_0 = arith.constant dense<0.1> : tensor + %cst_2 = arith.constant dense<2> : tensor + %cst_12 = arith.constant dense<0> : tensor + %cst_13 = arith.constant dense<> : tensor<0x!tf_type.string> + %51 = "tf.RepeatDataset"(%arg1, %cst_2) {device = "", + output_shapes = [#tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape], + output_types = [!tf_type.string, f32, f32, i32, i32, f32, f32, !tf_type.string, !tf_type.string], + metadata = ""} : (tensor<*x!tf_type.variant>, tensor) -> tensor + + %117:6 = "tf.ReduceDataset"(%51, %cst_12, %cst_12, %cst_12, %cst_12, %cst_13, %cst_13, %arg2, %arg3, %arg4, %arg5, %arg6, %arg6, %arg6, %arg6, %arg7, %arg8, %arg9, %arg10, %cst_0, %arg0, %arg6, %arg11, %arg6, %arg6, %arg6, %arg6, %arg9) { + + // The reduce function __reduce_func has 36 arguments, and the source of those arguments are listed below: + // [arg0, arg5]: these are states coming from ReduceDatasetOp's [arg1, arg6] + // [arg6: arg14]: these are the input elements coming from the output of %51:9 + // [arg15: arg35]: these are the captured elements from ReduceDatasetOp's [arg7:arg27] + + Targuments = [!tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, f32, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource], + Tstate = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], device = "", + f = @__reduce_func, f._tf_data_function = true, + output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape, #tf_type.shape], + output_types = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], use_inter_op_parallelism = true} : (tensor, tensor, tensor, tensor, tensor, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) + func.return + } + + // CHECK: private @__reduce_func( + // CHECK-SAME: %arg15: tensor>> + // CHECK-SAME: %arg16: tensor>> + // CHECK-SAME: %arg17: tensor>> + // CHECK-SAME: %arg18: tensor>> + // CHECK-SAME: %arg19: tensor>> + // CHECK-SAME: %arg20: tensor>> + // CHECK-SAME: %arg21: tensor>> + // CHECK-SAME: %arg22: tensor>> + // CHECK-SAME: %arg23: tensor>> + // CHECK-SAME: %arg24: tensor>> + // CHECK-SAME: %arg25: tensor>> + // CHECK-SAME: %arg26: tensor>> + // CHECK-SAME: %arg28: tensor>> + // CHECK-SAME: %arg29: tensor>> + // CHECK-SAME: %arg30: tensor>> + // CHECK-SAME: %arg31: tensor>> + // CHECK-SAME: %arg32: tensor>> + // CHECK-SAME: %arg33: tensor>> + // CHECK-SAME: %arg34: tensor>> + // CHECK-SAME: %arg35: tensor>>) + func.func private @__reduce_func(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor {tf._user_specified_name = "args_1"}, %arg2: tensor {tf._user_specified_name = "args_2"}, %arg3: tensor {tf._user_specified_name = "args_3"}, %arg4: tensor {tf._user_specified_name = "args_4"}, %arg5: tensor {tf._user_specified_name = "args_5"}, %arg6: tensor {tf._user_specified_name = "args_6"}, %arg7: tensor {tf._user_specified_name = "args_7"}, %arg8: tensor {tf._user_specified_name = "args_8"}, %arg9: tensor {tf._user_specified_name = "args_9"}, %arg10: tensor {tf._user_specified_name = "args_10"}, %arg11: tensor {tf._user_specified_name = "args_11"}, %arg12: tensor {tf._user_specified_name = "args_12"}, %arg13: tensor {tf._user_specified_name = "args_13"}, %arg14: tensor {tf._user_specified_name = "args_14"}, %arg15: tensor>>, %arg16: tensor>>, %arg17: tensor>>, %arg18: tensor>>, %arg19: tensor>>, %arg20: tensor>>, %arg21: tensor>>, %arg22: tensor>>, %arg23: tensor>>, %arg24: tensor>>, %arg25: tensor>>, %arg26: tensor>>, %arg27: tensor {tf._class = ["loc:@md/rnnt_step_decoder/rnnt_decoder/joint/w_b_y/var"]}, %arg28: tensor>>, %arg29: tensor>>, %arg30: tensor>>, %arg31: tensor>>, %arg32: tensor>>, %arg33: tensor>>, %arg34: tensor>>, %arg35: tensor>>) -> (tensor, tensor, tensor, tensor, tensor, tensor) attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %0:6 = "tf.JustPretend"() : () -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5: tensor, tensor, tensor, tensor, tensor, tensor + } + + // CHECK-LABEL: passthrough_reducedataset_scalar_result + func.func @passthrough_reducedataset_scalar_result( + %arg0 : tensor>>, + %arg1 : tensor<*x!tf_type.variant>, + %arg2 : tensor>>, + %arg3 : tensor>>, + %arg4 : tensor>>, + %arg5 : tensor>>, + %arg6 : tensor>>, + %arg7 : tensor>>, + %arg8 : tensor>>, + %arg9 : tensor>>, + %arg10 : tensor>>, + %arg11 : tensor>> + ) { + %cst_0 = arith.constant dense<0.1> : tensor + %cst_2 = arith.constant dense<2> : tensor + %cst_12 = arith.constant dense<0> : tensor + %cst_13 = arith.constant dense<> : tensor<0x!tf_type.string> + %51 = "tf.RepeatDataset"(%arg1, %cst_2) {device = "", + output_shapes = [#tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape], + output_types = [!tf_type.string, f32, f32, i32, i32, f32, f32, !tf_type.string, !tf_type.string], + metadata = ""} : (tensor<*x!tf_type.variant>, tensor) -> tensor + + %117:4 = "tf.ReduceDataset"(%51, %cst_12, %cst_12, %cst_12, %cst_12, %cst_13, %cst_13, %arg2, %arg3, %arg4, %arg5, %arg6, %arg6, %arg6, %arg6, %arg7, %arg8, %arg9, %arg10, %cst_0, %arg0, %arg6, %arg11, %arg6, %arg6, %arg6, %arg6, %arg9) { + + // The reduce function __reduce_func_1 has 36 arguments, and the source of those arguments are listed below: + // [arg0, arg5]: these are states coming from ReduceDatasetOp's [arg1, arg6] + // [arg6: arg14]: these are the input elements coming from the output of %51:9 + // [arg15: arg35]: these are the captured elements from ReduceDatasetOp's [arg7:arg27] + + Targuments = [!tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, f32, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource], + Tstate = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], device = "", + f = @__reduce_func_1, f._tf_data_function = true, + output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>], + output_types = [i32, i32, i32, i32], use_inter_op_parallelism = true} : (tensor, tensor, tensor, tensor, tensor, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>) -> (tensor, tensor, tensor, tensor) + func.return + } + + // CHECK: private @__reduce_func_1( + // CHECK-SAME: %arg8: tensor + // CHECK-SAME: %arg14: tensor<0x!tf_type.string> {tf._user_specified_name = "args_14"} + // CHECK-SAME: %arg15: tensor>> + // CHECK-SAME: %arg16: tensor>> + // CHECK-SAME: %arg17: tensor>> + // CHECK-SAME: %arg18: tensor>> + // CHECK-SAME: %arg19: tensor>> + // CHECK-SAME: %arg20: tensor>> + // CHECK-SAME: %arg21: tensor>> + // CHECK-SAME: %arg22: tensor>> + // CHECK-SAME: %arg23: tensor>> + // CHECK-SAME: %arg24: tensor>> + // CHECK-SAME: %arg25: tensor>> + // CHECK-SAME: %arg26: tensor>> + // CHECK-SAME: %arg28: tensor>> + // CHECK-SAME: %arg29: tensor>> + // CHECK-SAME: %arg30: tensor>> + // CHECK-SAME: %arg31: tensor>> + // CHECK-SAME: %arg32: tensor>> + // CHECK-SAME: %arg33: tensor>> + // CHECK-SAME: %arg34: tensor>> + // CHECK-SAME: %arg35: tensor>>) + func.func private @__reduce_func_1(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor {tf._user_specified_name = "args_1"}, %arg2: tensor {tf._user_specified_name = "args_2"}, %arg3: tensor {tf._user_specified_name = "args_3"}, %arg4: tensor {tf._user_specified_name = "args_4"}, %arg5: tensor {tf._user_specified_name = "args_5"}, %arg6: tensor {tf._user_specified_name = "args_6"}, %arg7: tensor {tf._user_specified_name = "args_7"}, %arg8: tensor {tf._user_specified_name = "args_8"}, %arg9: tensor {tf._user_specified_name = "args_9"}, %arg10: tensor {tf._user_specified_name = "args_10"}, %arg11: tensor {tf._user_specified_name = "args_11"}, %arg12: tensor {tf._user_specified_name = "args_12"}, %arg13: tensor {tf._user_specified_name = "args_13"}, %arg14: tensor {tf._user_specified_name = "args_14"}, %arg15: tensor>>, %arg16: tensor>>, %arg17: tensor>>, %arg18: tensor>>, %arg19: tensor>>, %arg20: tensor>>, %arg21: tensor>>, %arg22: tensor>>, %arg23: tensor>>, %arg24: tensor>>, %arg25: tensor>>, %arg26: tensor>>, %arg27: tensor {tf._class = ["loc:@md/rnnt_step_decoder/rnnt_decoder/joint/w_b_y/var"]}, %arg28: tensor>>, %arg29: tensor>>, %arg30: tensor>>, %arg31: tensor>>, %arg32: tensor>>, %arg33: tensor>>, %arg34: tensor>>, %arg35: tensor>>) -> (tensor, tensor, tensor, tensor) attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %0:6 = "tf.JustPretend"() : () -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0, %0#1, %0#2, %0#3: tensor, tensor, tensor, tensor + } + + // CHECK-LABEL: passthrough_reducedataset_with_identity + func.func @passthrough_reducedataset_with_identity( + %arg0 : tensor>>, + %arg1 : tensor<*x!tf_type.variant>, + %arg2 : tensor>>, + %arg3 : tensor>>, + %arg4 : tensor>>, + %arg5 : tensor>>, + %arg6 : tensor>>, + %arg7 : tensor>>, + %arg8 : tensor>>, + %arg9 : tensor>>, + %arg10 : tensor>>, + %arg11 : tensor>> + ) { + %cst_0 = arith.constant dense<0.1> : tensor + %cst_1 = arith.constant dense<2> : tensor + %cst_2 = arith.constant dense<0> : tensor + %cst_3 = arith.constant dense<> : tensor<0x!tf_type.string> + %0 = "tf.RepeatDataset"(%arg1, %cst_1) {device = "", + output_shapes = [#tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape, #tf_type.shape], + output_types = [!tf_type.string, f32, f32, i32, i32, f32, f32, !tf_type.string, !tf_type.string], + metadata = ""} : (tensor<*x!tf_type.variant>, tensor) -> tensor + + %1 = "tf.Identity"(%0) {device = ""} : (tensor) -> tensor + %2:6 = "tf.ReduceDataset"(%1, %cst_2, %cst_2, %cst_2, %cst_2, %cst_3, %cst_3, %arg2, %arg3, %arg4, %arg5, %arg6, %arg6, %arg6, %arg6, %arg7, %arg8, %arg9, %arg10, %cst_0, %arg0, %arg6, %arg11, %arg6, %arg6, %arg6, %arg6, %arg9) { + + // The reduce function __reduce_func has 36 arguments, and the source of those arguments are listed below: + // [arg0, arg5]: these are states coming from ReduceDatasetOp's [arg1, arg6] + // [arg6: arg14]: these are the input elements coming from the output of %51:9 + // [arg15: arg35]: these are the captured elements from ReduceDatasetOp's [arg7:arg27] + + Targuments = [!tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, f32, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource], + Tstate = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], device = "", + f = @__reduce_func_2, f._tf_data_function = true, + output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape, #tf_type.shape], + output_types = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], use_inter_op_parallelism = true} : (tensor, tensor, tensor, tensor, tensor, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) + func.return + } + + // CHECK: @__reduce_func_2(%arg0: tensor + // CHECK-SAME: %arg15: tensor>> + // CHECK-SAME: %arg17: tensor>> + // CHECK-SAME: %arg18: tensor>> + // CHECK-SAME: %arg19: tensor>> + // CHECK-SAME: %arg20: tensor>> + // CHECK-SAME: %arg21: tensor>> + // CHECK-SAME: %arg22: tensor>> + // CHECK-SAME: %arg23: tensor>> + // CHECK-SAME: %arg24: tensor>> + // CHECK-SAME: %arg25: tensor>> + // CHECK-SAME: %arg26: tensor>> + // CHECK-SAME: %arg28: tensor>> + // CHECK-SAME: %arg29: tensor>> + // CHECK-SAME: %arg31: tensor>> + // CHECK-SAME: %arg32: tensor>> + // CHECK-SAME: %arg33: tensor>> + // CHECK-SAME: %arg34: tensor>> + // CHECK-SAME: %arg35: tensor>>) + func.func private @__reduce_func_2(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor {tf._user_specified_name = "args_1"}, %arg2: tensor {tf._user_specified_name = "args_2"}, %arg3: tensor {tf._user_specified_name = "args_3"}, %arg4: tensor {tf._user_specified_name = "args_4"}, %arg5: tensor {tf._user_specified_name = "args_5"}, %arg6: tensor {tf._user_specified_name = "args_6"}, %arg7: tensor {tf._user_specified_name = "args_7"}, %arg8: tensor {tf._user_specified_name = "args_8"}, %arg9: tensor {tf._user_specified_name = "args_9"}, %arg10: tensor {tf._user_specified_name = "args_10"}, %arg11: tensor {tf._user_specified_name = "args_11"}, %arg12: tensor {tf._user_specified_name = "args_12"}, %arg13: tensor {tf._user_specified_name = "args_13"}, %arg14: tensor {tf._user_specified_name = "args_14"}, %arg15: tensor>>, %arg16: tensor>>, %arg17: tensor>>, %arg18: tensor>>, %arg19: tensor>>, %arg20: tensor>>, %arg21: tensor>>, %arg22: tensor>>, %arg23: tensor>>, %arg24: tensor>>, %arg25: tensor>>, %arg26: tensor>>, %arg27: tensor {tf._class = ["loc:@md/rnnt_step_decoder/rnnt_decoder/joint/w_b_y/var"]}, %arg28: tensor>>, %arg29: tensor>>, %arg30: tensor>>, %arg31: tensor>>, %arg32: tensor>>, %arg33: tensor>>, %arg34: tensor>>, %arg35: tensor>>) -> (tensor, tensor, tensor, tensor, tensor, tensor) attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %0:6 = "tf.JustPretend"() : () -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5: tensor, tensor, tensor, tensor, tensor, tensor + } + + // CHECK-LABEL: passthrough_reducedataset_unspecified_variant + func.func @passthrough_reducedataset_unspecified_variant( + %arg0 : tensor>>, + %arg1 : tensor, + %arg2 : tensor>>, + %arg3 : tensor>>, + %arg4 : tensor>>, + %arg5 : tensor>>, + %arg6 : tensor>>, + %arg7 : tensor>>, + %arg8 : tensor>>, + %arg9 : tensor>>, + %arg10 : tensor>>, + %arg11 : tensor>> + ) { + %cst_0 = arith.constant dense<0.1> : tensor + %cst_2 = arith.constant dense<2> : tensor + %cst_12 = arith.constant dense<0> : tensor + %cst_13 = arith.constant dense<> : tensor<0x!tf_type.string> + + %117:6 = "tf.ReduceDataset"(%arg1, %cst_12, %cst_12, %cst_12, %cst_12, %cst_13, %cst_13, %arg2, %arg3, %arg4, %arg5, %arg6, %arg6, %arg6, %arg6, %arg7, %arg8, %arg9, %arg10, %cst_0, %arg0, %arg6, %arg11, %arg6, %arg6, %arg6, %arg6, %arg9) { + + // The reduce function __reduce_func_3 has 36 arguments, and the source of those arguments are listed below: + // [arg0, arg5]: these are states coming from ReduceDatasetOp's [arg1, arg6] + // [arg6: arg14]: these are the input elements coming from arg1 variant with undefined dataset outputs. + // [arg15: arg35]: these are the captured elements from ReduceDatasetOp's [arg7:arg27] + + Targuments = [!tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, f32, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource, !tf_type.resource], + Tstate = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], device = "", + f = @__reduce_func_3, f._tf_data_function = true, + output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape, #tf_type.shape], + output_types = [i32, i32, i32, i32, !tf_type.string, !tf_type.string], use_inter_op_parallelism = true} : (tensor, tensor, tensor, tensor, tensor, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>, tensor>>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) + func.return + } + + // CHECK: private @__reduce_func_3( + // CHECK-SAME: %arg15: tensor>> + // CHECK-SAME: %arg16: tensor>> + // CHECK-SAME: %arg17: tensor>> + // CHECK-SAME: %arg18: tensor>> + // CHECK-SAME: %arg19: tensor>> + // CHECK-SAME: %arg20: tensor>> + // CHECK-SAME: %arg21: tensor>> + // CHECK-SAME: %arg22: tensor>> + // CHECK-SAME: %arg23: tensor>> + // CHECK-SAME: %arg24: tensor>> + // CHECK-SAME: %arg25: tensor>> + // CHECK-SAME: %arg26: tensor>> + // CHECK-SAME: %arg28: tensor>> + // CHECK-SAME: %arg29: tensor>> + // CHECK-SAME: %arg30: tensor>> + // CHECK-SAME: %arg31: tensor>> + // CHECK-SAME: %arg32: tensor>> + // CHECK-SAME: %arg33: tensor>> + // CHECK-SAME: %arg34: tensor>> + // CHECK-SAME: %arg35: tensor>>) + func.func private @__reduce_func_3(%arg0: tensor {tf._user_specified_name = "args_0"}, %arg1: tensor {tf._user_specified_name = "args_1"}, %arg2: tensor {tf._user_specified_name = "args_2"}, %arg3: tensor {tf._user_specified_name = "args_3"}, %arg4: tensor {tf._user_specified_name = "args_4"}, %arg5: tensor {tf._user_specified_name = "args_5"}, %arg6: tensor {tf._user_specified_name = "args_6"}, %arg7: tensor {tf._user_specified_name = "args_7"}, %arg8: tensor {tf._user_specified_name = "args_8"}, %arg9: tensor {tf._user_specified_name = "args_9"}, %arg10: tensor {tf._user_specified_name = "args_10"}, %arg11: tensor {tf._user_specified_name = "args_11"}, %arg12: tensor {tf._user_specified_name = "args_12"}, %arg13: tensor {tf._user_specified_name = "args_13"}, %arg14: tensor {tf._user_specified_name = "args_14"}, %arg15: tensor>>, %arg16: tensor>>, %arg17: tensor>>, %arg18: tensor>>, %arg19: tensor>>, %arg20: tensor>>, %arg21: tensor>>, %arg22: tensor>>, %arg23: tensor>>, %arg24: tensor>>, %arg25: tensor>>, %arg26: tensor>>, %arg27: tensor {tf._class = ["loc:@md/rnnt_step_decoder/rnnt_decoder/joint/w_b_y/var"]}, %arg28: tensor>>, %arg29: tensor>>, %arg30: tensor>>, %arg31: tensor>>, %arg32: tensor>>, %arg33: tensor>>, %arg34: tensor>>, %arg35: tensor>>) -> (tensor, tensor, tensor, tensor, tensor, tensor) attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %0:6 = "tf.JustPretend"() : () -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5: tensor, tensor, tensor, tensor, tensor, tensor + } + + // CHECK-LABEL: infer_output_type_for_restore + func.func @infer_output_type_for_restore(%arg0: tensor) -> (tensor>>, tensor>>) { + %cst = arith.constant dense<"client/1"> : tensor<1x!tf_type.string> + %cst_0 = arith.constant dense<"train/0"> : tensor<1x!tf_type.string> + %cst_1 = arith.constant dense<""> : tensor<1x!tf_type.string> + %0 = "tf.VarHandleOp"() {container = "", shared_name = "foo"} : () -> tensor>> + // CHECK: (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<192x2680xf32> + %1 = "tf.RestoreV2"(%arg0, %cst, %cst_1) {device = ""} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<*xf32> + "tf.AssignVariableOp"(%0, %1) : (tensor>>, tensor<*xf32>) -> () + %2 = "tf.VarHandleOp"() {container = "", shared_name = "bar"} : () -> tensor>> + // CHECK: (tensor, tensor<1x!tf_type.string>) -> tensor<128xf32> + %3 = "tf.Restore"(%arg0, %cst_0) {device = ""} : (tensor, tensor<1x!tf_type.string>) -> tensor<*xf32> + "tf.AssignVariableOp"(%2, %3) : (tensor>>, tensor<*xf32>) -> () + func.return %0, %2 : tensor>>, tensor>> + } + + // CHECK-LABEL: infer_output_type_for_restore_with_cast + func.func @infer_output_type_for_restore_with_cast(%arg0: tensor) -> (tensor>>, tensor>>) { + %cst = arith.constant dense<"client/1"> : tensor<1x!tf_type.string> + %cst_0 = arith.constant dense<"train/0"> : tensor<1x!tf_type.string> + %cst_1 = arith.constant dense<""> : tensor<1x!tf_type.string> + %0 = "tf.VarHandleOp"() {container = "", shared_name = "foo"} : () -> tensor>> + // CHECK: "tf.RestoreV2"({{.*}}) {device = ""} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<192x2680xf32> + %1 = "tf.RestoreV2"(%arg0, %cst, %cst_1) {device = ""} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<*xf32> + // CHECK: "tf.Cast"({{.*}}) : (tensor<192x2680xf32>) -> tensor<192x2680xbf16> + %2 = "tf.Cast"(%1) : (tensor<*xf32>) -> tensor<*xbf16> + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor<*xbf16>) -> () + %3 = "tf.VarHandleOp"() {container = "", shared_name = "bar"} : () -> tensor>> + // CHECK: "tf.Restore"({{.*}}) {device = ""} : (tensor, tensor<1x!tf_type.string>) -> tensor<128xf32> + %4 = "tf.Restore"(%arg0, %cst_0) {device = ""} : (tensor, tensor<1x!tf_type.string>) -> tensor<*xf32> + %5 = "tf.Identity"(%4) : (tensor<*xf32>) -> tensor<*xf32> + // CHECK: "tf.Cast"({{.*}}) : (tensor<128xf32>) -> tensor<128xbf16> + %6 = "tf.Cast"(%5) : (tensor<*xf32>) -> tensor<*xbf16> + "tf.AssignVariableOp"(%3, %6) : (tensor>>, tensor<*xbf16>) -> () + func.return %0, %3 : tensor>>, tensor>> + } + + // CHECK-LABEL: infer_var_handle_op_from_assign + func.func @infer_var_handle_op_from_assigns() -> tensor<1xi8> { + %cst = arith.constant dense<1> : tensor<1xi8> + %0 = "tf.VarHandleOp"() {container = "", shared_name = "bar"} : () -> tensor>> + // CHECK: "tf.VarHandleOp"() <{container = "", shared_name = "bar"}> : () -> tensor>> + "tf.AssignVariableOp"(%0, %cst) : (tensor>>, tensor<1xi8>) -> () + func.return %cst : tensor<1xi8> + } + + // CHECK-LABEL: infer_var_handle_op_from_read + func.func @infer_var_handle_op_from_read() -> tensor<1xi8> { + %cst = arith.constant dense<1> : tensor<1xi8> + %0 = "tf.VarHandleOp"() {container = "", shared_name = "bar"} : () -> tensor>> + // CHECK: "tf.VarHandleOp"() <{container = "", shared_name = "bar"}> : () -> tensor>> + %read = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<1xi8> + func.return %read : tensor<1xi8> + } + + // CHECK-LABEL: do_not_infer_var_handle_op_when_custom_op_uses_it + func.func @do_not_infer_var_handle_op_when_custom_op_uses_it() -> tensor<1xi8> { + %cst = arith.constant dense<1> : tensor<1xi8> + %0 = "tf.VarHandleOp"() {container = "", shared_name = "bar"} : () -> tensor>> + // CHECK: "tf.VarHandleOp"() <{container = "", shared_name = "bar"}> : () -> tensor>> + %read = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<1xi8> + %1 = "tf.MyCustomOp"(%0) : (tensor>>) -> tensor<4xi8> + func.return %read : tensor<1xi8> + } + + // CHECK-LABEL: testXlaConvV2OpInvalidFeatureGroupCount + func.func @testXlaConvV2OpInvalidFeatureGroupCount(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op feature_group_count must be a positive number, got -1}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2OpInvalidInputTensorShapeDim + func.func @testXlaConvV2OpInvalidInputTensorShapeDim(%lhs: tensor<8x4xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op input tensor argument is 2 which is invalid, since input tensor argument must has a rank greater than 2.}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2OpInvalidKernelTensorShapeDim + func.func @testXlaConvV2OpInvalidKernelTensorShapeDim(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<16x16xf32>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op kernel tensor argument is 2 which is invalid, since kernel tensor argument must has a rank greater than 2.}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2OpInconsistentInputAndKernelTensorShapeDim + func.func @testXlaConvV2OpInconsistentInputAndKernelTensorShapeDim(%lhs: tensor<8x4x16x16xf32>, %rhs: tensor<3x4x4x16x16xf32>) -> (tensor) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op both input tensor and kernel tensor must have same number of dimensions.}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16xf32>, tensor<3x4x4x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2OpInvalidRelationForLhsInputfeaturesAndRhsInputFeatures + func.func @testXlaConvV2OpInvalidRelationForLhsInputfeaturesAndRhsInputFeatures(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<3x4x4x16x16xf32>) -> (tensor) { + %feature_group_count = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op Expected the size of kernel_input_features (value 16) in rhs times feature_group_count (value 2) in lhs should equal the size of the z dimension (value 16) in lhs.}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<3x4x4x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor +} + + // CHECK-LABEL: func @testXlaConvV2OpInvalidRhsOutputfeaturesAndFeatureGroupCounts + func.func @testXlaConvV2OpInvalidRhsOutputfeaturesAndFeatureGroupCounts(%lhs: tensor<8x4x16x16x32xf32>, %rhs: tensor<3x4x4x16x9xf32>) -> (tensor) { + %feature_group_count = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @below {{'tf.XlaConvV2' op Expected output feature dimension (value 9) to be divisible by feature_group_count (value 2).}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x32xf32>, tensor<3x4x4x16x9xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2DynamicInputShape + func.func @testXlaConvV2DynamicInputShape(%lhs: tensor<8x?x?x?x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32> ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x?x?x?x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x?x?x?x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> + } + + // CHECK-LABEL: XlaConvV2DynamicKernelShape + func.func @testXlaConvV2DynamicKernelShape(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor) -> (tensor<8x4x14x14x16xf32> ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x4x16x16x16xf32>, tensor, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> + } + + // CHECK-LABEL: testXlaConvV2UnrankedInputAndOutput + func.func @testXlaConvV2UnrankedInputAndOutput(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2,_arg3,_arg4,_arg5,_arg6", outputs = "_retval0"}} { + %cst = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_0 = "tf.Const"() {value = dense<[[2, 1]]> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_1 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %0 = tf_executor.graph { + // CHECK: "tf.XlaConvV2"(%arg0, %arg1, %cst, %cst_0, %cst_1, %cst_2, %cst_3) <{dimension_numbers = "\18\012\01\02@\01P\01Z\01\02b\01\02", precision_config = "\0A\02\01\01"}> {_XlaHasReferenceVars = false, device = "/job:localhost/replica:0/task:0/device:XLA_CPU:0"} : (tensor<*xf32>, tensor<*xf32>, tensor<1xi32>, tensor<1x2xi32>, tensor<1xi32>, tensor<1xi32>, tensor) -> tensor<*xf32> + %outputs, %control = tf_executor.island wraps "tf.XlaConvV2"(%arg0, %arg1, %cst, %cst_0, %cst_1, %cst_2, %cst_3) {_XlaHasReferenceVars = false, device = "/job:localhost/replica:0/task:0/device:XLA_CPU:0", dimension_numbers = "\18\012\01\02@\01P\01Z\01\02b\01\02", precision_config = "\0A\02\01\01"} : (tensor<*xf32>, tensor<*xf32>, tensor<1xi32>, tensor<1x2xi32>, tensor<1xi32>, tensor<1xi32>, tensor) -> tensor<*xf32> + tf_executor.fetch %outputs : tensor<*xf32> + } + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: testXlaConvV2StaticOutputShape + func.func @testXlaConvV2StaticOutputShape(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32> ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> + } + + // CHECK-LABEL: testXlaConvV2DynamicOutputShape + func.func @testXlaConvV2DynamicOutputShape(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2DynamicOutputElementType + func.func @testXlaConvV2DynamicOutputElementType(%lhs: tensor<8x4x16x16x16xf16>, %rhs: tensor<4x3x3x16x16xf16>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x4x16x16x16xf16>, tensor<4x3x3x16x16xf16>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf16>, tensor<4x3x3x16x16xf16>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testXlaConvV2DynamicOutputShapeInt64Strides + func.func @testXlaConvV2DynamicOutputShapeInt64Strides(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor ) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi64>} : () -> tensor<3xi64> + // CHECK: %0 = "tf.XlaConvV2"(%arg0, %arg1, %cst_3, %cst_2, %cst_1, %cst_0, %cst) <{dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""}> : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi64>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi64>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: testSameOperandsAndResultTypeResolveRefBinary + func.func @testSameOperandsAndResultTypeResolveRefBinary(%lhs: tensor<2x3x?x?xf32>, %rhs: tensor<2x?x5x?xf32>) -> (tensor) { + // CHECK: (tensor<2x3x?x?xf32>, tensor<2x?x5x?xf32>) -> tensor<2x3x5x?xf32> + %0 = "tf.ReluGrad"(%lhs, %rhs) : (tensor<2x3x?x?xf32>, tensor<2x?x5x?xf32>) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @test_xla_sharding + // CHECK-SAME: (%arg0: tensor<1x2x3xf32>) -> tensor<1x2x3xf32> + func.func @test_xla_sharding(%arg0: tensor<1x2x3xf32>) -> tensor<*xf32> { + %0 = "tf.XlaSharding"(%arg0) : (tensor<1x2x3xf32>) -> tensor<*xf32> + return %0 : tensor<*xf32> + } + + // CHECK-LABEL: func @tensor_list_large + func.func @tensor_list_large(%arg0: tensor) { + %0 = "tf.Const"() {device = "", value = dense<2> : tensor<2xi32>} : () -> tensor<2xi32> + %1 = "tf.Const"() {device = "", value = dense<3> : tensor} : () -> tensor + // CHECK: TensorListReserve{{.*}}-> tensor, tensor) -> !tf_variant + + %if0 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%2) : (!tf_variant) -> ()}, + {"tf.Yield"(%2) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if1 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if0) : (!tf_variant) -> ()}, + {"tf.Yield"(%if0) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if2 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if1) : (!tf_variant) -> ()}, + {"tf.Yield"(%if1) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if3 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if2) : (!tf_variant) -> ()}, + {"tf.Yield"(%if2) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if4 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if3) : (!tf_variant) -> ()}, + {"tf.Yield"(%if3) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if5 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if4) : (!tf_variant) -> ()}, + {"tf.Yield"(%if4) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if6 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if5) : (!tf_variant) -> ()}, + {"tf.Yield"(%if5) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if7 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if6) : (!tf_variant) -> ()}, + {"tf.Yield"(%if6) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if8 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if7) : (!tf_variant) -> ()}, + {"tf.Yield"(%if7) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if9 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if8) : (!tf_variant) -> ()}, + {"tf.Yield"(%if8) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if10 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if9) : (!tf_variant) -> ()}, + {"tf.Yield"(%if9) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if11 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if10) : (!tf_variant) -> ()}, + {"tf.Yield"(%if10) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if12 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if11) : (!tf_variant) -> ()}, + {"tf.Yield"(%if11) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if13 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if12) : (!tf_variant) -> ()}, + {"tf.Yield"(%if12) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if14 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if13) : (!tf_variant) -> ()}, + {"tf.Yield"(%if13) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if15 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if14) : (!tf_variant) -> ()}, + {"tf.Yield"(%if14) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if16 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if15) : (!tf_variant) -> ()}, + {"tf.Yield"(%if15) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if17 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if16) : (!tf_variant) -> ()}, + {"tf.Yield"(%if16) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if18 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if17) : (!tf_variant) -> ()}, + {"tf.Yield"(%if17) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if19 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if18) : (!tf_variant) -> ()}, + {"tf.Yield"(%if18) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if20 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if19) : (!tf_variant) -> ()}, + {"tf.Yield"(%if19) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if21 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if20) : (!tf_variant) -> ()}, + {"tf.Yield"(%if20) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if22 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if21) : (!tf_variant) -> ()}, + {"tf.Yield"(%if21) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if23 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if22) : (!tf_variant) -> ()}, + {"tf.Yield"(%if22) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if24 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if23) : (!tf_variant) -> ()}, + {"tf.Yield"(%if23) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if25 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if24) : (!tf_variant) -> ()}, + {"tf.Yield"(%if24) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if26 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if25) : (!tf_variant) -> ()}, + {"tf.Yield"(%if25) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if27 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if26) : (!tf_variant) -> ()}, + {"tf.Yield"(%if26) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if28 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if27) : (!tf_variant) -> ()}, + {"tf.Yield"(%if27) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if29 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if28) : (!tf_variant) -> ()}, + {"tf.Yield"(%if28) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if30 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if29) : (!tf_variant) -> ()}, + {"tf.Yield"(%if29) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if31 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if30) : (!tf_variant) -> ()}, + {"tf.Yield"(%if30) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if32 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if31) : (!tf_variant) -> ()}, + {"tf.Yield"(%if31) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if33 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if32) : (!tf_variant) -> ()}, + {"tf.Yield"(%if32) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if34 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if33) : (!tf_variant) -> ()}, + {"tf.Yield"(%if33) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if35 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if34) : (!tf_variant) -> ()}, + {"tf.Yield"(%if34) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if36 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if35) : (!tf_variant) -> ()}, + {"tf.Yield"(%if35) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if37 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if36) : (!tf_variant) -> ()}, + {"tf.Yield"(%if36) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if38 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if37) : (!tf_variant) -> ()}, + {"tf.Yield"(%if37) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if39 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if38) : (!tf_variant) -> ()}, + {"tf.Yield"(%if38) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if40 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if39) : (!tf_variant) -> ()}, + {"tf.Yield"(%if39) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if41 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if40) : (!tf_variant) -> ()}, + {"tf.Yield"(%if40) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if42 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if41) : (!tf_variant) -> ()}, + {"tf.Yield"(%if41) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if43 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if42) : (!tf_variant) -> ()}, + {"tf.Yield"(%if42) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if44 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if43) : (!tf_variant) -> ()}, + {"tf.Yield"(%if43) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if45 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if44) : (!tf_variant) -> ()}, + {"tf.Yield"(%if44) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if46 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if45) : (!tf_variant) -> ()}, + {"tf.Yield"(%if45) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if47 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if46) : (!tf_variant) -> ()}, + {"tf.Yield"(%if46) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if48 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if47) : (!tf_variant) -> ()}, + {"tf.Yield"(%if47) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + %if49 = "tf.IfRegion"(%arg0) ({"tf.Yield"(%if48) : (!tf_variant) -> ()}, + {"tf.Yield"(%if48) : (!tf_variant) -> ()}) + {is_stateless = true} : (tensor) -> !tf_variant + + %4 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %5 = "tf.Const"() {device = "", value = dense<[[1.000000e+00, 2.000000e+00], [3.000000e+00, 4.000000e+00]]> : tensor<2x2xf32>} : () -> tensor<2x2xf32> + %6 = "tf.TensorListSetItem"(%if49, %4, %5) {device = ""} : (!tf_variant, tensor, tensor<2x2xf32>)-> tensor<*x!tf_type.variant> + func.return + } + + // CHECK-LABEL: func @xla_host_compute_shape_inferred + func.func @xla_host_compute_shape_inferred() -> (tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1120x8xi32>, tensor<2xi32>) { + %2335 = "tf.OpA"() : () -> tensor<3360x8xi32> + %cst_2 = "tf.Const"() <{value = dense<[3, 1120, -1]> : tensor<3xi32>}> : () -> tensor<3xi32> + %cst_68 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %cst_1 = "tf.Const"() <{value = dense<[1120, -1]> : tensor<2xi32>}> : () -> tensor<2xi32> + %2336:4 = "tf.XlaHostCompute"(%2335, %cst_2, %cst_68, %cst_1) <{ancestors = [], cost_estimate_ns = 1000000 : i64, key = "", recv_key = "host_compute_channel_0_retvals", send_key = "host_compute_channel_0_args", shape_inference_graph = @host_func_0, shapes = [#tf_type.shape<1x1120x8>, #tf_type.shape<1x1120x8>, #tf_type.shape<1120x8>, #tf_type.shape<2>], tpu_core = 0 : i64}> {_xla_original_oc_node_name = "Shape_4", _xla_token_input_nodes = ["_xla_token_arg_node"]} : (tensor<3360x8xi32>, tensor<3xi32>, tensor, tensor<2xi32>) -> (tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1120x8xi32>, tensor<2xi32>) + func.return %2336#0, %2336#1, %2336#2, %2336#3: tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1120x8xi32>, tensor<2xi32> + } + + func.func @host_func_0(%arg0: tensor<3360x?xi32>, %arg1: tensor<3xi32>, %arg2: tensor, %arg3: tensor<2xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>) { + // CHECK: %cst = "tf.Const"() <{value = dense<[3, 1120, -1]> : tensor<3xi32>}> : () -> tensor<3xi32> + // CHECK-NEXT: %cst_0 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %cst_1 = "tf.Const"() <{value = dense<[1120, -1]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK: %0 = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + // CHECK: %1:4 = "tf._XlaRecvAtHost"(%0) <{device_ordinal = 0 : i64, device_type = "TPU", key = "host_compute_channel_0_args"}> : (tensor<3x!tf_type.string>) -> (tensor<3360x8xi32>, tensor<3xi32>, tensor, tensor<2xi32>) + // CHECK: %2 = "tf.Reshape"(%1#0, %cst) {_xla_outside_compilation = "0"} : (tensor<3360x8xi32>, tensor<3xi32>) -> tensor<3x1120x8xi32> + // CHECK: %3:3 = "tf.Split"(%cst_0, %2) {_xla_outside_compilation = "0"} : (tensor, tensor<3x1120x8xi32>) -> (tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1x1120x8xi32>) + // CHECK: %4 = "tf.Reshape"(%3#0, %cst_1) {_xla_outside_compilation = "0"} : (tensor<1x1120x8xi32>, tensor<2xi32>) -> tensor<1120x8xi32> + // CHECK: %5 = "tf.Shape"(%4) {_xla_outside_compilation = "0"} : (tensor<1120x8xi32>) -> tensor<2xi32> + // CHECK: "tf._XlaSendFromHost"(%3#1, %3#2, %4, %5, %0) <{device_ordinal = 0 : i64, device_type = "TPU", key = "host_compute_channel_0_retvals"}> : (tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1120x8xi32>, tensor<2xi32>, tensor<3x!tf_type.string>) -> () + // CHECK: return %3#1, %3#2, %4, %5 : tensor<1x1120x8xi32>, tensor<1x1120x8xi32>, tensor<1120x8xi32>, tensor<2xi32> + %0 = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + %1:4 = "tf._XlaRecvAtHost"(%0) <{device_ordinal = 0 : i64, device_type = "TPU", key = "host_compute_channel_0_args"}> : (tensor<3x!tf_type.string>) -> (tensor<3360x8xi32>, tensor<3xi32>, tensor, tensor<2xi32>) + %2 = "tf.Reshape"(%1#0, %1#1) {_xla_outside_compilation = "0"} : (tensor<3360x8xi32>, tensor<3xi32>) -> tensor<3x1120x?xi32> + %3:3 = "tf.Split"(%1#2, %2) {_xla_outside_compilation = "0"} : (tensor, tensor<3x1120x?xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1x1120x?xi32>) + %4 = "tf.Reshape"(%3#0, %1#3) {_xla_outside_compilation = "0"} : (tensor<1x1120x?xi32>, tensor<2xi32>) -> tensor<1120x?xi32> + %5 = "tf.Shape"(%4) {_xla_outside_compilation = "0"} : (tensor<1120x?xi32>) -> tensor<2xi32> + "tf._XlaSendFromHost"(%3#1, %3#2, %4, %5, %0) <{device_ordinal = 0 : i64, device_type = "TPU", key = "host_compute_channel_0_retvals"}> : (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>, tensor<3x!tf_type.string>) -> () + return %3#1, %3#2, %4, %5 : tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32> + } + + // CHCK-LABEL: func @infer_return_type_static_out + func.func @infer_return_type_static_out(%arg0: tensor, %arg1: tensor) -> tensor<1x28x28x3xf32> { + %0 = "tf.Conv2D"(%arg0, %arg1) {data_format = "NHWC", device = "", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true}: (tensor, tensor) -> tensor<1x28x28x3xf32> + func.return %0 : tensor<1x28x28x3xf32> + } + + // CHCK: %0 = "tf.Conv2D"(%arg0, %arg1) <{data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true}> {device = ""} : (tensor, tensor) -> tensor<1x28x28x3xf32> + + +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_for_xla_call_module.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_for_xla_call_module.mlir new file mode 100644 index 0000000000000000000000000000000000000000..98ce48c44915a9b73773d7efc987549723ea06ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_for_xla_call_module.mlir @@ -0,0 +1,72 @@ +// RUN: tf-opt %s -tf-xla-call-module-serialization -tf-shape-inference='enable-stablehlo-propagation=true' -tf-xla-call-module-deserialization | FileCheck %s --check-prefixes=COMMON,ENABLED +// RUN: tf-opt %s -tf-xla-call-module-serialization -tf-shape-inference='enable-stablehlo-propagation=false' -tf-xla-call-module-deserialization | FileCheck %s --check-prefixes=COMMON,DISABLED +// RUN: tf-opt %s -tf-xla-call-module-serialization -tf-standard-pipeline='enable-stablehlo-shape-propagation=true' -tf-xla-call-module-deserialization | FileCheck %s --check-prefixes=COMMON,ENABLED +// RUN: tf-opt %s -tf-xla-call-module-serialization -tf-standard-pipeline='enable-stablehlo-shape-propagation=false' -tf-xla-call-module-deserialization | FileCheck %s --check-prefixes=COMMON,DISABLED + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 130 : i32}} { + + // COMMON-LABEL: func @module_attr_updated_with_stablehlo_shape_refinement + // COMMON-SAME: (%arg0: tensor<4xf32>) -> tensor<4xi32> + func.func @module_attr_updated_with_stablehlo_shape_refinement(%arg0: tensor<4xf32>) -> tensor { + // COMMON: "tf.XlaCallModule" + // COMMON-SAME: _entry_function = @main, + // COMMON-SAME: (tensor<4xf32>) -> tensor<4xi32> + %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], _entry_function = @_stablehlo_main, _stablehlo_module_attrs = { jax.uses_shape_polymorphism = true }, _stablehlo_version = "1.1.0", module = "", platforms = ["cpu"], version = 9 : i64} : (tensor<4xf32>) -> tensor + func.return %0 : tensor + } + + // COMMON-LABEL: func @module_attr_not_updated_without_stablehlo_output_shape_refinement + // COMMON-SAME: (%arg0: tensor<1xf32>) -> tensor<1xi32> + func.func @module_attr_not_updated_without_stablehlo_output_shape_refinement(%arg0: tensor<1xf32>) -> tensor<*xi32> { + // COMMON: "tf.XlaCallModule" + // COMMON-SAME: _entry_function = @main_0, + // COMMON-SAME: (tensor<1xf32>) -> tensor<1xi32> + %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], _entry_function = @_stablehlo_main_0, _stablehlo_module_attrs = { jax.uses_shape_polymorphism = true }, _stablehlo_version = "1.1.0", module = "", platforms = ["cpu"], version = 9 : i64} : (tensor<1xf32>) -> tensor<*xi32> + func.return %0 : tensor<*xi32> + } + + // COMMON-LABEL: func @xla_call_module_shape_refinement_failure_ok + func.func @xla_call_module_shape_refinement_failure_ok(%arg0: tensor) -> tensor<*xf32> attributes {tf._original_func_name = "main_0"} { + %cst = "tf.Const"() <{value = dense<1.000000e+00> : tensor<1024x3xf32>}> : () -> tensor<1024x3xf32> + // COMMON: tf.XlaCallModule + // COMMON-SAME: _entry_function = @main_1, + // COMMON-SAME: (tensor, tensor<1024x3xf32>) -> tensor + %0 = "tf.XlaCallModule"(%arg0, %cst) {Sout = [#tf_type.shape], dim_args_spec = [], _entry_function = @_stablehlo_main_1, _stablehlo_module_attrs = { jax.uses_shape_polymorphism = true }, _stablehlo_version = "1.1.0", module = "", platforms = ["cpu", "tpu"], version = 9 : i64} : (tensor, tensor<1024x3xf32>) -> tensor<*xf32> + // COMMON: return %0 : tensor + return %0 : tensor<*xf32> + } + + // ENABLED-LABEL: func.func private @main({{.*}}: tensor<4xf32>) -> tensor<4xi32> + // ENABLED: stablehlo.convert {{.*}} : (tensor<4xf32>) -> tensor<4xi32> + // ENABLED: return {{.*}} : tensor<4xi32> + // DISABLED-LABEL: func.func private @main({{.*}}: tensor<4xf32>) -> tensor + // DISABLED: stablehlo.convert {{.*}} : (tensor<4xf32>) -> tensor + // DISABLED: return {{.*}} : tensor + func.func private @_stablehlo_main(%arg0 : tensor<4xf32>) -> tensor attributes {_from_xla_call_module} { + %0 = stablehlo.convert %arg0 : (tensor<4xf32>) -> tensor + return %0 : tensor + } + + // COMMON-LABEL: func.func private @main_0({{.*}}: tensor<1xf32>) -> tensor<1xi32> + // COMMON: stablehlo.convert {{.*}} : (tensor<1xf32>) -> tensor + // COMMON: return {{.*}} : tensor<1xi32> + func.func private @_stablehlo_main_0(%arg0 : tensor<1xf32>) -> tensor<1xi32> attributes {_from_xla_call_module} { + %0 = stablehlo.convert %arg0 : (tensor<1xf32>) -> tensor + %1 = stablehlo.constant dense<1> : tensor<1xi32> + return %1 : tensor<1xi32> + } + + // COMMON-LABEL: func.func private @main_1({{.*}}: tensor, {{.*}}: tensor, {{.*}}: tensor<1024x3xf32>) -> tensor + // COMMON: return {{.*}} : tensor + func.func private @_stablehlo_main_1(%arg0: tensor, %arg1: tensor, %arg2: tensor<1024x3xf32>) -> tensor attributes {_from_xla_call_module} { + %0 = stablehlo.constant dense<0> : tensor + %1 = stablehlo.get_dimension_size %arg1, dim = 0 : (tensor) -> tensor + %2 = stablehlo.compare EQ, %0, %1 : (tensor, tensor) -> tensor + stablehlo.custom_call @shape_assertion(%2) {error_message = "Shape assertion failed", has_side_effect = true} : (tensor) -> () + %3 = stablehlo.dot_general %arg1, %arg2, + batching_dims = [] x [], contracting_dims = [1] x [0] + {mhlo.frontend_attributes = {grad_x = "false", grad_y = "false"}} + : (tensor, tensor<1024x3xf32>) -> tensor + return %3 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_with_shape_specialization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_with_shape_specialization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c9df64dfcb210eae179007a0cc7c49150e8dbbcf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/shape_inference_with_shape_specialization.mlir @@ -0,0 +1,42 @@ +// RUN: tf-opt %s -tf-shape-inference=input-arg-shapes=1 -verify-diagnostics -split-input-file | FileCheck %s +// RUN: not tf-opt %s -tf-shape-inference=input-arg-shapes=* 2>&1 | FileCheck --check-prefix=INPUT_ARG_SHAPES_ERROR %s +// INPUT_ARG_SHAPES_ERROR: Missing input argument shapes + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func.func @main + // CHECK-DAG: %[[CST_0:.*]] = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor + // CHECK-DAG: %[[CST_1:.*]] = "tf.Const"() <{value = dense<3> : tensor}> : () -> tensor + // CHECK-NEXT: %[[UQ:.*]] = "tf.UniformQuantize"(%arg0, %cst, %cst_0) <{quantization_axis = -1 : i64, quantization_max_val = 127 : i64, quantization_min_val = -128 : i64}> : (tensor<1xf32>, tensor, tensor) -> tensor<1x!tf_type.qint8> + // CHECK-NEXT: %[[UDQ:.*]] = "tf.UniformDequantize"(%[[UQ]], %[[CST_0]], %[[CST_1]]) <{quantization_axis = -1 : i64, quantization_max_val = 127 : i64, quantization_min_val = -128 : i64}> : (tensor<1x!tf_type.qint8>, tensor, tensor) -> tensor<1xf32> + // CHECK-NEXT: return %[[UDQ]] : tensor<1xf32> + func.func @main(%arg0 : tensor) -> tensor { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + %1 = "tf.UniformDequantize"(%0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + func.return %1 : tensor + } +} + +// ----- + +// expected-error@+1 {{Input shapes provided but no `main` function found.}} +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @non_main(%arg0 : tensor) -> tensor { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + %1 = "tf.UniformDequantize"(%0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + func.return %1 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c10c0e50e5b5bcb07502b13ce5c8fc86c1dd40a1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir @@ -0,0 +1,3046 @@ +// RUN: tf-opt -split-input-file -tf-test-side-effect-analysis -verify-diagnostics %s | FileCheck %s + +// Tests that the pass tracks control dependencies for reads/writes on the same +// resource. + +// CHECK-LABEL: func @non_aliasing_reads_writes +func.func @non_aliasing_reads_writes( +// expected-remark@above {{ID: 13}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + // expected-remark@above {{ID: 11}} + // CHECK: tf_executor.island + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Successors: {10}}} + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf.AssignVariableOp"(%arg0, %arg2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {6}}} + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {5}}} + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 3}} + %read2 = "tf.ReadVariableOp"(%var_handle) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {8}}} + "tf.AssignVariableOp"(%arg1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {2}}} + // expected-remark@above {{Successors: {8}}} + "tf.AssignVariableOp"(%arg0, %read2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {7}}} + %read3 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {6}}} + // expected-remark@above {{Successors: {8}}} + tf_executor.yield %read3 : tensor<32xf32> + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {4,5,7}}} + } + tf_executor.fetch %island#0 : tensor<32xf32> + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Predecessors: {9}}} + } + func.return %graph : tensor<32xf32> + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Sinks: {11}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for reads/writes on the two +// resource handles that refer to the same variable. + +// CHECK-LABEL: func @aliasing_reads_writes +func.func @aliasing_reads_writes(%arg0: tensor<32xf32>) -> () { +// expected-remark@above {{ID: 14}} + tf_executor.graph { + // expected-remark@above {{ID: 12}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Successors: {11}}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 0}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 1}} + %vh1_id:2 = "tf.IdentityN"(%vh1, %arg0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> (tensor<*x!tf_type.resource>>, tensor<32xf32>) + // expected-remark@above {{ID: 2}} + %read0 = "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf.AssignVariableOp"(%vh1_id#0, %arg0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + // expected-remark@above {{Successors: {5,6}}} + %read1 = "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Successors: {7}}} + %read2 = "tf.ReadVariableOp"(%vh1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Successors: {7}}} + "tf.AssignVariableOp"(%vh0, %read2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {5,6}}} + // expected-remark@above {{Successors: {8}}} + "tf.AssignVariableOp"(%vh1_id#0, %read1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {7}}} + // expected-remark@above {{Successors: {9}}} + tf_executor.yield + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {8}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Predecessors: {10}}} + } + func.return + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Sinks: {12}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for side-effecting on unknown +// resources. + +// CHECK-LABEL: func @unknown_side_effecting_op +func.func @unknown_side_effecting_op(%arg0: tensor<32xf32>) -> () { +// expected-remark@above {{ID: 14}} + tf_executor.graph { + // expected-remark@above {{ID: 12}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Successors: {11}}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 0}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v1"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 1}} + %read0 = "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {4}}} + "tf.AssignVariableOp"(%vh1, %arg0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {2,3}}} + // expected-remark@above {{Successors: {5,6,7}}} + %read1 = "tf.ReadVariableOp"(%vh1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Successors: {8}}} + %read2 = "tf.ReadVariableOp"(%vh1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Successors: {8}}} + "tf.AssignVariableOp"(%vh0, %read1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Successors: {9}}} + "tf.AssignVariableOp"(%vh1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {5,6}}} + // expected-remark@above {{Successors: {9}}} + tf_executor.yield + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {7,8}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Predecessors: {10}}} + } + func.return + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Sinks: {12}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for read-only ops on unknown +// resources. + +// CHECK-LABEL: func @read_only_unknown_resource +func.func @read_only_unknown_resource(%arg0: tensor<32xf32>) -> () { +// expected-remark@above {{ID: 10}} + tf_executor.graph { + // expected-remark@above {{ID: 8}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Successors: {7}}} + %vh0 = "tf._UnknownSideEffectingOp_"() : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2,3}}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v1"} : () -> tensor<*x!tf_type.resource>> + // expected-remark@above {{ID: 1}} + %read0 = "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {4}}} + %read1 = "tf.ReadVariableOp"(%vh1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {4}}} + "tf.AssignVariableOp"(%vh1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {2,3}}} + // expected-remark@above {{Successors: {5}}} + tf_executor.yield + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {6}}} + } + func.return + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Sinks: {8}}} +} + +// ----- + +// Tests that the pass adds control dependencies in nested regions with +// tf_device.replicate + +func.func @with_replicate( + // expected-remark@above {{ID: 12}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>, + %arg3: tensor<*x!tf_type.resource>>) { + tf_executor.graph { + // expected-remark@above {{ID: 10}} + %island = tf_executor.island { + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Successors: {9}}} + %u0:2 = "tf._UnknownSideEffectingOp_"() : () -> (tensor<32xf32>, tensor<32xf32>) + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {5}}} + tf_device.replicate( + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {6}}} + [%arg0, %arg1] as %r0: tensor<*x!tf_type.resource>>, + [%arg2, %arg3] as %r1: tensor<*x!tf_type.resource>>, + [%u0#0, %u0#1] as %u : tensor<32xf32>) + {n = 2 : i32, devices = {CORE_0 = ["/CPU:0", "/GPU:1"]}} { + %read0 = "tf.ReadVariableOp"(%r0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 1}} + "tf.AssignVariableOp"(%r1, %u) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + %read1 = "tf.ReadVariableOp"(%r1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + tf_device.return + // expected-remark@above {{ID: 4}} + } + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + // expected-remark@above {{Successors: {7}}} + tf_executor.yield + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {6}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {8}}} + } + func.return + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Sinks: {10}}} +} + +// ----- + +// Tests that the pass does not add control dependencies for a stateless if op. + +// CHECK-LABEL: func @stateless_if_op +func.func @stateless_if_op( + // expected-remark@above {{ID: 8}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %if = "tf.If"(%arg1, %arg1) { + // expected-remark@above {{ID: 1}} + then_branch = @if_then, else_branch = @if_else, is_stateless = true} + : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// CHECK-LABEL: func @if_then +func.func @if_then(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %graph = tf_executor.graph { + // expected-remark@above {{ID: 3}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 1}} + tf_executor.yield %arg0 : tensor + // expected-remark@above {{ID: 0}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 2}} + } + func.return %graph : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {}}} +} + +// CHECK-LABEL: func @if_else +func.func @if_else(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %graph = tf_executor.graph { + // expected-remark@above {{ID: 3}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 1}} + tf_executor.yield %arg0 : tensor + // expected-remark@above {{ID: 0}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 2}} + } + func.return %graph : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that the pass does not add control dependencies for a stateless +// IfRegion op. + +// CHECK-LABEL: func @stateless_ifregion_op +func.func @stateless_ifregion_op( + // expected-remark@above {{ID: 18}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 16}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 14}} + // expected-remark@above {{Successors: {15}}} + + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {12}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + + %if = "tf.IfRegion"(%arg1) ( + // expected-remark@above {{ID: 11}} + { // Then region. + %graph = tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 2}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 3}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 5}} + }, { // Else region + %graph = tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 6}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 8}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 10}} + } + ) { is_stateless = true} : (tensor) -> tensor + + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {13}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + + tf_executor.yield + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Predecessors: {12}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Predecessors: {14}}} + } + func.return + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Sinks: {16}}} +} + +// ----- + +// Tests that the pass does not add control dependencies a stateless while op. + +// CHECK-LABEL: func @stateless_if_op +func.func @stateless_if_op( + // expected-remark@above {{ID: 8}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %while = "tf.While"(%arg1) { + // expected-remark@above {{ID: 1}} + body = @while_body, cond = @while_cond, is_stateless = true} + : (tensor) -> tensor + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// CHECK-LABEL: func @while_body +func.func @while_body(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %graph = tf_executor.graph { + // expected-remark@above {{ID: 3}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 1}} + tf_executor.yield %arg0 : tensor + // expected-remark@above {{ID: 0}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 2}} + } + func.return %graph : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {}}} +} + +// CHECK-LABEL: func @while_cond +func.func @while_cond(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %graph = tf_executor.graph { + // expected-remark@above {{ID: 3}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 1}} + tf_executor.yield %arg0 : tensor + // expected-remark@above {{ID: 0}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 2}} + } + func.return %graph : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that the pass does not add control dependencies a stateless WhileRegion +// op. + +// CHECK-LABEL: func @stateless_whileregion_op +func.func @stateless_whileregion_op( + // expected-remark@above {{ID: 18}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 16}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 14}} + // expected-remark@above {{Successors: {15}}} + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {12}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + + %while = "tf.WhileRegion"(%arg1) ( + // expected-remark@above {{ID: 11}} + { + ^bb0(%carg: tensor): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 2}} + tf_executor.yield %carg : tensor + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 3}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 5}} + }, { + ^bb0(%barg: tensor): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + tf_executor.yield %barg : tensor + // expected-remark@above {{ID: 6}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 8}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 10}} + } + ) {is_stateless = true} : (tensor) -> tensor + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {13}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Predecessors: {12}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Predecessors: {14}}} + } + func.return + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Sinks: {16}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for variables from an if op's +// output. + +// In this test, the resources computed and used are as follows: +// (* = unknown resource id which aliases with everything else) +// id0 = arg0 +// if-then-branch: [u0, arg0, arg0] +// if-else-branch: [arg0, arg0, arg1] +// => first result is unknown, second and third is passthrough +// if results : [*, arg0, {arg0, arg1}[ +// ID #2: read (unknown) -> succ {5, 6) +// ID #3: read (arg0) -> succ {5} +// ID #4: read({arg0,arg1}) -> succ {5,6} +// ID #5: write(arg0) +// ID #6: write(arg1) + +// CHECK-LABEL: func @output_of_if_op +func.func @output_of_if_op( + // expected-remark@above {{ID: 12}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 10}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Successors: {9}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %if:3 = "tf.If"(%arg2, %id0, %arg1) { + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2,3,4}}} + then_branch = @if_then, else_branch = @if_else, is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) + %r0 = "tf.ReadVariableOp"(%if#0) : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5,6}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%if#1) : + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%if#2) : + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5,6}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {2,3,4}}} + // expected-remark@above {{Successors: {7}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {2,4}}} + // expected-remark@above {{Successors: {7}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {5,6}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {8}}} + } + func.return + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Sinks: {10}}} +} + +// CHECK-LABEL: func @if_then +func.func @if_then( + // expected-remark@above {{ID: 7}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>) -> + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + -> tensor<*x!tf_type.resource>> + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 1}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %u0, %id0, %id0 : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + func.return %graph#0, %graph#1, %graph#2 : + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> +} + +// CHECK-LABEL: func @if_else +func.func @if_else( + // expected-remark@above {{ID: 7}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>) -> + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 3}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %id1 = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 1}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %id0, %id0, %id1 : + // expected-remark@above {{ID: 2}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 4}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + func.return %graph#0, %graph#1, %graph#2 : + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> +} + +// ----- + +// Tests that the pass tracks control dependencies for variables from an +// IfRegion op's output. + +// CHECK-LABEL: func @output_of_ifregion_op +func.func @output_of_ifregion_op( + // expected-remark@above {{ID: 26}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 24}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 22}} + // expected-remark@above {{Successors: {23}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %if:3 = "tf.IfRegion"(%arg2) ( + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Successors: {16,17,18}}} + { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + -> tensor<*x!tf_type.resource>> + %iid0 = "tf.Identity"(%id0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 2}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %u0, %iid0, %iid0 : + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2) : + // expected-remark@above {{ID: 7}} + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> () + }, + { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 13}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 11}} + %iid0 = "tf.Identity"(%id0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 8}} + -> tensor<*x!tf_type.resource>> + %iid1 = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 9}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %iid0, %iid0, %iid1 : + // expected-remark@above {{ID: 10}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 12}} + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2) : + // expected-remark@above {{ID: 14}} + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> () + }) { is_stateless = false} + : (tensor) -> + (tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) + %r0 = "tf.ReadVariableOp"(%if#0) : + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%if#1) : + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%if#2) : + // expected-remark@above {{ID: 18}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 19}} + // expected-remark@above {{Predecessors: {16,17,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 20}} + // expected-remark@above {{Predecessors: {16,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 21}} + // expected-remark@above {{Predecessors: {19,20}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 23}} + // expected-remark@above {{Predecessors: {22}}} + } + func.return + // expected-remark@above {{ID: 25}} + // expected-remark@above {{Sinks: {24}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for variables from a while +// op's output. + +// Here: +// id0 = arg0 +// while-inputs = (id0/arg0, arg1, arg1) +// while body pass through first and second arg, not last one +// while-results = (arg0, arg1, Unknown) +// #ID 2: read(arg0) -> succ{5} +// #ID 3: read(arg1) -> succ{6} +// #ID 4: read(unknown) -> succ{5,6} +// #ID 5 : write(arg0) +// #ID 6 : write(arg1) + + +// CHECK-LABEL: func @output_of_while_op +func.func @output_of_while_op( + // expected-remark@above {{ID: 12}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 10}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Successors: {9}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %while:4 = "tf.While"(%arg2, %id0, %arg1, %arg1) { + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2,3,4}}} + body = @while_body, cond = @while_cond, is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> + (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) + %r0 = "tf.ReadVariableOp"(%while#1) : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%while#2) : + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {6}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%while#3) : + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5,6}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {2,4}}} + // expected-remark@above {{Successors: {7}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {3,4}}} + // expected-remark@above {{Successors: {7}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {5,6}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {8}}} + } + func.return + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Sinks: {10}}} +} + +// CHECK-LABEL: func @while_body +func.func @while_body( + // expected-remark@above {{ID: 7}} + %pred: tensor, + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>) -> + (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) { + %graph:4 = tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island:5 = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %pred, %id0, %arg1, %u0 : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2, %island#3 : + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + func.return %graph#0, %graph#1, %graph#2, %graph#3 : + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} + tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> +} + +// CHECK-LABEL: func @while_cond +func.func @while_cond( + // expected-remark@above {{ID: 7}} + %pred: tensor, + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>) -> tensor { + %graph = tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 3}} + %const = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // expected-remark@above {{ID: 0}} + %eq = "tf.Equal"(%pred, %const) : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + tf_executor.yield %eq : tensor + // expected-remark@above {{ID: 2}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 4}} + } + func.return %graph : tensor + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for variables from a +// WhileRegion op's output. + +// CHECK-LABEL: func @output_of_whileregion_op +func.func @output_of_whileregion_op( + // expected-remark@above {{ID: 26}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 24}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 22}} + // expected-remark@above {{Successors: {23}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf_type.resource>> + %while:4 = "tf.WhileRegion"(%arg2, %id0, %arg1, %arg1) ( + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Successors: {16,17,18}}} + { + ^bb0(%pred: tensor, + %carg1: tensor<*x!tf_type.resource>>, + %carg2: tensor<*x!tf_type.resource>>, + %carg3: tensor<*x!tf_type.resource>>): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 4}} + %const = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // expected-remark@above {{ID: 1}} + %eq = "tf.Equal"(%pred, %const) : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + tf_executor.yield %eq : tensor + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 5}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 7}} + }, + { + ^bb0(%pred: tensor, + %barg0: tensor<*x!tf_type.resource>>, + %barg1: tensor<*x!tf_type.resource>>, + %barg2: tensor<*x!tf_type.resource>>): + %graph:4 = tf_executor.graph { + // expected-remark@above {{ID: 13}} + %island:5 = tf_executor.island { + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Successors: {12}}} + %iid0 = "tf.Identity"(%barg0) : (tensor<*x!tf_type.resource>>) + // expected-remark@above {{ID: 8}} + -> tensor<*x!tf_type.resource>> + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Successors: {10}}} + -> tensor<*x!tf_type.resource>> + tf_executor.yield %pred, %iid0, %barg1, %u0 : + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Predecessors: {9}}} + tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2, %island#3 : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {11}}} + tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2, %graph#3) : + // expected-remark@above {{ID: 14}} + (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> () + } + ) {is_stateless = false} + : (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) -> + (tensor, tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>, + tensor<*x!tf_type.resource>>) + %r0 = "tf.ReadVariableOp"(%while#1) : + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%while#2) : + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {20}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%while#3) : + // expected-remark@above {{ID: 18}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 19}} + // expected-remark@above {{Predecessors: {16,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 20}} + // expected-remark@above {{Predecessors: {17,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 21}} + // expected-remark@above {{Predecessors: {19,20}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 23}} + // expected-remark@above {{Predecessors: {22}}} + } + func.return + // expected-remark@above {{ID: 25}} + // expected-remark@above {{Sinks: {24}}} +} + +// ----- + +// Tests that the pass tracks control dependencies based on TF op registry +// statefulness flag, for ops not yet defined in ODS. + +// CHECK-LABEL: func @tf_registry_ops +func.func @tf_registry_ops( + // expected-remark@above {{ID: 8}} + %arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf.PrintV2"(%arg0) { output_stream = "stderr", end = "\n" } + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + : (tensor) -> () + %merge_summary = "tf.MergeSummary"(%arg0, %arg1) { N = 2 } + // expected-remark@above {{ID: 1}} + : (tensor, tensor) -> (tensor) + "tf.PrintV2"(%merge_summary) { output_stream = "stderr", end = "\n" } + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + : (tensor) -> () + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that the pass tracks control dependencies for resource arguments with +// aliasing table (unique IDs). + +// CHECK-LABEL: func @arguments_with_unique_ids +func.func @arguments_with_unique_ids( + // expected-remark@above {{ID: 9}} + %arg0: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %arg1: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %arg2: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 33 : i64}) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {3}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%arg1) : + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%arg2) : + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {4}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {0,1}}} + // expected-remark@above {{Successors: {4}}} + (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {2,3}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests interplay of value-based side-effects for non-resource values and +// unknown side effects. +func.func @value_based_side_effect_non_resource_to_unknown( + // expected-remark@above {{ID: 8}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf._InternalTestNonResourceValueSideEffects_"(%arg0) : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests interplay of value-based side-effects for non-resource values and +// other known side effects. +func.func @value_based_side_effect_non_resource_to_known( + // expected-remark@above {{ID: 8}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %0 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf._InternalTestNonResourceValueSideEffects_"(%arg0) : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + %1 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1,2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that the analysis correctly handles a sequence of ops using the +// `DatasetIterator` resource. +func.func @dataset_op_sequence( + // expected-remark@above {{ID: 9}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %handle, %deleter = "tf.AnonymousIteratorV2"() {_class = ["loc:@GeneratorDataset_2"], device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string]} : () -> (tensor, tensor) + // expected-remark@above {{ID: 0}} + "tf.MakeIterator"(%arg0, %handle) {_class = ["loc:@GeneratorDataset_2"], device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + %0 = "tf.IteratorGetNext"(%handle) {_class = ["loc:@GeneratorDataset_2"], device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {3}}} + "tf.DeleteIterator"(%handle, %deleter) {device = ""} : (tensor, tensor) -> () + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests `tf.GeneratorDataset` with surrounding ops with unknown side-effects. +func.func @generator_dataset_with_unknown_side_effect_ops( + // expected-remark@above {{ID: 8}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %0 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that two resources allocated with identical non-empty `shared_name` +// attributes are dependent. That means, `%handle1` and `%handle2` point to +// the same resources and the second `InitializeTableV2` op depends on the +// first one. +func.func @resources_allocated_with_same_nonempty_shared_name( + // expected-remark@above {{ID: 9}} + %key: tensor, + %value: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %handle1 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "some_name", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + // expected-remark@above {{ID: 0}} + %handle2 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "some_name", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + // expected-remark@above {{ID: 1}} + "tf.InitializeTableV2"(%handle1, %key, %value) : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + "tf.InitializeTableV2"(%handle2, %key, %value) : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests two side-effecting ops operating on resources passed as function +// parameters. The expectation is that the ops are treated as independent (as +// no `tf._resource_arg_unique_id` attributes are present). +func.func @side_effecting_ops_with_different_resources( + // expected-remark@above {{ID: 7}} + %arg0: tensor, + %arg1: tensor, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.StackPushV2"(%arg0, %arg2) {device = "", swap_memory = false} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %1 = "tf.StackPushV2"(%arg1, %arg2) {device = "", swap_memory = false} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests two side-effecting ops operating on resources that are allocated in the +// same function. The expectation is that the ops are treated as independent +// (as the involved resource allocators have the `UniqueResourceAllocation` +// trait). +func.func @side_effecting_ops_with_different_resources_and_allocations( + // expected-remark@above {{ID: 9}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %stack_handle1 = "tf.StackV2"(%arg0) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + %stack_handle2 = "tf.StackV2"(%arg0) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + %0 = "tf.StackPushV2"(%stack_handle1, %arg1) {device = "", swap_memory = false} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {4}}} + %1 = "tf.StackPushV2"(%stack_handle2, %arg1) {device = "", swap_memory = false} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {2,3}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests that we create a dependency for op instances with +// `TPUEmbeddingSideEffect` with same device ordinal. +func.func @embedding_effect_same_device( + // expected-remark@above {{ID: 7}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we treat different op instances with `TPUEmbeddingSideEffect` as +// independent if they have different device ordinals. +func.func @embedding_effect_different_devices( + // expected-remark@above {{ID: 7}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 2} : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we treat different op instances with `TPUEmbeddingSideEffect` as +// independent if they have different devices (even if device ordinals are +// equal). +func.func @embedding_effect_same_ordinal_different_devices( + // expected-remark@above {{ID: 7}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1, device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1, device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"} : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we create dependencies between ops with `TPUEmbeddingSideEffect` +// and unknown side-effecting ops. +func.func @mixed_embedding_and_unknown_effects( + // expected-remark@above {{ID: 8}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2]} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2]} : (tensor) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we don't create dependencies between ops `EnqueueTPUEmbedding` +// ops and other embedding ops that don't have a device ordinal. +func.func @mixed_embedding_and_unknown_effects( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor<8xf32>, + %arg2: tensor<8xf32>) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {3}}} + "tf.LoadTPUEmbeddingAdagradParameters"(%arg1, %arg2) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table1"} : (tensor<8xf32>, tensor<8xf32>) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2], device_ordinal = 2} : (tensor) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {0,1,2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we create a dependency between two ops with the same op-based +// write effect. +func.func @same_op_based_write_effect( + // expected-remark@above {{ID: 7}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we treat ops with different op-based side effects as independent. +func.func @different_op_based_side_effects( + // expected-remark@above {{ID: 8}} + %arg0: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {3}}} + %0 = "tf.GeneratorDataset"(%arg0, %arg0, %arg0) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2], device_ordinal = 5} : (tensor) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {0,1,2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we don't create dependencies between ops with different op-based +// and value-based side effects. +!tf_res = tensor<*x!tf_type.resource>> +func.func @mixed_op_based_value_based_side_effects( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: !tf_res, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf.AssignVariableOp"(%arg1, %arg2) : (!tf_res, tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0){table_ids = [1, 2]} : (tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + "tf.ReadVariableOp"(%arg1) : (!tf_res) -> tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1,2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we create dependencies between `_XlaRecvAtHostV2` ops with equal +// keys. +!tf_res = tensor<*x!tf_type.resource>> +func.func @recv_equal_keys( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %0 = "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %const = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + // expected-remark@above {{ID: 1}} + %1 = "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we create dependencies between `_XlaSendFromHostV2` ops with equal +// keys. +!tf_res = tensor<*x!tf_type.resource>> +func.func @send_equal_keys( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %const = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + // expected-remark@above {{ID: 0}} + "tf._XlaSendFromHostV2"(%const, %arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_retvals"} : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + "tf._XlaSendFromHostV2"(%const, %arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_retvals"} : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we don't create dependencies between `_XlaRecvAtHostV2` ops with +// different keys (corresponding to different resources). +!tf_res = tensor<*x!tf_type.resource>> +func.func @recv_different_keys( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %0 = "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + %const = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + // expected-remark@above {{ID: 1}} + %1 = "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_1_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} +// ----- + +// Tests that we don't create dependencies between `_XlaSendFromHostV2` ops with +// different keys (corresponding to different resources). +!tf_res = tensor<*x!tf_type.resource>> +func.func @send_different_keys( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %const = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + // expected-remark@above {{ID: 0}} + "tf._XlaSendFromHostV2"(%const, %arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_0_retvals"} : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 1}} + "tf._XlaSendFromHostV2"(%const, %arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_1_retvals"} : (tensor, tensor, tensor) -> () + // expected-remark@above {{ID: 2}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we create a dependency between ops with `TF_TPUExecuteSideEffect`. +func.func @tpu_execute_effect( + // expected-remark@above {{ID: 7}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf.TPUExecute"(%arg0, %arg0) : (tensor, tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf.TPUExecute"(%arg1, %arg1) : (tensor, tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between TPU compile ops. +func.func @tpu_compile_ops( + // expected-remark@above {{ID: 7}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + %0:2 = "tf._TPUCompileMlir"() { metadata = "...", mlir_module = "..." } : () -> (tensor, tensor) + // expected-remark@above {{ID: 0}} + %1:2 = "tf._TPUCompileMlir"() { metadata = "...", mlir_module = "..." } : () -> (tensor, tensor) + // expected-remark@above {{ID: 1}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that `_TPUDeviceOrdinalPlaceholder` is side-effect-free. +func.func @device_ordinal_placeholder_side_effect_free( + // expected-remark@above {{ID: 7}} + ) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf._TPUDeviceOrdinalPlaceholder"() {logical_core = 0} : () -> tensor + // expected-remark@above {{ID: 0}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies from or to ops with `TF_MustExecute` +// trait. +func.func @must_execute_ops( + // expected-remark@above {{ID: 7}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf._InternalTestMustExecuteTrait_"() : () -> () + // expected-remark@above {{ID: 1}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between ops with resources that only +// have self-dependency (like `_XlaRecvAtHostV2`) to or from ops with unknown +// resources. +func.func @no_unknown_side_effect_dependencies( + // expected-remark@above {{ID: 8}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_1_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + "tf._UnknownSideEffectingOp_"() : () -> () + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + func.return + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Sinks: {6}}} +} + +// ----- + +// Tests that we create dependencies from ops with resources that only +// have self-dependency (the island containing `_XlaRecvAtHostV2`) to `Fetch`. +func.func @fetch_dependencies( + // expected-remark@above {{ID: 6}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island = tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_1_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + tf_executor.yield + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + func.return + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Sinks: {4}}} +} + +// ----- + +// Tests that we don't create dependencies between islands with one stateless op +// each. +func.func @single_stateless_op_islands() { + // expected-remark@above {{ID: 9}} + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island1 = tf_executor.island { + // expected-remark@above {{ID: 2}} + "tf.A"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 0}} + tf_executor.yield + // expected-remark@above {{ID: 1}} + } + %island2 = tf_executor.island { + // expected-remark@above {{ID: 5}} + "tf.B"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 3}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + } + tf_executor.fetch %island1, %island2 : !tf_executor.control, !tf_executor.control + // expected-remark@above {{ID: 6}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we create dependencies between islands with one stateful op each. +func.func @single_stateful_op_islands() { + // expected-remark@above {{ID: 9}} + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island1 = tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {5}}} + "tf.A"() {is_stateless=false} : () -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + tf_executor.yield + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + } + %island2 = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {2}}} + // expected-remark@above {{Successors: {6}}} + "tf.B"() {is_stateless=false} : () -> () + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + tf_executor.fetch %island1, %island2 : !tf_executor.control, !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests that we don't create dependencies between islands with multiple +// stateless ops each. +func.func @multi_stateless_op_islands() { + // expected-remark@above {{ID: 11}} + tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island1 = tf_executor.island { + // expected-remark@above {{ID: 3}} + "tf.A"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 0}} + "tf.B"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 1}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + } + %island2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + "tf.C"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 4}} + "tf.D"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 5}} + tf_executor.yield + // expected-remark@above {{ID: 6}} + } + tf_executor.fetch %island1, %island2 : !tf_executor.control, !tf_executor.control + // expected-remark@above {{ID: 8}} + } + func.return + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we create dependencies between islands that write to the same +// resource variable and also have different op-based effects. +func.func @nontrivial_multi_op_islands( + // expected-remark@above {{ID: 11}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<32xf32>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island1 = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {7,8}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg2) {table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + "tf.AssignVariableOp"(%arg0, %arg1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + %island2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Successors: {8}}} + // expected-remark@above {{Predecessors: {3}}} + %0 = "tf.GeneratorDataset"(%arg2, %arg2, %arg2) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {6}}} + "tf.AssignVariableOp"(%arg0, %arg1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + tf_executor.yield + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {4,5}}} + } + tf_executor.fetch %island1, %island2 : !tf_executor.control, !tf_executor.control + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {3,7}}} + } + func.return + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Sinks: {9}}} +} + +// ----- + +// Tests that we create dependencies between `CollectiveReduceV2` ops if no +// ordering tokens are present (TF_CollectiveReduceOrderingEffect). +func.func @collective_reduce_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor, + %group_key: tensor, + %group_size: tensor, + %instance_key: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Mul", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between `CollectiveReduceV2` ops if +// the `ordered` attribute is false +func.func @collective_reduce_not_ordered( +// expected-remark@above {{ID: 7}} + %input: tensor, + %group_key: tensor, + %group_size: tensor, + %instance_key: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id", is_stateless = true} : (tensor, tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Mul", final_op = "Id", is_stateless = true} : (tensor, tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we don't create dependencies between `CollectiveReduceV2` ops if +// ordering tokens are present and independent. +func.func @collective_reduce_independent_ordering_tokens( + // expected-remark@above {{ID: 7}} + %arg0: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %arg1: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 1 : i64}, + %input: tensor, + %group_key: tensor, + %group_size: tensor, + %instance_key: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key, %arg0) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor, tensor<*x!tf_type.resource>>) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key, %arg1) {merge_op = "Mul", final_op = "Id"} : (tensor, tensor, tensor, tensor, tensor<*x!tf_type.resource>>) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we do create dependencies between `CollectiveReduceV2` ops if +// ordering tokens are present and dependent. +func.func @collective_reduce_dependent_ordering_tokens( + // expected-remark@above {{ID: 7}} + %arg0: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %arg1: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %input: tensor, + %group_key: tensor, + %group_size: tensor, + %instance_key: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key, %arg0) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor, tensor<*x!tf_type.resource>>) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key, %arg1) {merge_op = "Mul", final_op = "Id"} : (tensor, tensor, tensor, tensor, tensor<*x!tf_type.resource>>) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + // expected-remark@above {{Predecessors: {0}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between `CollectiveReduceV2` ops if +// one has ordering tokens and the other one doesn't. +func.func @collective_reduce_dependent_ordering_tokens( + // expected-remark@above {{ID: 7}} + %arg0: tensor<*x!tf_type.resource>> {tf._resource_arg_unique_id = 0 : i64}, + %input: tensor, + %group_key: tensor, + %group_size: tensor, + %instance_key: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key, %arg0) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor, tensor<*x!tf_type.resource>>) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Mul", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between device launch ops with +// multiple stateless ops each. +func.func @multi_stateless_op_launches() { + // expected-remark@above {{ID: 13}} + tf_executor.graph { + // expected-remark@above {{ID: 11}} + %island = tf_executor.island { + // expected-remark@above {{ID: 9}} + "tf_device.launch"() ({ + // expected-remark@above {{ID: 3}} + "tf.A"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 0}} + "tf.B"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 1}} + tf_device.return + // expected-remark@above {{ID: 2}} + }) {device = "CPU:0"} : () -> () + "tf_device.launch"() ({ + // expected-remark@above {{ID: 7}} + "tf.C"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 4}} + "tf.D"() {is_stateless=true} : () -> () + // expected-remark@above {{ID: 5}} + tf_device.return + // expected-remark@above {{ID: 6}} + }) {device = "CPU:0"} : () -> () + tf_executor.yield + // expected-remark@above {{ID: 8}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 10}} + } + func.return + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// CHECK-LABEL: func @if_region_with_resource_var +func.func @if_region_with_resource_var( + // expected-remark@above {{ID: 19}} + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 17}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Successors: {16}}} + + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {12}}} + (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + + %if = "tf.IfRegion"(%arg1) ( + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Successors: {14}}} + // expected-remark@above {{Predecessors: {0}}} + { // Then region. + %graph = tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 2}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 3}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 5}} + }, { // Else region + %graph = tf_executor.graph { + // expected-remark@above {{ID: 10}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Successors: {9}}} + "tf.AssignVariableOp"(%arg0, %r0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Successors: {7}}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Predecessors: {6}}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Predecessors: {8}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 11}} + } + ) { is_stateless = false} : (tensor) -> tensor + + %0 = "tf.GeneratorDataset"(%arg2, %arg2, %arg2) {device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", finalize_func = @__func_a, init_func = @__func_b, next_func = @__func_c, next_func.experimental_ints_on_device = true, operandSegmentSizes = array, output_shapes = [#tf_type.shape<>], output_types = [!tf_type.string], metadata = ""} : (tensor, tensor, tensor) -> tensor + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Successors: {14}}} + tf_executor.yield + // expected-remark@above {{ID: 14}} + // expected-remark@above {{Predecessors: {12,13}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Predecessors: {15}}} + } + func.return + // expected-remark@above {{ID: 18}} + // expected-remark@above {{Sinks: {17}}} +} + +// ----- + +// Tests that we create dependencies from ops with resources that only have +// self-dependency (like `_XlaRecvAtHostV2`) to `fetch`, even if there is an op +// with unknown side effects in between. See b/253049649. +func.func @self_dependent_only_feeds_into_fetch( + // expected-remark@above {{ID: 9}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {6}}} + "tf._XlaRecvAtHostV2"(%arg0, %arg1) {_xla_has_host_transfer = true, key = "host_compute_channel_1_args"} : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 0}} + tf_executor.yield + // expected-remark@above {{ID: 1}} + } + tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + "tf._UnknownSideEffectingOp_"() {is_stateless=false} : () -> () + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + tf_executor.fetch + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {2,5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests that we create dependencies between `NcclAllReduce` ops on same device. +func.func @collective_reduce_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name", device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name", device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we create dependencies between `NcclAllReduce` ops for unspecified +// devices. +func.func @collective_reduce_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between `NcclAllReduce` ops on +// different devices. +func.func @collective_reduce_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name", device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %1 = "tf.NcclAllReduce"(%input) { reduction = "min", num_devices = 2, shared_name = "name", device = "CPU:1"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we create dependencies to a fetch op, even if the fetch op has +// known effects (in this case due to resource operand) and the resources of +// other side-effecting ops are independent. +func.func @fetch_with_resource_operand( + // expected-remark@above {{ID: 9}} + %arg0: tensor, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>) { + tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island1 = tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {6}}} + "tf.EnqueueTPUEmbeddingRaggedTensorBatch"(%arg0) {table_ids = [1, 2], device_ordinal = 1} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + tf_executor.yield + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + } + %island2 = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %read = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + tf_executor.fetch %arg2, %island1, %island2 : tensor<*x!tf_type.resource>>, !tf_executor.control, !tf_executor.control + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {2,5}}} + } + func.return + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests about we create the dependency PwStreamResults `start` and `end` +// within in XlaCallModule +func.func @_pws_program(%arg0: tensor {tf_saved_model.index_path = ["arg0"]}) -> (tensor {tf_saved_model.index_path = ["result0"]}, tensor {tf_saved_model.index_path = ["result1"]}) attributes {pws.program_id = 4722582128360897113 : i64, tf.entry_function = {}} { + // expected-remark@above {{ID: 7}} + "tf.PwStreamResults"(%arg0) {_callback_id = -2694175233261920887 : i64, _controller_address = "[2002:afb:afb::]:10004", _has_manual_control_dependencies = true, _model_name = "test", device = "/device/CPU", names = ["begin"]} : (tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {4}}} + %0:2 = "tf_device.cluster"() ({ + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %1 = "tf.XlaSharding"(%arg0) {_XlaSharding = "", sharding = "", unspecified_dims = []} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + %2:2 = "tf.XlaCallModule"(%1) {Sout = [#tf_type.shape<>, #tf_type.shape<>], dim_args_spec = [], function_list = [@__inference_callable_flat_tf_150], module = "ML\EFR\00__inference_callable_flat_tf_15\00", platforms = [], version = 5 : i64} : (tensor) -> (tensor, tensor) + // expected-remark@above {{ID: 2}} + tf_device.return %2#0, %2#1 : tensor, tensor + // expected-remark@above {{ID: 3}} + }) {_tpu_replicate = "cluster_0", allow_soft_placement = false, computation_shape = [], device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> (tensor, tensor) + "tf.PwStreamResults"(%arg0) {_callback_id = -2694175233261920887 : i64, _controller_address = "[2002:afb:afb::]:10004", _model_name = "test", device = "/device/CPU", names = ["end"]} : (tensor) -> () + // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{ID: 5}} + return %0#0, %0#1 : tensor, tensor + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} +// expected-remark@below {{ID: 2}} +func.func private @__inference_callable_flat_tf_150(%arg0: tensor {tf._user_specified_name = "args_tf_flat_0"}, %arg1: tensor {tf._user_specified_name = "args_tf_flat_1"}) attributes {tf._XlaMustCompile = false, tf._construction_context = "kEagerRuntime", tf._input_shapes = [#tf_type.shape<>, #tf_type.shape<>], tf._original_func_name = "__inference_callable_flat_tf_15", tf.signature.is_stateful} { + "tf.PwStreamResults"(%arg0, %arg1) {_callback_id = -2694175233261920887 : i64, _controller_address = "[2002:afb:afb::]:10004", _model_name = "test", names = ["foo", "bar"]} : (tensor, tensor) -> () + // expected-remark@above {{ID: 0}} + return + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Sinks: {0}}} +} + +// ----- + +// Tests that we create dependencies between `XlaLaunch` ops on same device. +func.func @xla_launch_ordering_effect ( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +func.func @func(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 1}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we create dependencies between `XlaLaunch` ops for unspecified +// devices. +func.func @xla_launch_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +func.func @func(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 1}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we don't create dependencies between `XlaLaunch` ops on +// different devices. +func.func @xla_launch_ordering_effect( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {2}}} + %1 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:1"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {0,1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +func.func @func(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 1}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +// Tests that we create dependencies between `XlaLaunch` ops on different +// devices that access the same resource. +func.func @xla_launch_ordering_effect ( + // expected-remark@above {{ID: 7}} + %input: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + %0 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + %1 = "tf.XlaLaunch"(%input) {function = @func, operandSegmentSizes = array, device = "CPU:0"} : (tensor) -> tensor + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +func.func @func(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 1}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +func.func @add(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor<1xf32> { + // expected-remark@above {{ID: 2}} + %sum = "tf.Add"(%arg0, %arg1) : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + func.return %sum : tensor<1xf32> + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Sinks: {}}} +} + +// CHECK-LABEL: func @call_pure_function +func.func @call_pure_function(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %one = "tf.Const"() { value = dense<1.0> : tensor<1xf32> } : () -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + %r1 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 1}} + %two = "tf.StatefulPartitionedCall"(%one, %one) {config="", config_proto="", executor_type="", f=@add} : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 2}} + %r2 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 3}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {1,3}}} +} + +// ----- + +// Tests that we create a dependency between ops with `TF__XlaRunSideEffect`. +func.func @tpu_execute_effect( + // expected-remark@above {{ID: 7}} + %arg0: tensor, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + "tf._XlaRun"(%arg0, %arg0) : (tensor, tensor) -> () + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {1}}} + "tf._XlaRun"(%arg1, %arg1) : (tensor, tensor) -> () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {2}}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Predecessors: {1}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {3}}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {5}}} +} + +// ----- + +// Tests that we don't create dependencies between any two `RandomUniform` ops. +func.func @random_uniform_ordering_effect() -> (tensor<3xf32>) { + // expected-remark@above {{ID: 9}} + %graph = tf_executor.graph { + // expected-remark@above {{ID: 7}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Successors: {6}}} + %0 = arith.constant dense<[3]> : tensor<1xi32> + // expected-remark@above {{ID: 0}} + %1 = "tf.RandomUniform"(%0) {device = "", seed = 3 : i64, seed2 = 5 : i64} : (tensor<1xi32>) -> tensor<3xf32> + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {4}}} + %2 = "tf.RandomUniform"(%0) {device = "", seed = 3 : i64, seed2 = 5 : i64} : (tensor<1xi32>) -> tensor<3xf32> + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {4}}} + %3 = "tf.RandomUniform"(%0) {device = "CPU:0", seed = 3 : i64, seed2 = 5 : i64} : (tensor<1xi32>) -> tensor<3xf32> + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Successors: {4}}} + tf_executor.yield %3: tensor<3xf32> + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {1,2,3}}} + } + tf_executor.fetch %island#0 : tensor<3xf32> + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Predecessors: {5}}} + } + func.return %graph : tensor<3xf32> + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Sinks: {7}}} +} + +// ----- + +// Tests that we don't create dependencies between `GlobalIterId`s. +func.func @global_iter_id_effect() -> () { + // expected-remark@above {{ID: 7}} + tf_executor.graph { + // expected-remark@above {{ID: 5}} + %island = tf_executor.island { + // expected-remark@above {{ID: 3}} + %0 = "tf.GlobalIterId"() : () -> tensor + // expected-remark@above {{ID: 0}} + %1 = "tf.GlobalIterId"() : () -> tensor + // expected-remark@above {{ID: 1}} + tf_executor.yield + // expected-remark@above {{ID: 2}} + } + tf_executor.fetch + // expected-remark@above {{ID: 4}} + } + func.return + // expected-remark@above {{ID: 6}} + // expected-remark@above {{Sinks: {}}} +} + +// ----- + +func.func @add(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor<1xf32> { + // expected-remark@above {{ID: 2}} + %sum = "tf.Add"(%arg0, %arg1) : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + func.return %sum : tensor<1xf32> + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Sinks: {}}} +} + +func.func @intermediary(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor<1xf32> { + // expected-remark@above {{ID: 2}} + %result = "tf.StatefulPartitionedCall"(%arg0, %arg1) {config="", config_proto="", executor_type="", f=@add} : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + func.return %result : tensor<1xf32> + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Sinks: {}}} +} + +// CHECK-LABEL: func @call_pure_function +func.func @call_pure_function(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %one = "tf.Const"() { value = dense<1.0> : tensor<1xf32> } : () -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + %r1 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 1}} + %two = "tf.StatefulPartitionedCall"(%one, %one) {config="", config_proto="", executor_type="", f=@intermediary} : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 2}} + %r2 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 3}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {1,3}}} +} + +// ----- + +func.func @assert(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor { + // expected-remark@above {{ID: 3}} + %cond = builtin.unrealized_conversion_cast to tensor + // expected-remark@above {{ID: 0}} + "tf.Assert"(%cond, %arg1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", summarize = 3 : i64} : (tensor, tensor<1xf32>) -> () + // expected-remark@above {{ID: 1}} + func.return %cond : tensor + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Sinks: {1}}} +} + +func.func @intermediary(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor<1xf32> { + // expected-remark@above {{ID: 3}} + %cond = builtin.unrealized_conversion_cast to tensor + // expected-remark@above {{ID: 0}} + %sum = "tf.If"(%cond, %arg0, %arg1) { + then_branch = @assert, + else_branch = @assert, + is_stateless = false + } : (tensor, tensor<1xf32>, tensor<1xf32>) -> tensor + // expected-remark@-5 {{ID: 1}} + func.return %arg0 : tensor<1xf32> + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Sinks: {1}}} +} + +// CHECK-LABEL: func @assert_within_if +func.func @assert_within_if(%arg0: tensor) -> tensor { + // expected-remark@above {{ID: 5}} + %one = "tf.Const"() { value = dense<1.0> : tensor<1xf32> } : () -> tensor<1xf32> + // expected-remark@above {{ID: 0}} + %r1 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 1}} + %result = "tf.StatefulPartitionedCall"(%one, %one) {config="", config_proto="", executor_type="", f=@intermediary} : (tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32> + // expected-remark@above {{ID: 2}} + %r2 = "tf.ReadVariableOp"(%arg0) : (tensor) -> tensor<1xf32> + // expected-remark@above {{ID: 3}} + func.return %arg0 : tensor + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Sinks: {1,3}}} +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/sink_constant.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/sink_constant.mlir new file mode 100644 index 0000000000000000000000000000000000000000..067b48ccd1dc43824d32383834138dcf009c31c4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/sink_constant.mlir @@ -0,0 +1,41 @@ +// RUN: tf-opt %s -tf-device-constant-sinking | FileCheck %s + +// CHECK-LABEL: func @sink_const +func.func @sink_const(%arg0 : tensor<16xf32>) -> (tensor<16xf32>, tensor) { + // Verify that the constant are sunk in the tf_device.cluster region using them + // and removed if no other use is left. + + // Only the 2.0 and 3.0 constants are removed, the 4.0 has a use in the return + // CHECK-NOT:"tf.Const"2.0 + // CHECK-NOT:"tf.Const"3.0 + %0 = "tf.Const"() {value = dense<2.000000e+00> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense<3.000000e+00> : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<4.000000e+00> : tensor} : () -> tensor + %3 = tf_executor.graph { + %res, %ctl = tf_executor.island { + %3 = "tf_device.cluster"() ({ + + // In the device region, check that the 3 constants are materialized and + // remapped to the uses. + // CHECK: tf_device.cluster + // CHECK-DAG: %[[CST2:.*]] = "tf.Const"{{.*}}2.0 + // CHECK-DAG: %[[CST3:.*]] = "tf.Const"{{.*}}3.0 + // CHECK-DAG: %[[CST4:.*]] = "tf.Const"{{.*}}4.0 + // CHECK-NOT:"tf.Const" + // CHECK: %[[MUL1:.*]] = "tf.Mul"(%arg0, %[[CST2]]) + // CHECK-NEXT: %[[MUL2:.*]] = "tf.Mul"(%[[MUL1]], %[[CST2]]) + // CHECK-NEXT: %[[MUL3:.*]] = "tf.Mul"(%[[MUL2]], %[[CST3]]) + // CHECK-NEXT: = "tf.Mul"(%[[MUL3]], %[[CST4]]) + %3 = "tf.Mul"(%arg0, %0) : (tensor<16xf32>, tensor) -> tensor<16xf32> + %4 = "tf.Mul"(%3, %0) : (tensor<16xf32>, tensor) -> tensor<16xf32> + %5 = "tf.Mul"(%4, %1) : (tensor<16xf32>, tensor) -> tensor<16xf32> + %6 = "tf.Mul"(%5, %2) : (tensor<16xf32>, tensor) -> tensor<16xf32> + tf_device.return %6 : tensor<16xf32> + }) {} : () -> tensor<16xf32> + tf_executor.yield %3 : tensor<16xf32> + } + tf_executor.fetch %res : tensor<16xf32> + } + func.return %3, %2 : tensor<16xf32>, tensor +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/split_into_island_per_op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/split_into_island_per_op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..daea1a41e0343c52c247ae37d6393eb671bd81d6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/split_into_island_per_op.mlir @@ -0,0 +1,435 @@ +// RUN: tf-opt -tf-executor-split-into-island-per-op %s --split-input-file --verify-diagnostics | FileCheck %s + +// All tests also test for idempotence. + +// Test that functions must have a single graph op in their body, otherwise, the +// pass will signal failure. + +// expected-error@+1 {{expected function to contain only a graph_op}} +func.func @multiple_func_body_ops() { + tf_executor.graph { + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + tf_executor.graph { + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Test that functions' graph op must have a single island op and a single fetch +// op, otherwise, the pass will signal failure. + +func.func @graph_multiple_islands() { + // expected-error@+1 {{expected graph op to contain only a single island_op and a single fetch_op}} + tf_executor.graph { + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Test that V1 control flow is not allowed in this pass. Otherwise, if found, +// the pass should signal failure. + +func.func @switch_and_merge(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor) { + // expected-error@below {{expected graph op to contain only a single island_op and a single fetch_op}} + %graph:2 = tf_executor.graph { + %island0:3 = tf_executor.island { + %add = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %less = "tf.Less"(%arg1, %arg1) : (tensor, tensor) -> tensor + %res = "tf.Print"(%add) { message = "add result 1" } : (tensor<*xi32>) -> (tensor<*xi32>) + tf_executor.yield %add, %less : tensor<*xi32>, tensor + } + %switch:3 = tf_executor.Switch %island0#0, %island0#1 : tensor<*xi32> + %island1:2 = tf_executor.island { + %add = "tf.Add"(%switch#0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %res = "tf.Print"(%add) { message = "add result 2" } : (tensor<*xi32>) -> (tensor<*xi32>) + tf_executor.yield %add : tensor<*xi32> + } + %merge_out:3 = tf_executor.Merge %island1#0, %switch#1 : tensor<*xi32> + tf_executor.fetch %merge_out#0, %merge_out#1 : tensor<*xi32>, tensor + } + func.return %graph#0, %graph#1 : tensor<*xi32>, tensor +} + +// ----- + +// Test that external functions aren't processed (used to crash). + +// CHECK-LABEL: func private @unused_external_func +func.func private @unused_external_func() + +func.func @multiple_return(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %island:3 = tf_executor.island { + %add1 = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2 = "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + tf_executor.yield %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + tf_executor.fetch %island#0, %island#1 : tensor<*xi32>, tensor<*xi32> + } + func.return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @multiple_return +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 +// CHECK: } + +func.func @dangling_print(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %island1:3 = tf_executor.island { + %add1 = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2 = "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %res = "tf.Print"(%add2) { message = "add result" } : (tensor<*xi32>) -> (tensor<*xi32>) + tf_executor.yield %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + tf_executor.fetch %island1#0, %island1#1 : tensor<*xi32>, tensor<*xi32> + } + func.return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @dangling_print +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: %[[PRINT:.*]], %[[PRINT_control:.*]] = tf_executor.island wraps "tf.Print"(%[[ADD2]]) <{message = "add result"}> +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 + +func.func @drop_fetch_control_dep(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %island1:3 = tf_executor.island { + %add1 = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2 = "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + tf_executor.yield %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + tf_executor.fetch %island1#0, %island1#1, %island1#2 : tensor<*xi32>, tensor<*xi32>, !tf_executor.control + } + func.return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @drop_fetch_control_dep +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 + +func.func @fetching_arg(%arg0: tensor<*xi32>) { + tf_executor.graph { + %island:3 = tf_executor.island { + %add1 = "tf.Add"(%arg0, %arg0) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + %add2 = "tf.Add"(%add1, %arg0) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + tf_executor.yield %arg0, %arg0 : tensor<*xi32>, tensor<*xi32> + } + tf_executor.fetch %island#2 : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @fetching_arg +// CHECK: tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg0) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg0) +// CHECK: tf_executor.fetch +// CHECK: } + +func.func @non_aliasing_reads_writes( + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + %island:2 = tf_executor.island { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %arg2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + %read2 = "tf.ReadVariableOp"(%var_handle) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg0, %read2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read3 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + tf_executor.yield %read3 : tensor<32xf32> + } + tf_executor.fetch %island#0 : tensor<32xf32> + } + func.return %graph : tensor<32xf32> +} + +// CHECK-LABEL: func @non_aliasing_reads_writes +// CHECK: %[[GRAPH:.*]] = tf_executor.graph { +// CHECK: %[[READ0:.*]], %[[READ0_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) +// CHECK: %[[ASSIGN0_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) +// CHECK: %[[READ1:.*]], %[[READ1_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) +// CHECK: %[[VH0:.*]], %[[VH0_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v0"}> +// CHECK: %[[READ2:.*]], %[[READ2_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%[[VH0]]) +// CHECK: %[[ASSIGN1_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%arg1, %[[READ0]]) +// CHECK: %[[ASSIGN2_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %[[READ2]]) +// CHECK: %[[READ3:.*]], %[[READ3_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) +// CHECK: tf_executor.fetch %[[READ3]] +// CHECK: } + +func.func @unknown_side_effecting_op(%arg0: tensor<32xf32>) -> () { + tf_executor.graph { + %island = tf_executor.island { + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v1"} : () -> tensor<*x!tf_type.resource>> + %read0 = "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh1, %arg0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf._UnknownSideEffectingOp_"() : () -> () + %read1 = "tf.ReadVariableOp"(%vh1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh0, %read1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%vh1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @unknown_side_effecting_op +// CHECK: tf_executor.graph { +// CHECK: %[[VH0:.*]], %[[VH0_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v0"}> +// CHECK: %[[VH1:.*]], %[[VH1_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v1"}> +// CHECK: %[[READ0:.*]], %[[READ0_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%[[VH0]]) +// CHECK: %[[ASSIGN0_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%[[VH1]], %arg0) +// CHECK: %[[UNKNOWN_CONTROL:.*]] = tf_executor.island wraps "tf._UnknownSideEffectingOp_"() +// CHECK: %[[READ1:.*]], %[[READ1_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%[[VH1]]) +// CHECK: %[[ASSIGN1_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%[[VH0]], %[[READ1]]) +// CHECK: %[[ASSIGN2_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%[[VH1]], %[[READ0]]) +// CHECK: tf_executor.fetch + +// Checks empty tf_executor.island ops are populated with tf.NoOp/tf.Identity/ +// tf.IdentityN ops depending on the number of data results the +// tf_executor.island has. + +func.func @empty_island_no_data_results() { + tf_executor.graph { + %0 = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch + } + func.return +} +// CHECK-LABEL: empty_island_no_data_results +// CHECK: "tf.NoOp" + +func.func @empty_island_single_data_result(%arg0: tensor<*xf32>) { + tf_executor.graph { + %0:2 = tf_executor.island { + tf_executor.yield %arg0 : tensor<*xf32> + } + tf_executor.fetch + } + func.return +} +// CHECK-LABEL: empty_island_single_data_result +// CHECK-SAME: (%[[ARG_0:.*]]: tensor<*xf32>) +// CHECK: %[[IDENTITY:.*]] = "tf.Identity" +// CHECK-SAME: (%[[ARG_0]]) +// CHECK: tf_executor.yield %[[IDENTITY]] + +func.func @empty_island_multiple_data_results(%arg0: tensor<*xf32>, %arg1: tensor<*xi32>) { + tf_executor.graph { + %0:3 = tf_executor.island { + tf_executor.yield %arg0, %arg1 : tensor<*xf32>, tensor<*xi32> + } + tf_executor.fetch + } + func.return +} +// CHECK-LABEL: empty_island_multiple_data_results +// CHECK-SAME: (%[[ARG_0:.*]]: tensor<*xf32>, %[[ARG_1:.*]]: tensor<*xi32>) +// CHECK: %[[IDENTITY_N:.*]]:2 = "tf.IdentityN" +// CHECK-SAME: (%[[ARG_0]], %[[ARG_1]]) +// CHECK: tf_executor.yield %[[IDENTITY_N]]#0, %[[IDENTITY_N]]#1 + +func.func @single_op_island_forward_block_arg(%arg0: tensor) -> (tensor<2048xf32>, tensor) { + %0:2 = tf_executor.graph { + %outputs:2, %control = tf_executor.island { + %1 = "tf.Const"() {value = dense<0.000000e+00> : tensor<2048xf32>} : () -> tensor<2048xf32> + tf_executor.yield %1, %arg0 : tensor<2048xf32>, tensor + } + tf_executor.fetch %outputs#0, %outputs#1 : tensor<2048xf32>, tensor + } + func.return %0#0, %0#1 : tensor<2048xf32>, tensor +} +// CHECK-LABEL: func @single_op_island_forward_block_arg +// CHECK: %[[CONST:.*]], %{{.*}} = tf_executor.island wraps "tf.Const" +// CHECK: tf_executor.fetch %[[CONST]], %arg0 + +func.func @single_op_island_duplicate_result() -> (tensor<2048xf32>, tensor<2048xf32>) { + %0:2 = tf_executor.graph { + %outputs:2, %control = tf_executor.island { + %1 = "tf.Const"() {value = dense<0.000000e+00> : tensor<2048xf32>} : () -> tensor<2048xf32> + tf_executor.yield %1, %1 : tensor<2048xf32>, tensor<2048xf32> + } + tf_executor.fetch %outputs#0, %outputs#1 : tensor<2048xf32>, tensor<2048xf32> + } + func.return %0#0, %0#1 : tensor<2048xf32>, tensor<2048xf32> +} +// CHECK-LABEL: func @single_op_island_duplicate_result +// CHECK: %[[CONST:.*]], %{{.*}} = tf_executor.island wraps "tf.Const" +// CHECK: tf_executor.fetch %[[CONST]], %[[CONST]] + +func.func @tpu_load_embedding_ops_sink_controls(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor<*x!tf_type.resource>>, %arg3: tensor<*x!tf_type.resource>>) { + tf_executor.graph { + %control = tf_executor.island { + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %1 = "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %2 = "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + "tf.LoadTPUEmbeddingAdagradParameters"(%0, %1) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table1"} : (tensor<8xf32>, tensor<8xf32>) -> () + %3 = "tf.ReadVariableOp"(%arg3) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + "tf.LoadTPUEmbeddingAdagradParameters"(%2, %3) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table2"} : (tensor<8xf32>, tensor<8xf32>) -> () + "tf.UnknownOp"() : () -> () + "tf.UnknownOp"() : () -> () + "tf.LoadTPUEmbeddingAdagradParameters"(%0, %1) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table3"} : (tensor<8xf32>, tensor<8xf32>) -> () + "tf.LoadTPUEmbeddingAdagradParameters"(%2, %3) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table4"} : (tensor<8xf32>, tensor<8xf32>) -> () + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + func.return +} +// CHECK-LABEL: func @tpu_load_embedding_ops_sink_controls +// CHECK: {{%.+}}, [[READ0:%.+]] = tf_executor.island wraps "tf.ReadVariableOp" +// CHECK: {{%.+}}, [[READ1:%.+]] = tf_executor.island wraps "tf.ReadVariableOp" +// CHECK: {{%.+}}, [[READ2:%.+]] = tf_executor.island wraps "tf.ReadVariableOp" +// CHECK: [[LOAD0:%.+]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters" +// CHECK: {{%.+}}, [[READ3:%.+]] = tf_executor.island wraps "tf.ReadVariableOp" +// CHECK: [[LOAD1:%.+]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters" +// CHECK: [[UNKNOWN0:%.+]] = tf_executor.island wraps "tf.UnknownOp" +// CHECK: [[UNKNOWN1:%.+]] = tf_executor.island wraps "tf.UnknownOp" +// CHECK: [[LOAD2:%.+]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters" +// CHECK: [[LOAD3:%.+]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters" +// CHECK: tf_executor.fetch + + + +// CHECK-LABEL: func @stateful_composite_op_control +func.func @stateful_composite_op_control(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>) -> tensor { + %0 = tf_executor.graph { + %output, %control = tf_executor.island { + // CHECK: {{%.+}}, [[IF_CONTROL:%.+]] = tf_executor.island wraps "tf.If" + %1 = "tf.If"(%arg0, %arg1) {device = "", else_branch = @stateful_composite_op_control_else, is_stateless = false, then_branch = @stateful_composite_op_control_then} : (tensor, tensor<*x!tf_type.resource>>) -> tensor + // CHECK: [[IDENTITY_OUTPUT:%.+]], [[IDENTITY_CONTROL:%.+]] = tf_executor.island wraps "tf.Identity" + %2 = "tf.Identity"(%1) {device = ""} : (tensor) -> tensor + tf_executor.yield %2 : tensor + } + // CHECK: tf_executor.fetch [[IDENTITY_OUTPUT]] + tf_executor.fetch %output : tensor + } + func.return %0 : tensor +} + +// CHECK: func @stateful_composite_op_control_else +// This is a helper function for the stateful_composite_op_control test. +func.func @stateful_composite_op_control_else(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island { + %1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %1) : (tensor<*x!tf_type.resource>>, tensor) -> () + tf_executor.yield %1 : tensor + } + tf_executor.fetch %outputs : tensor + } + func.return %0 : tensor +} + +// CHECK: func @stateful_composite_op_control_then +// This is a helper function for the stateful_composite_op_control test. +func.func @stateful_composite_op_control_then(%arg0: tensor<*x!tf_type.resource>>) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island { + %1 = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + "tf.AssignVariableOp"(%arg0, %1) : (tensor<*x!tf_type.resource>>, tensor) -> () + tf_executor.yield %1 : tensor + } + tf_executor.fetch %outputs : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @generator_op +func.func @generator_op(%str : tensor, %arg0: tensor<*x!tf_type.string>, %arg1: tensor, %arg2: tensor<*xi64>, %arg3: tensor) { + tf_executor.graph { + tf_executor.island { + // CHECK: %{{.*}}, %[[CONTROL:[^ ,]*]] = tf_executor.island wraps "tf.GeneratorDataset" + %gen0 = "tf.GeneratorDataset"(%str, %arg0, %arg1, %arg2, %arg3) { + finalize_func = @__finalize_func_790, + init_func = @__init_func_530, next_func = @__next_func_680, + next_func.experimental_ints_on_device = true, + operandSegmentSizes = array, + output_shapes = [#tf_type.shape], + output_types = [f32], + metadata = "" + } : + (tensor, tensor<*x!tf_type.string>, tensor, tensor<*xi64>, tensor) -> tensor<*x!tf_type.variant> + // CHECK: %{{.*}}, %[[CONTROL2:[^ ,]*]] = tf_executor.island wraps "tf.Add" + %add1 = "tf.Add"(%str, %arg3) : (tensor, tensor) -> tensor + // CHECK: %{{.*}}, %[[CONTROL3:[^ ,]*]] = tf_executor.island wraps "tf.GeneratorDataset" + %gen1 = "tf.GeneratorDataset"(%str, %arg0, %arg1, %arg2, %arg3) { + finalize_func = @__finalize_func_790, + init_func = @__init_func_530, next_func = @__next_func_680, + next_func.experimental_ints_on_device = true, + operandSegmentSizes = array, + output_shapes = [#tf_type.shape], + output_types = [f32], + metadata = "" + } : + (tensor, tensor<*x!tf_type.string>, tensor, tensor<*xi64>, tensor) -> tensor<*x!tf_type.variant> + tf_executor.yield + } + // CHECK: tf_executor.fetch + tf_executor.fetch + } + func.return +} + +func.func @then_function() { + tf_executor.graph { + tf_executor.island { + "tf.OpA"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +func.func @else_function() { + tf_executor.graph { + tf_executor.island { + "tf.OpB"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/stack_ops_decomposition.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/stack_ops_decomposition.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6b50b246a748decaebfd9e0ac997794b481e5c12 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/stack_ops_decomposition.mlir @@ -0,0 +1,462 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-stack-ops-decomposition | FileCheck %s + +// Tests simple scalar stack operations without control flow. + +// CHECK-LABEL: func @main +func.func @main() -> tensor { + // CHECK-NEXT: "tf.Const"() <{value = dense<10> : tensor}> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NEXT: %[[ZERO_SCALAR:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %[[CAST_ZERO:.*]] = "tf.Cast"(%[[ZERO_SCALAR]]) : (tensor) -> tensor + // CHECK-NEXT: %[[CONST10:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[CAST_ZERO]], %[[CONST10]]) : (tensor, tensor<1xi32>) -> tensor<10xf32> + // CHECK-NEXT: %[[BUFFER:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NEXT: %[[SIZE:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NEXT: %[[ZERO:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[SIZE]], %[[ZERO]]) + // CHECK-NEXT: "tf.AssignVariableOp"(%[[BUFFER]], %[[BROADCAST]]) + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %id = "tf.Identity"(%stack) : (tensor) -> tensor + // CHECK-NEXT: %[[PUSHVAL:.*]] = "tf._SomeOp"() + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NEXT: %[[READ_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK-NEXT: %[[READ_SIZE:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK-NEXT: %[[UPDATE_SHAPE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[UPDATE_SLICE:.*]] = "tf.Reshape"(%[[PUSHVAL]], %[[UPDATE_SHAPE]]) : (tensor, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_VAL]], %[[UPDATE_SLICE]], %[[READ_SIZE]]) : (tensor<10xf32>, tensor<1xf32>, tensor<1xi32>) -> tensor<10xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[BUFFER]], %[[UPDATE]]) : (tensor>>, tensor<10xf32>) -> () + // CHECK-NEXT: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[NEW_SIZE:.*]] = "tf.AddV2"(%[[READ_SIZE]], %[[CONST1]]) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[SIZE]], %[[NEW_SIZE]]) : (tensor>>, tensor<1xi32>) -> () + %push = "tf.StackPushV2"(%id, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + %pop = "tf.StackPopV2"(%stack) : (tensor) -> tensor + // CHECK-NEXT: %[[READ_VAL1:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK-NEXT: %[[READ_SIZE1:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK-NEXT: %[[CONST1_1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SUB:.*]] = "tf.Sub"(%[[READ_SIZE1]], %[[CONST1_1]]) + // CHECK-NEXT: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE:.*]] = "tf.Slice"(%[[READ_VAL1]], %[[SUB]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK-NEXT: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + // CHECK-NEXT: "tf.AssignVariableOp"(%[[SIZE]], %[[SUB]]) : (tensor>>, tensor<1xi32>) -> () + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK-NEXT: return %[[ELEM]] : tensor + func.return %pop : tensor +} + +// ----- + +// Tests simple non-scalar stack operations without control flow. + +// CHECK-LABEL: func @main +func.func @main() -> tensor<2xi32> { + // CHECK-NEXT: "tf.Const"() <{value = dense<10> : tensor}> : () -> tensor + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NEXT: %[[ZERO_CONST:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %[[STACK_SHAPE:.*]] = "tf.Const"() <{value = dense<[10, 2]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK-NEXT: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[ZERO_CONST]], %[[STACK_SHAPE]]) : (tensor, tensor<2xi32>) -> tensor<10x2xi32> + // CHECK-NEXT: %[[BUFFER:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NEXT: %[[SIZE:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NEXT: %[[ZERO_SIZE:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[SIZE]], %[[ZERO_SIZE]]) : (tensor>>, tensor<1xi32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[BUFFER]], %[[BROADCAST]]) : (tensor>>, tensor<10x2xi32>) -> () + %stack = "tf.StackV2"(%size) {elem_type = i32, stack_name = "s"} : (tensor) -> tensor + // CHECK-NEXT: %[[PUSH_VAL:.*]] = "tf._SomeOp"() : () -> tensor<2xi32> + %elem = "tf._SomeOp"() : () -> tensor<2xi32> + // CHECK-NEXT: %[[STACK_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) : (tensor>>) -> tensor<10x2xi32> + // CHECK-NEXT: %[[STACK_SIZE:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) : (tensor>>) -> tensor<1xi32> + // CHECK-NEXT: %[[UPDATE_SHAPE:.*]] = "tf.Const"() <{value = dense<[1, 2]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK-NEXT: %[[UPDATE_SLICE:.*]] = "tf.Reshape"(%[[PUSH_VAL]], %[[UPDATE_SHAPE]]) : (tensor<2xi32>, tensor<2xi32>) -> tensor<1x2xi32> + // CHECK-NEXT: %[[ZERO_INDS:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[CONCAT_DIM:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %[[CONCAT_OFFETS:.*]] = "tf.ConcatV2"(%[[STACK_SIZE]], %[[ZERO_INDS]], %[[CONCAT_DIM]]) : (tensor<1xi32>, tensor<1xi32>, tensor) -> tensor<2xi32> + // CHECK-NEXT: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[STACK_VAL]], %[[UPDATE_SLICE]], %[[CONCAT_OFFETS]]) : (tensor<10x2xi32>, tensor<1x2xi32>, tensor<2xi32>) -> tensor<10x2xi32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[BUFFER]], %[[UPDATE]]) : (tensor>>, tensor<10x2xi32>) -> () + // CHECK-NEXT: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[NEW_SIZE:.*]] = "tf.AddV2"(%[[STACK_SIZE]], %[[CONST1]]) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[SIZE]], %[[NEW_SIZE]]) : (tensor>>, tensor<1xi32>) -> () + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor<2xi32>) -> tensor<2xi32> + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK-NEXT: return %[[PUSH_VAL]] : tensor<2xi32> + func.return %push : tensor<2xi32> +} + +// ----- + +// Tests while loop. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %1:2 = "tf.While"(%stack, %max_size) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor, tensor) -> (tensor, tensor) + // CHECK: "tf.Slice" + %pop = "tf.StackPopV2"(%1#0) : (tensor) -> tensor + // CHECK-NOT: tf.Stack + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor>>, %[[BARG1:.*]]: tensor, %[[BARG2:.*]]: tensor>>) +func.func @while_body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BARG1]], %[[CONST1]]) + %sub = "tf.Sub"(%arg1, %const1) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.StackPushV2" + // CHECK: "tf.XlaDynamicUpdateSlice" + // CHECK-NOT: "tf.StackPushV2" + %push = "tf.StackPushV2"(%arg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + // CHECK: return %[[BARG0]], %[[SUB]], %[[BARG2]] + func.return %arg0, %sub : tensor, tensor +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor>>, %[[CARG1:.*]]: tensor, %[[CARG2:.*]]: tensor>>) +func.func @while_cond(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NEXT: return %[[CARG1]] + func.return %arg1 : tensor +} + +// ----- + +// Tests WhileRegion Op. + +// CHECK-LABEL: func @main() +func.func @main() -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + // CHECK: %[[BUFFER:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: %[[SIZE:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: tf.AssignVariableOp + // CHECK: tf.AssignVariableOp + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // CHECK: tf.WhileRegion + %while = "tf.WhileRegion"(%max_size) ({ + // CHECK: ^bb0(%[[BARG0:.*]]: tensor + ^bb0(%barg0: tensor): + // CHECK: "tf._SomeOp"(%[[BARG0]]) + %pred = "tf._SomeOp"(%barg0) : (tensor) -> tensor + "tf.Yield"(%pred) : (tensor) -> () + }, { + // CHECK: ^bb0(%[[BARG0:.*]]: tensor + ^bb0(%barg0: tensor): + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BARG0]], %[[CONST1]]) + %sub = "tf.Sub"(%barg0, %const1) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.StackPushV2" + // CHECK: %[[BUFFER_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK: %[[SIZE_VAL:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[BUFFER_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[BUFFER]], %[[UPDATE]]) + // CHECK: "tf.AssignVariableOp"(%[[SIZE]] + // CHECK-NOT: "tf.StackPushV2" + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + // CHECK: "tf.Yield"(%[[SUB]]) + "tf.Yield"(%sub) : (tensor) -> () + }) {is_stateless = false} + : (tensor) -> tensor + // CHECK-NOT: tf.StackPopV2 + // CHECK: %[[BUFFER_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK: %[[SIZE_VAL:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK: %[[POP_VAL:.*]] = "tf.Slice"(%[[BUFFER_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[SIZE]] + %pop = "tf.StackPopV2"(%stack) : (tensor) -> tensor + // CHECK-NOT: tf.StackCloseV2 + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return +} + +// ----- + +// Test CaseRegionOp + +// CHECK-LABEL: func @main +// CHECK-SAME: %[[BRANCH_INDEX:.*]]: tensor +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.StackV2 + // CHECK: %[[BUFFER:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: %[[SIZE:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: tf.AssignVariableOp + // CHECK: tf.AssignVariableOp + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // CHECK: %[[CASE_OUTPUT:.*]] = "tf.CaseRegion"(%[[BRANCH_INDEX]]) {{.*}} ({ + %case_op = "tf.CaseRegion"(%arg0) ({ + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: tf.StackPushV2 + // CHECK: %[[BUFFER_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK: %[[SIZE_VAL:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[BUFFER_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[BUFFER]], %[[UPDATE]]) + // CHECK: "tf.AssignVariableOp"(%[[SIZE]] + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + "tf.Yield"(%elem) : (tensor) -> () + }, { + %elem = "tf._SomeOtherOp"() : () -> tensor + // CHECK-NOT: tf.StackPushV2 + // CHECK: %[[BUFFER_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK: %[[SIZE_VAL:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[BUFFER_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[BUFFER]], %[[UPDATE]]) + // CHECK: "tf.AssignVariableOp"(%[[SIZE]] + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + "tf.Yield"(%elem) : (tensor) -> () + }, { + // CHECK-NOT: tf.StackPopV2 + // CHECK: %[[BUFFER_VAL:.*]] = "tf.ReadVariableOp"(%[[BUFFER]]) + // CHECK: %[[SIZE_VAL:.*]] = "tf.ReadVariableOp"(%[[SIZE]]) + // CHECK: %[[POP_VAL:.*]] = "tf.Slice"(%[[BUFFER_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[SIZE]] + %pop = "tf.StackPopV2"(%stack) : (tensor) -> tensor + "tf.Yield"(%pop) : (tensor) -> () + }) {is_stateless = false} + : (tensor) -> tensor + // CHECK-NOT: tf.StackPopV2 + %pop = "tf.StackPopV2"(%stack) : (tensor) -> tensor + // CHECK-NOT: tf.StackCloseV2 + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return +} + +// ----- +// Tests IfOp. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %if_op = "tf.If"(%arg0, %stack) {then_branch = @if_then, else_branch = @if_else, is_stateless = false} + : (tensor, tensor) -> tensor + // CHECK: "tf.Slice" + %pop = "tf.StackPopV2"(%if_op) : (tensor) -> tensor + // CHECK-NOT: tf.Stack + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK: return + func.return +} +// CHECK: func @if_then(%[[TARG0:.*]]: tensor>>, %[[TARG1:.*]]: tensor>>) +func.func @if_then(%arg0: tensor) -> tensor { + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.StackPushV2" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: "tf.AssignVariableOp"(%[[TARG0:.*]], %[[UPDATE]]) + // CHECK: "tf.AssignVariableOp"(%[[EARG1:.*]], + // CHECK-NOT: "tf.StackPushV2" + %push = "tf.StackPushV2"(%arg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return %arg0 : tensor +} +// CHECK: func @if_else(%[[EARG0:.*]]: tensor>>, %[[EARG1:.*]]: tensor>>) +func.func @if_else(%arg0: tensor) -> tensor { + // CHECK-NOT: "tf.StackPopV2" + // CHECK: "tf.Slice" + // CHECK: "tf.AssignVariableOp"(%[[EARG1:.*]], + // CHECK-NOT: "tf.StackPopV2" + %pop = "tf.StackPopV2"(%arg0) : (tensor) -> tensor + func.return %arg0 : tensor +} + +// ----- + +// Tests PartitionedCall/StatefulPartitionedCall. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // CHECK: "tf.StatefulPartitionedCall" + // CHECK-SAME: f = @callee_stack_decomposed + %call = "tf.StatefulPartitionedCall"(%stack, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor) -> tensor + // CHECK: "tf.PartitionedCall" + // CHECK-SAME: f = @callee_stack_decomposed + %call2 = "tf.PartitionedCall"(%stack, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor) -> tensor + // CHECK: "tf.Slice" + %pop = "tf.StackPopV2"(%call) : (tensor) -> tensor + // CHECK-NOT: tf.Stack + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK: return + func.return +} + +// CHECK: func @callee(%[[AARG0:.*]]: tensor, %[[AARG1:.*]]: tensor) -> tensor +func.func @callee(%arg0: tensor, %arg1: tensor) -> tensor { + %elem = "tf._SomeOp"(%arg1) : (tensor) -> tensor + // CHECK: tf.StackPushV2" + %push = "tf.StackPushV2"(%arg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return %arg0 : tensor +} + +// CHECK: func private @callee_stack_decomposed(%[[ARG0:.*]]: tensor>>, %[[ARG1:.*]]: tensor, %[[ARG2:.*]]: tensor>>) +// CHECK-NOT: "tf.StackPushV2" +// CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" +// CHECK: "tf.AssignVariableOp"(%[[TARG0:.*]], %[[UPDATE]]) +// CHECK: "tf.AssignVariableOp"(%[[EARG1:.*]], +// CHECK-NOT: "tf.StackPushV2" + +// ----- + +// Tests PartitionedCall/StatefulPartitionedCall with private callee function. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // CHECK: "tf.StatefulPartitionedCall" + // CHECK-SAME: f = @callee + %call = "tf.StatefulPartitionedCall"(%stack, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor) -> tensor + // CHECK: "tf.PartitionedCall" + // CHECK-SAME: f = @callee + %call2 = "tf.PartitionedCall"(%stack, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor, tensor) -> tensor + // CHECK: "tf.Slice" + %pop = "tf.StackPopV2"(%call) : (tensor) -> tensor + // CHECK-NOT: tf.Stack + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK: return + func.return +} + +// CHECK: func private @callee(%[[ARG0:.*]]: tensor>>, %[[ARG1:.*]]: tensor, %[[ARG2:.*]]: tensor>>) +func.func private @callee(%arg0: tensor, %arg1: tensor) -> tensor { + %elem = "tf._SomeOp"(%arg1) : (tensor) -> tensor + // CHECK-NOT: "tf.StackPushV2" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: "tf.AssignVariableOp"(%[[TARG0:.*]], %[[UPDATE]]) + // CHECK: "tf.AssignVariableOp"(%[[EARG1:.*]], + // CHECK-NOT: "tf.StackPushV2" + %push = "tf.StackPushV2"(%arg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return %arg0 : tensor +} + +// ----- + +// Tests PartitionedCall op with no signature change on callee. + +// CHECK-LABEL: func @main +func.func @main() -> () { + "tf.PartitionedCall"() {f = @callee, config = "", config_proto = "", executor_type = ""} : () -> () + func.return +} +// CHECK: func @callee() +func.func @callee() -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.Stack + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return +} + +// ----- + +// Tests that the pass reports error on unknown stack size. + +func.func @main(%arg0: tensor) -> tensor<2xi32> { + // expected-error @+1 {{unknown max element count}} + %stack = "tf.StackV2"(%arg0) {elem_type = i32, stack_name = "s"} : (tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor<2xi32> + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor<2xi32>) -> tensor<2xi32> + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return %push : tensor<2xi32> +} + +// ----- + +// Tests that the pass reports error on unknown element shape. + +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // expected-error @+1 {{cannot infer element shape of stack}} + %stack = "tf.StackV2"(%max_size) {elem_type = i32, stack_name = "s"} : (tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor<*xi32> + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor<*xi32>) -> tensor<*xi32> + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return +} + +// ----- + +// Tests that the pass reports error on ambiguous stack. + +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %stack2 = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s2"} : (tensor) -> tensor + %if_op = "tf.If"(%arg0, %stack, %stack2) {then_branch = @if_then, else_branch = @if_else, is_stateless = false} + : (tensor, tensor, tensor) -> tensor + // expected-error @+1 {{unknown stack}} + %pop = "tf.StackPopV2"(%if_op) : (tensor) -> tensor + "tf.StackCloseV2"(%stack) : (tensor) -> () + // CHECK: return + func.return +} +func.func @if_then(%arg0: tensor, %arg1: tensor) -> tensor { + %elem = "tf._SomeOp"() : () -> tensor + %push = "tf.StackPushV2"(%arg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return %arg0 : tensor +} +func.func @if_else(%arg0: tensor, %arg1: tensor) -> tensor { + %elem = "tf._SomeOp"() : () -> tensor + %push = "tf.StackPushV2"(%arg1, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + func.return %arg1 : tensor +} + +// ----- + +// Tests that the pass returns meaningful error message when WhileRegion op has +// resource arguments. +func.func @main() -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + %push_0 = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + // expected-error @+1 {{found unexpected type 'tensor>>' of operand #0, resource type operands are expected to have been canonicalized away for region based control flow ops}} + %1:2 = "tf.WhileRegion"(%stack, %max_size) ({ + ^bb0 (%carg0: tensor, %carg1: tensor): + %pred = "tf._SomeOp"(%carg1) : (tensor) -> tensor + "tf.Yield"(%pred) : (tensor) -> () + }, { + ^bb0 (%carg0: tensor, %carg1: tensor): + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %sub = "tf.Sub"(%carg1, %const1) : (tensor, tensor) -> tensor + %push_1 = "tf.StackPushV2"(%carg0, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + "tf.Yield"(%carg0, %sub) : (tensor, tensor) -> () + }) {is_stateless = false} + : (tensor, tensor) -> (tensor, tensor) + %pop = "tf.StackPopV2"(%1#0) : (tensor) -> tensor + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return +} + +// ----- + +// Tests that the pass returns meaningful error message when IfRegion op has +// resource returns. + +func.func @main(%arg0: tensor) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %stack = "tf.StackV2"(%max_size) {elem_type = f32, stack_name = "s"} : (tensor) -> tensor + // expected-error @+1 {{found unexpected type 'tensor' of result #0, resource type results are expected to have been canonicalized away for region based control flow ops}} + %if_op = "tf.IfRegion"(%arg0) ({ + %elem = "tf._SomeOp"() : () -> tensor + %push = "tf.StackPushV2"(%stack, %elem) {swap_memory = false} : (tensor, tensor) -> tensor + "tf.Yield"(%stack) : (tensor) -> () + }, { + %pop = "tf.StackPopV2"(%stack) : (tensor) -> tensor + "tf.Yield"(%stack) : (tensor) -> () + }) {is_stateless = false} + : (tensor) -> tensor + %pop = "tf.StackPopV2"(%if_op) : (tensor) -> tensor + "tf.StackCloseV2"(%stack) : (tensor) -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_noinline.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_noinline.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e46693b0a00f8d6833b5bb682d1c3b079481df06 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_noinline.mlir @@ -0,0 +1,9 @@ +// RUN: tf-opt %s -tf-strip-noinline-attribute | FileCheck %s + +// CHECK-LABEL: func @strip_simple( +// CHECK-NOT: tf._noinline +func.func @strip_simple() -> tensor<2xi32> attributes {tf._noinline = true} { + // CHECK-NEXT: %[[CST:.*]] = "tf.Const" + %cst = "tf.Const"() { value = dense<2> : tensor<2xi32> } : () -> tensor<2xi32> + func.return %cst : tensor<2xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_saved_module_metadata.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_saved_module_metadata.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5fb55d1109d7230f167d12ec6ba07b809f492f8a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_saved_module_metadata.mlir @@ -0,0 +1,15 @@ +// RUN: tf-opt %s --tf-strip-saved-module-metadata --split-input-file | FileCheck %s + +// CHECK-LABEL: module +// CHECK-NOT: tf_saved_model +module attributes {tf_saved_model.semantics} { + // CHECK: tf_saved_model.global_tensor + // CHECK-NOT: tf_saved_model + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<1.> : tensor<1xf32> } : () -> () + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + -> (tensor>> {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: return + return %arg0 : tensor>> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_tf_attributes.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_tf_attributes.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f3ec527eff024cec7901ed3bbc30beb351299d97 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/strip_tf_attributes.mlir @@ -0,0 +1,41 @@ +// RUN: tf-opt %s -allow-unregistered-dialect --tf-strip-tf-attributes --split-input-file | FileCheck %s + +// CHECK-LABEL: strips_attributes +// CHECK-NOT: tf +func.func @strips_attributes(%arg0: tensor<32x28x28x1xf32> {tf._user_specified_name = "x"}, + %arg1: tensor<3x3x1x5xf32> {tf._user_specified_name = "w1"}, + %arg2: tensor<5xf32> {tf._user_specified_name = "b1"}, + %arg3: tensor<3920x10xf32> {tf._user_specified_name = "w2"}, + %arg4: tensor<10xf32> {tf._user_specified_name = "b2"}) -> tensor<10xf32> + attributes {tf._construction_context = "kEagerRuntime", tf._input_shapes = [#tf_type.shape<32x28x28x1>, #tf_type.shape<3x3x1x5>, #tf_type.shape<5>, #tf_type.shape<3920x10>, #tf_type.shape<10>]} { + return %arg4 : tensor<10xf32> +} + +// ----- + +// CHECK-LABEL: strips_attributes_with_tf_values +// CHECK-NOT: tf +func.func @strips_attributes_with_tf_values() + attributes {foo = #tf_type.shape<32x28x28x1>} { + return +} + +// ----- + +// CHECK-LABEL: strips_result_attributes +// CHECK-NOT: tf +func.func @strips_result_attributes() -> (f32 {tf.foo = "bar"}) { + %0 = "foo.constant"() : () -> f32 + return %0 : f32 +} + +// ----- + +// CHECK-LABEL: strips_module_attributes +module @strips_module_attributes attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1235 : i32}} { +// CHECK-NOT: tf +// CHECK: body + func.func private @body() -> () { + return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_array_ops_decomposition.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_array_ops_decomposition.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0e5d8cfbddf329cb1062d080177c6ed8752f3cf1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_array_ops_decomposition.mlir @@ -0,0 +1,715 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tensor-array-ops-decomposition | FileCheck %s + +// Test read and write on a tensor list. + +// CHECK-LABEL: func @main +func.func @main() -> tensor<3xf32> { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[BUFFER:.*]] = "tf.BroadcastTo" + // CHECK-SAME: -> tensor<5x3xf32> + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[BUFFER]]) + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[IND:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAL:.*]] = "tf.Const"() <{value = dense<[1.000000e+00, 2.000000e+00, 3.000000e+00]> : tensor<3xf32>}> : () -> tensor<3xf32> + %value = "tf.Const"() {value = dense<[1.0, 2.0, 3.0]> : tensor<3xf32>} : () -> tensor<3xf32> + // CHECK: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: %[[UPDATE_SLICE:.*]] = "tf.Reshape"(%[[VAL]] + // CHECK-SAME: -> tensor<1x3xf32> + // CHECK: %[[NEW_BUFFER:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_VAR]], %[[UPDATE_SLICE]], + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[NEW_BUFFER]]) + %write = "tf.TensorArrayWriteV3"(%ta#0, %index, %value, %ta#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[READ_VAR2:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: %[[READ_SLICE:.*]] = "tf.Slice"(%[[READ_VAR2]], + // CHECK: %[[READ:.*]] = "tf.Reshape"(%[[READ_SLICE]], + // CHECK-SAME: -> tensor<3xf32> + %read = "tf.TensorArrayReadV3"(%ta#0, %index, %write) : (tensor, tensor, tensor) -> tensor<3xf32> + // CHECK-NOT: TensorArrayCloseV3 + "tf.TensorArrayCloseV3"(%ta#0) : (tensor) -> () + // CHECK: return %[[READ]] : tensor<3xf32> + func.return %read: tensor<3xf32> +} + +// ----- + +// Test inferring shape from the first write. + +// CHECK-LABEL: func @main +func.func @main() -> tensor { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[BUFFER:.*]] = "tf.BroadcastTo" + // CHECK-SAME: -> tensor<5x3xf32> + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[BUFFER]]) + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %value = "tf.Const"() {value = dense<[1.0, 2.0, 3.0]> : tensor<3xf32>} : () -> tensor<3xf32> + %write = "tf.TensorArrayWriteV3"(%ta#0, %index, %value, %ta#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size_out = "tf.TensorArraySizeV3"(%ta#0, %write) : (tensor, tensor) -> tensor + // CHECK: return %[[SIZE]] : tensor + func.return %size_out : tensor +} + +// ----- + +// Test inferring shape from the first scatter. + +// CHECK-LABEL: func @main +func.func @main() -> tensor { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %indices = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %values = "tf.Const"() {value = dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : tensor<2x3xf32>} : () -> tensor<2x3xf32> + %write = "tf.TensorArrayScatterV3"(%ta#0, %indices, %values, %ta#1) : (tensor, tensor<2xi32>, tensor<2x3xf32>, tensor) -> tensor + %size_out = "tf.TensorArraySizeV3"(%ta#0, %write) : (tensor, tensor) -> tensor + func.return %size_out : tensor +} + +// ----- + +// Test inferring shape from the result type of gather. + +// CHECK-LABEL: func @main +func.func @main() -> tensor<2x3xf32> { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %indices = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %gather = "tf.TensorArrayGatherV3"(%ta#0, %indices, %ta#1) : (tensor, tensor<2xi32>, tensor) -> tensor<2x3xf32> + func.return %gather : tensor<2x3xf32> +} + +// ----- + +// Test inferring shape from the element_shape attribute of gather. + +// CHECK-LABEL: func @main +func.func @main() -> tensor<*xf32> { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %indices = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %gather = "tf.TensorArrayGatherV3"(%ta#0, %indices, %ta#1) {element_shape = #tf_type.shape<3>} : (tensor, tensor<2xi32>, tensor) -> tensor<*xf32> + func.return %gather : tensor<*xf32> +} + + +// ----- + +// Test tensor array concat and split. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp" + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[CONCAT_RESHAPE:.*]] = "tf.Reshape"(%[[READ]], + // CHECK-SAME: -> tensor<15xf32> + // CHECK: %[[LENS:.*]] = "tf.Const"() <{value = dense<3> : tensor<5xi64>}> : () -> tensor<5xi64> + %concat:2 = "tf.TensorArrayConcatV3"(%ta#0, %ta#1) {element_shape_except0 = #tf_type.shape<*>} : (tensor, tensor) -> (tensor<*xf32>, tensor<*xi64>) + // CHECK: %[[SPLIT_RESHAPE:.*]] = "tf.Reshape"(%[[CONCAT_RESHAPE]], + // CHECK-SAME: -> tensor<5x3xf32> + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[READ2]], %[[SPLIT_RESHAPE]]) + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[ADD]]) + %split = "tf.TensorArraySplitV3"(%ta#0, %concat#0, %concat#1, %ta#1) {element_shape_except0 = #tf_type.shape<*>} : (tensor, tensor<*xf32>, tensor<*xi64>, tensor) -> tensor + func.return +} + +// ----- + +// Test tensor array gather and scatter on contiguous indices. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp" + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %indices = "tf.Const"() {value = dense<[0, 1, 2, 3, 4]> : tensor<5xi32>} : () -> tensor<5xi32> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[GATHER_SLICE:.*]] = "tf.Slice"(%[[READ]] + // CHECK-SAME: (tensor<5x3xf32>, tensor<2xi32>, tensor<2xi32>) -> tensor<5x3xf32> + %gather = "tf.TensorArrayGatherV3"(%ta#0, %indices, %ta#1) {element_shape = #tf_type.shape<*>} : (tensor, tensor<5xi32>, tensor) -> tensor<*xf32> + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[READ2]], %[[GATHER_SLICE]]) + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[ADD]]) + %scatter = "tf.TensorArrayScatterV3"(%ta#0, %indices, %gather, %ta#1) {element_shape_except0 = #tf_type.shape<*>} : (tensor, tensor<5xi32>, tensor<*xf32>, tensor) -> tensor + func.return +} + +// ----- + +// Test tensor array gather and scatter on non-contiguous indices. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp" + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[INDS:.*]] = "tf.Const"() <{value = dense<[2, 1]> : tensor<2xi32>}> : () -> tensor<2xi32> + %indices = "tf.Const"() {value = dense<[2, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK: %[[GATHER:.*]] = "tf.GatherV2"(%[[READ]], %[[INDS]], %[[AXIS]]) : (tensor<5x3xf32>, tensor<2xi32>, tensor) -> tensor<2x3xf32> + %gather = "tf.TensorArrayGatherV3"(%ta#0, %indices, %ta#1) {element_shape = #tf_type.shape<*>} : (tensor, tensor<2xi32>, tensor) -> tensor<*xf32> + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK-DAG: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<[1, 3]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK-DAG: %[[IND_SLICE0_START:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-DAG: %[[IND_SLICE0_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[IND_SLICE0:.*]] = "tf.Slice"(%[[INDS]], %[[IND_SLICE0_START]], %[[IND_SLICE0_SIZE]]) : (tensor<2xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK: %[[SLICE0_START:.*]] = "tf.ConcatV2"(%[[IND_SLICE0]], + // CHECK: %[[OLD_SLICE0:.*]] = "tf.Slice"(%[[READ2]], %[[SLICE0_START]], + // CHECK-SAME: (tensor<5x3xf32>, tensor<2xi32>, tensor<2xi32>) -> tensor<1x3xf32> + // CHECK: %[[UPDATE_SLICE0_START:.*]] = "tf.Const"() <{value = dense<0> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK: %[[UPDATE_SLICE0:.*]] = "tf.Slice"(%[[GATHER]], %[[UPDATE_SLICE0_START]], %[[SLICE_SIZE]]) : (tensor<2x3xf32>, tensor<2xi32>, tensor<2xi32>) -> tensor<1x3xf32> + // CHECK: %[[ADD0:.*]] = "tf.AddV2"(%[[OLD_SLICE0]], %[[UPDATE_SLICE0]]) + // CHECK: %[[UPDATE0:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], %[[ADD0]] + // CHECK-SAME: (tensor<5x3xf32>, tensor<1x3xf32>, tensor<2xi32>) -> tensor<5x3xf32> + + // CHECK-DAG: %[[IND_SLICE1_START:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-DAG: %[[IND_SLICE1_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[IND_SLICE1:.*]] = "tf.Slice"(%[[INDS]], %[[IND_SLICE1_START]], %[[IND_SLICE1_SIZE]]) : (tensor<2xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK: %[[SLICE1_START:.*]] = "tf.ConcatV2"(%[[IND_SLICE1]], + // CHECK: %[[OLD_SLICE1:.*]] = "tf.Slice"(%[[UPDATE0]], %[[SLICE1_START]], + // CHECK-SAME: (tensor<5x3xf32>, tensor<2xi32>, tensor<2xi32>) -> tensor<1x3xf32> + // CHECK: %[[UPDATE_SLICE1_START:.*]] = "tf.Const"() <{value = dense<[1, 0]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK: %[[UPDATE_SLICE1:.*]] = "tf.Slice"(%[[GATHER]], %[[UPDATE_SLICE1_START]], %[[SLICE_SIZE]]) : (tensor<2x3xf32>, tensor<2xi32>, tensor<2xi32>) -> tensor<1x3xf32> + // CHECK: %[[ADD1:.*]] = "tf.AddV2"(%[[OLD_SLICE1]], %[[UPDATE_SLICE1]]) + // CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[UPDATE0]], %[[ADD1]] + // CHECK-SAME: (tensor<5x3xf32>, tensor<1x3xf32>, tensor<2xi32>) -> tensor<5x3xf32> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[UPDATE1]]) + %scatter = "tf.TensorArrayScatterV3"(%ta#0, %indices, %gather, %ta#1) {element_shape_except0 = #tf_type.shape<*>} : (tensor, tensor<2xi32>, tensor<*xf32>, tensor) -> tensor + func.return +} + +// ----- + +// Test tensor array grads. + +// CHECK-LABEL: func @main +func.func @main() { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VALUE:.*]] = "tf.Const"() <{value = dense<[1.000000e+00, 2.000000e+00, 3.000000e+00]> : tensor<3xf32>}> : () -> tensor<3xf32> + %value = "tf.Const"() {value = dense<[1.0, 2.0, 3.0]> : tensor<3xf32>} : () -> tensor<3xf32> + // CHECK: %[[GVAR1:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[GVAR1]], + %grad1:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %grad2:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[GVAR3:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[GVAR3]], + %grad3:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "b"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[GVAR1]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[OLD_SLICE1:.*]] = "tf.Slice"(%[[READ1]], + // CHECK: %[[RESHAPE1:.*]] = "tf.Reshape"(%[[VALUE]], + // CHECK: %[[ADD1:.*]] = "tf.AddV2"(%[[RESHAPE1]], %[[OLD_SLICE1]]) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + // CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ1]], %[[ADD1]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR1]], %[[UPDATE1]]) + %write1 = "tf.TensorArrayWriteV3"(%grad1#0, %index, %value, %grad1#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[GVAR1]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[OLD_SLICE2:.*]] = "tf.Slice"(%[[READ2]], + // CHECK: %[[RESHAPE2:.*]] = "tf.Reshape"(%[[VALUE]], + // CHECK: %[[ADD2:.*]] = "tf.AddV2"(%[[RESHAPE2]], %[[OLD_SLICE2]]) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + // CHECK: %[[UPDATE2:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], %[[ADD2]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR1]], %[[UPDATE2]]) + %write2 = "tf.TensorArrayWriteV3"(%grad2#0, %index, %value, %grad2#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[READ3:.*]] = "tf.ReadVariableOp"(%[[GVAR3]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[OLD_SLICE3:.*]] = "tf.Slice"(%[[READ3]], + // CHECK: %[[RESHAPE3:.*]] = "tf.Reshape"(%[[VALUE]], + // CHECK: %[[ADD3:.*]] = "tf.AddV2"(%[[RESHAPE3]], %[[OLD_SLICE3]]) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + // CHECK: %[[UPDATE3:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ3]], %[[ADD3]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR3]], %[[UPDATE3]]) + %write3 = "tf.TensorArrayWriteV3"(%grad3#0, %index, %value, %grad3#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + func.return +} + +// ----- + +// Tests while loop with access to the tensor array defined outside and its +// gradient defined inside. The gradient creation should be moved outside. + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: %[[GVAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: "tf.While"(%[[VAR]], %[[SIZE]], %[[GVAR]]) + %1:2 = "tf.While"(%ta#0, %size) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: "tf.Slice"(%[[READ]], + %read = "tf.TensorArrayReadV3"(%1#0, %index, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor>>, %[[BARG1:.*]]: tensor, %[[BARG2:.*]]: tensor>>) +func.func @while_body(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BARG1]], %[[CONST1]]) + %sub = "tf.Sub"(%arg1, %const1) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[BARG0]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ1]], + // CHECK: "tf.AssignVariableOp"(%[[BARG0]], %[[UPDATE1]]) + %write = "tf.TensorArrayWriteV3"(%arg0, %sub, %elem, %flow) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %write) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[BARG2]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE2:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], + // CHECK: "tf.AssignVariableOp"(%[[BARG2]], %[[UPDATE2]]) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %sub, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: return %[[BARG0]], %[[SUB]], %[[BARG2]] + func.return %arg0, %sub : tensor, tensor +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor>>, %[[CARG1:.*]]: tensor, %[[CARG2:.*]]: tensor>>) +func.func @while_cond(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NEXT: return %[[CARG1]] + func.return %arg1 : tensor +} + +// ----- + +// Tests If op with access to the tensor array defined outside and its gradient +// defined inside. The gradient creation should be moved outside. + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[COND:.*]] = "tf._SomeOp"() : () -> tensor + %cond = "tf._SomeOp"() : () -> tensor + // CHECK: %[[GVAR1:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: %[[GVAR2:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.If"(%[[COND]], %[[VAR]], %[[GVAR1]], %[[GVAR2]]) + %1 = "tf.If"(%cond, %ta#0) { + then_branch = @then_branch, else_branch = @else_branch, device = "", is_stateless = false} + : (tensor, tensor) -> tensor + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: "tf.Slice"(%[[READ]], + %read = "tf.TensorArrayReadV3"(%1, %index, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + // CHECK: %[[READ_GVAR1:.*]] = "tf.ReadVariableOp"(%[[GVAR1]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_GVAR1]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR1]], %[[UPDATE]]) + %const = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %grad:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %const, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + func.return +} +// CHECK: func @then_branch(%[[TARG0:.*]]: tensor>>, %[[TARG1:.*]]: tensor>>, %[[TARG2:.*]]: tensor>>) +func.func @then_branch(%arg0: tensor) -> tensor { + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[TARG1]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ1]], + // CHECK: "tf.AssignVariableOp"(%[[TARG1]], %[[UPDATE1]]) + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %const1, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: return %[[TARG0]] + func.return %arg0 : tensor +} +// CHECK: func @else_branch(%[[EARG0:.*]]: tensor>>, %[[EARG1:.*]]: tensor>>, %[[EARG2:.*]]: tensor>>) +func.func @else_branch(%arg0: tensor) -> tensor { + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[EARG2]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE2:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], + // CHECK: "tf.AssignVariableOp"(%[[EARG2]], %[[UPDATE2]]) + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "b"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %const1, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: return %[[EARG0]] + func.return %arg0 : tensor +} + +// ----- + +// Tests WhileRegion loop with access to the tensor array defined outside and +// its gradient defined inside. + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NOT: tf.TensorArrayV3 + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[FLOW_INIT:.*]] = "tf.Const"() <{value = dense<0.000000e+00> : tensor}> + // CHECK: %[[WHILE:.*]]:2 = "tf.WhileRegion"(%[[FLOW_INIT]], %[[SIZE]]) {{.*}} ({ + %while:2 = "tf.WhileRegion"(%ta#1, %size) ({ + // CHECK: ^bb0(%[[BARG0:.*]]: tensor, %[[BARG1:.*]]: tensor): + ^bb0(%barg0: tensor, %barg1: tensor): + // CHECK: %[[PRED:.*]] = "tf._SomeOp"(%[[BARG1]]) + // CHECK: "tf.Yield"(%[[PRED]]) + %pred = "tf._SomeOp"(%barg1) : (tensor) -> tensor + "tf.Yield" (%pred) : (tensor) -> () + }, { + // CHECK: ^bb0(%[[BARG0:.*]]: tensor, %[[BARG1:.*]]: tensor): + ^bb0(%barg0: tensor, %barg1: tensor): + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %sub = "tf.Sub"(%barg1, %const1) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + // CHECK: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_VAR]], + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[UPDATE]]) + %write = "tf.TensorArrayWriteV3"(%ta#0, %sub, %elem, %flow) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[GVAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %grad:2 = "tf.TensorArrayGradV3"(%ta#0, %write) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[READ_GVAR:.*]] = "tf.ReadVariableOp"(%[[GVAR]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_GVAR]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR]], %[[UPDATE]]) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %sub, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + "tf.Yield"(%gwrite, %sub) : (tensor, tensor) -> () + }) {is_stateless = false} + : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: "tf.Slice"(%[[READ_VAR]] + %read = "tf.TensorArrayReadV3"(%ta#0, %index, %while#0) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} + +// ----- + +// Test IfRegion op. + +// CHECK-LABEL: func @main +// CHECK-SAME: %[[PRED:.*]]: tensor +func.func @main(%arg0: tensor) -> () { + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.TensorArrayV3 + // CHECK: %[[TA_BUFFER:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp"(%[[TA_BUFFER]] + // CHECK-NOT: tf.TensorArrayV3 + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: "tf.IfRegion"(%[[PRED]]) <{is_stateless = false}> ({ + %case_op = "tf.IfRegion"(%arg0) <{is_stateless = false}> ({ + // CHECK: %[[TA_VAL:.*]] = "tf.ReadVariableOp"(%[[TA_BUFFER]]) + // CHECK: "tf.Slice"(%[[TA_VAL]] + // CHECK-NOT: tf.TensorArrayReadV3 + %idx = "tf.Const"() {value = dense<3> : tensor} : () -> tensor + %read = "tf.TensorArrayReadV3"(%ta#0, %idx, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + "tf.Yield"(%ta#1) : (tensor) -> () + // CHECK: }, { + }, { + // CHECK: %[[TA_VAL:.*]] = "tf.ReadVariableOp"(%[[TA_BUFFER]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[TA_VAL]] + // CHECK: "tf.AssignVariableOp"(%[[TA_BUFFER]], %[[UPDATE]]) + // CHECK-NOT: tf.TensorArrayWriteV3 + %idx = "tf.Const"() {value = dense<4> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %write = "tf.TensorArrayWriteV3"(%ta#0, %idx, %elem, %ta#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + "tf.Yield"(%write) : (tensor) -> () + // CHECK: }) : (tensor) -> tensor + }) : (tensor) -> tensor + %idx = "tf.Const"() {value = dense<6> : tensor} : () -> tensor + // CHECK-NOT: tf.TensorArrayReadV3 + %read_val = "tf.TensorArrayReadV3"(%ta#0, %idx, %case_op) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} + +// ----- + +// Tests (Stateful)PartitionedCall op with access to the tensor array defined +// outside and its gradient defined inside. The gradient creation should be +// moved outside. + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[COND:.*]] = "tf._SomeOp"() : () -> tensor + %cond = "tf._SomeOp"() : () -> tensor + // CHECK: %[[GVAR1:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %grad:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[GVAR2:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%[[VAR]], %[[GVAR1]], %[[GVAR2]]) + // CHECK-SAME: f = @callee_tensorarray_decomposed + %call = "tf.StatefulPartitionedCall"(%ta#0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor) -> tensor + // CHECK: "tf.PartitionedCall"(%[[VAR]], %[[GVAR1]], %[[GVAR2]]) + // CHECK-SAME: f = @callee_tensorarray_decomposed + %call2 = "tf.PartitionedCall"(%call) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor) -> tensor + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: "tf.Slice"(%[[READ]], + %read = "tf.TensorArrayReadV3"(%call2, %index, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} +// CHECK-LABEL: func @callee +// CHECK-SAME: (%[[OCARG0:.*]]: tensor) -> tensor +func.func @callee(%arg0: tensor) -> tensor { + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %const1, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + %grad2:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "b"} : (tensor, tensor) -> (tensor, tensor) + %gwrite2 = "tf.TensorArrayWriteV3"(%grad2#0, %const1, %elem, %grad2#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + func.return %arg0 : tensor +} +// CHECK: func private @callee_tensorarray_decomposed(%[[CARG0:.*]]: tensor>>, %[[CARG1:.*]]: tensor>>, %[[CARG2:.*]]: tensor>>) +// CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[CARG1]]) : (tensor>>) -> tensor<5x3xf32> +// CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ1]], +// CHECK: "tf.AssignVariableOp"(%[[CARG1]], %[[UPDATE1]]) +// CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[CARG2]]) : (tensor>>) -> tensor<5x3xf32> +// CHECK: %[[UPDATE2:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], +// CHECK: "tf.AssignVariableOp"(%[[CARG2]], %[[UPDATE2]]) +// CHECK: return %[[CARG0]] + +// ----- + +// Tests (Stateful)PartitionedCall op with private callee function. + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[COND:.*]] = "tf._SomeOp"() : () -> tensor + %cond = "tf._SomeOp"() : () -> tensor + // CHECK: %[[GVAR1:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %grad:2 = "tf.TensorArrayGradV3"(%ta#0, %ta#1) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + // CHECK: %[[GVAR2:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%[[VAR]], %[[GVAR1]], %[[GVAR2]]) + // CHECK-SAME: f = @callee + %call = "tf.StatefulPartitionedCall"(%ta#0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor) -> tensor + // CHECK: "tf.PartitionedCall"(%[[VAR]], %[[GVAR1]], %[[GVAR2]]) + // CHECK-SAME: f = @callee + %call2 = "tf.PartitionedCall"(%call) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor) -> tensor + // CHECK: %[[READ:.*]] = "tf.ReadVariableOp"(%[[VAR]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: "tf.Slice"(%[[READ]], + %read = "tf.TensorArrayReadV3"(%call2, %index, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} +// CHECK: func private @callee(%[[CARG0:.*]]: tensor>>, %[[CARG1:.*]]: tensor>>, %[[CARG2:.*]]: tensor>>) +func.func private @callee(%arg0: tensor) -> tensor { + // CHECK: %[[READ1:.*]] = "tf.ReadVariableOp"(%[[CARG1]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE1:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ1]], + // CHECK: "tf.AssignVariableOp"(%[[CARG1]], %[[UPDATE1]]) + // CHECK: %[[READ2:.*]] = "tf.ReadVariableOp"(%[[CARG2]]) : (tensor>>) -> tensor<5x3xf32> + // CHECK: %[[UPDATE2:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ2]], + // CHECK: "tf.AssignVariableOp"(%[[CARG2]], %[[UPDATE2]]) + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %const1, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + %grad2:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "b"} : (tensor, tensor) -> (tensor, tensor) + %gwrite2 = "tf.TensorArrayWriteV3"(%grad2#0, %const1, %elem, %grad2#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: return %[[CARG0]] + func.return %arg0 : tensor +} + +// ----- + +// Tests PartitionedCall op with no signature change on callee. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %call = "tf.PartitionedCall"() {f = @callee, config = "", config_proto = "", executor_type = ""} : () -> tensor + func.return +} +// CHECK: func private @callee() -> tensor +func.func @callee() -> tensor { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.AssignVariableOp" + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // CHECK: %[[SIZE:.*]] = "tf.Const"() <{value = dense<5> : tensor}> : () -> tensor + %size_out = "tf.TensorArraySizeV3"(%ta#0, %ta#1) : (tensor, tensor) -> tensor + // CHECK: return %[[SIZE]] : tensor + func.return %size_out : tensor +} + +// ----- + +// CHECK-LABEL: func @main +func.func @main() -> () { + // CHECK: "tf.PartitionedCall"() <{config = "", config_proto = "", executor_type = "", f = @callee}> : () -> tensor<*xf32> + %call = "tf.PartitionedCall"() {config = "", config_proto = "", executor_type = "", f = @callee} : () -> (tensor<*xf32>) + func.return +} +func.func private @callee() -> (tensor<*xf32>) { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: %[[LOCAL_VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor>>, tensor) + %index = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %value = "tf.Const"() {value = dense<[1.0, 2.0, 3.0]> : tensor<3xf32>} : () -> tensor<3xf32> + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: "tf.AssignVariableOp"(%[[LOCAL_VAR]], %[[UPDATE]]) : (tensor>>, tensor<5x3xf32>) -> () + %flow = "tf.TensorArrayWriteV3"(%ta#0, %index, %value, %ta#1) : (tensor>>, tensor, tensor<3xf32>, tensor) -> tensor + // CHECK: %[[SLICE:.*]] = "tf.Slice" + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<3> : tensor<1xi32>}> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) + %val = "tf.TensorArrayReadV3"(%ta#0, %index, %ta#1) : (tensor>>, tensor, tensor) -> tensor<*xf32> + // CHECK: %[[CAST:.*]] = tensor.cast %[[ELEM]] : tensor<3xf32> to tensor<*xf32> + // CHECK: return %[[CAST]] : tensor<*xf32> + func.return %val : tensor<*xf32> +} +// ----- + +// Test CaseRegion with gradient inside PartitionedCall Op. The gradient local +// variable should be inserted before the PartitionedCall op. + +// CHECK-LABEL: func @main() +func.func @main() -> () { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[VAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK-NOT: tf.TensorArrayV3 + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %cond = "tf._SomeOp"() : () -> tensor + // CHECK: %[[GVAR:.*]] = "tf.MlirLocalVarOp"() : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%[[VAR]], %[[GVAR]]) + %call = "tf.StatefulPartitionedCall"(%ta#0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor) -> tensor + // CHECK-NOT: tf.TensorArrayReadV3 + %read = "tf.TensorArrayReadV3"(%call, %index, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} + +// CHECK-LABEL: func private @callee +// CHECK-SAME: %[[VAR:.*]]: tensor>>, %[[GVAR:.*]]: tensor>> +func.func private @callee(%arg0: tensor) -> tensor { + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %elem = "tf._SomeOp"() : () -> tensor<3xf32> + %flow = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + // CHECK: %[[BR_INDEX:.*]] = "tf.SomeOp"() : () -> tensor + %branch_index = "tf.SomeOp"() : () -> tensor + // CHECK: "tf.CaseRegion"(%[[BR_INDEX]]) {{.*}} ({ + "tf.CaseRegion"(%branch_index) ({ + // CHECK: %[[READ_GVAR:.*]] = "tf.ReadVariableOp"(%[[GVAR]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_GVAR]], + // CHECK: "tf.AssignVariableOp"(%[[GVAR]], %[[UPDATE]]) + %grad:2 = "tf.TensorArrayGradV3"(%arg0, %flow) {source = "a"} : (tensor, tensor) -> (tensor, tensor) + %gwrite = "tf.TensorArrayWriteV3"(%grad#0, %index, %elem, %grad#1) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + "tf.Yield"() : () -> () + }, { + // CHECK: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: "tf.Slice"(%[[READ_VAR]] + %read = "tf.TensorArrayReadV3"(%arg0, %index, %flow) : (tensor, tensor, tensor) -> tensor<3xf32> + "tf.Yield"() : () -> () + }, { + // CHECK: %[[READ_VAR:.*]] = "tf.ReadVariableOp"(%[[VAR]]) + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[READ_VAR]], + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[UPDATE]]) + %write = "tf.TensorArrayWriteV3"(%arg0, %index, %elem, %flow) : (tensor, tensor, tensor<3xf32>, tensor) -> tensor + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> () + // CHECK: return %[[VAR]] + func.return %arg0 : tensor +} + +// ----- + +// Test the pass reports failure on unknown size. + +func.func @main(%arg0: tensor) -> () { + // expected-error @+1 {{unknown max element count}} + %ta:2 = "tf.TensorArrayV3"(%arg0) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + func.return +} + +// ----- + +// Test the pass reports failure on unknown shape. + +func.func @main(%arg0: tensor) -> () { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // expected-error @+1 {{unknown element shape}} + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + func.return +} + +// ----- + +// Tests that the pass reports error on ambiguous tensor array. + +func.func @main(%arg0: tensor) -> () { + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %ta0:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %ta1:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + %if_op = "tf.If"(%arg0, %ta0#0, %ta1#0) {then_branch = @if_then, else_branch = @if_else, is_stateless = false} + : (tensor, tensor, tensor) -> tensor + %index = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // expected-error @+1 {{unknown tensor array}} + %read = "tf.TensorArrayReadV3"(%if_op, %index, %ta0#1) : (tensor, tensor, tensor) -> tensor<3xf32> + func.return +} +func.func @if_then(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg0 : tensor +} +func.func @if_else(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg1 : tensor +} + +// ----- + +// Tests that the pass returns meaningful error message when region based +// control flow op has resource arguments. +func.func @main() -> () { + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // expected-error @+1 {{found unexpected type 'tensor>>' of operand #0, resource type operands are expected to have been canonicalized away for region based control flow ops}} + %1:2 = "tf.WhileRegion"(%ta#0, %size) ({ + ^bb0 (%carg0: tensor, %carg1: tensor): + %pred = "tf._SomeOp"(%carg1) : (tensor) -> tensor + "tf.Yield"(%pred) : (tensor) -> () + }, { + ^bb0 (%carg0: tensor, %carg1: tensor): + %idx = "tf.Const"() {value = dense<3> : tensor} : () -> tensor + %read_true = "tf.TensorArrayReadV3"(%carg0, %idx, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + "tf.Yield"(%carg0, %idx) : (tensor, tensor) -> () + }) {is_stateless = false} + : (tensor, tensor) -> (tensor, tensor) + func.return +} + +// ----- + +// Tests that the pass returns meaningful error message when region based +// control flow op has resource returns. + +func.func @main(%arg0: tensor) -> (tensor<3xf32>) { + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %ta:2 = "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #tf_type.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + // expected-error @+1 {{found unexpected type 'tensor' of result #1, resource type results are expected to have been canonicalized away for region based control flow ops}} + %if_op:2 = "tf.IfRegion"(%arg0) ({ + %idx = "tf.Const"() {value = dense<3> : tensor} : () -> tensor + %read_true = "tf.TensorArrayReadV3"(%ta#0, %idx, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + "tf.Yield"(%read_true, %ta#0) : (tensor<3xf32>, tensor) -> () + }, { + %idx = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %read_false = "tf.TensorArrayReadV3"(%ta#0, %idx, %ta#1) : (tensor, tensor, tensor) -> tensor<3xf32> + "tf.Yield"(%read_false, %ta#0) : (tensor<3xf32>, tensor) -> () + }) {is_stateless = false} : (tensor) -> (tensor<3xf32>, tensor) + func.return %if_op : tensor<3xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_list_ops_decomposition.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_list_ops_decomposition.mlir new file mode 100644 index 0000000000000000000000000000000000000000..902c7e47709ffcc23f6a96590b39350de338dfaf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tensor_list_ops_decomposition.mlir @@ -0,0 +1,616 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tensor-list-ops-decomposition | FileCheck %s + +// Test push and pop on a tensor list which is initially empty. + +// CHECK-LABEL: func @main +func.func @main() -> (tensor, tensor) { + // CHECK-NEXT: "tf.Const"() <{value = dense<> : tensor<0xi32>}> + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NEXT: "tf.Const"() <{value = dense<10> : tensor}> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NEXT: %[[ZERO_SCALAR:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %[[CAST_ZERO:.*]] = "tf.Cast"(%[[ZERO_SCALAR]]) : (tensor) -> tensor + // CHECK-NEXT: %[[CONST10:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[CAST_ZERO]], %[[CONST10]]) : (tensor, tensor<1xi32>) -> tensor<10xf32> + // CHECK-NEXT: %[[ZERO:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32> + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + %id = "tf.Identity"(%tl) : (tensor>>) -> tensor>> + // CHECK-NEXT: %[[PUSHVAL:.*]] = "tf._SomeOp"() + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NEXT: %[[UPDATE_SHAPE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[UPDATE_SLICE:.*]] = "tf.Reshape"(%[[PUSHVAL]], %[[UPDATE_SHAPE]]) : (tensor, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[BROADCAST]], %[[UPDATE_SLICE]], %[[ZERO]]) : (tensor<10xf32>, tensor<1xf32>, tensor<1xi32>) -> tensor<10xf32> + // CHECK-NEXT: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[NEW_SIZE:.*]] = "tf.AddV2"(%[[ZERO]], %[[CONST1]]) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + %push = "tf.TensorListPushBack"(%id, %elem) : (tensor>>, tensor) -> tensor>> + // CHECK-NEXT: %[[COPY:.*]] = "tf.Identity"(%[[UPDATE]]) + // CHECK-NEXT: %[[CONST1_1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SUB:.*]] = "tf.Sub"(%[[NEW_SIZE]], %[[CONST1_1]]) + // CHECK-NEXT: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK-NEXT: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + %pop:2 = "tf.TensorListPopBack"(%push, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NEXT: %[[SCALAR_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> + // CHECK-NEXT: %[[LENGTH:.*]] = "tf.Reshape"(%[[NEW_SIZE]], %[[SCALAR_SHAPE]]) + %length = "tf.TensorListLength"(%push) : (tensor>>) -> tensor + // CHECK-NEXT: return %[[ELEM]], %[[LENGTH]] : tensor, tensor + func.return %pop#1, %length: tensor, tensor +} + +// ----- + +// Test get and set, and other operations on a tensor list which has reserved +// initial size. + +// CHECK-LABEL: func @main +// CHECK-SAME: (%[[ARG0:.*]]: tensor) -> (tensor, tensor<10xf32>, tensor) +func.func @main(%arg0: tensor) -> (tensor, tensor<10xf32>, tensor) { + // CHECK-NEXT: "tf.Const"() <{value = dense<> : tensor<0xi32>}> + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NEXT: %[[NUM:.*]] = "tf.Const"() <{value = dense<10> : tensor}> + %num = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NEXT: %[[ZERO_SCALAR:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-NEXT: %[[CAST_ZERO:.*]] = "tf.Cast"(%[[ZERO_SCALAR]]) : (tensor) -> tensor + // CHECK-NEXT: %[[CONST10:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[CAST_ZERO]], %[[CONST10]]) : (tensor, tensor<1xi32>) -> tensor<10xf32> + // CHECK-NEXT: %[[SIZE_SHAPE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK-NEXT: %[[SIZE:.*]] = "tf.Reshape"(%[[NUM]], %[[SIZE_SHAPE]]) + %tl = "tf.TensorListReserve"(%elem_shape, %num) : (tensor<0xi32>, tensor) -> tensor>> + // CHECK-NEXT: %[[SETVAL:.*]] = "tf._SomeOp"() + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NEXT: %[[SIZE_SHAPE1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK-NEXT: %[[SET_INDEX:.*]] = "tf.Reshape"(%[[ARG0]], %[[SIZE_SHAPE1]]) : (tensor, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: %[[UPDATE_SHAPE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[UPDATE_SLICE:.*]] = "tf.Reshape"(%[[SETVAL]], %[[UPDATE_SHAPE]]) : (tensor, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[BROADCAST]], %[[UPDATE_SLICE]], %[[SET_INDEX]]) : (tensor<10xf32>, tensor<1xf32>, tensor<1xi32>) -> tensor<10xf32> + %set = "tf.TensorListSetItem"(%tl, %arg0, %elem) : (tensor>>, tensor, tensor) -> tensor>> + // CHECK-NEXT: %[[SIZE_SHAPE2:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK-NEXT: %[[GET_INDEX:.*]] = "tf.Reshape"(%[[ARG0]], %[[SIZE_SHAPE2]]) : (tensor, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE:.*]] = "tf.Slice"(%[[UPDATE]], %[[GET_INDEX]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK-NEXT: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + %get = "tf.TensorListGetItem"(%set, %arg0, %elem_shape) : (tensor>>, tensor, tensor<0xi32>) -> tensor + // CHECK-NEXT: %[[ADDN:.*]] = "tf.AddN"(%[[UPDATE]], %[[BROADCAST]]) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32> + %addn = "tf.AddN"(%set, %tl) : (tensor>>, tensor>>) -> tensor>> + // CHECK-NEXT: %[[ZEROS_LIKE:.*]] = "tf.ZerosLike"(%[[ADDN]]) : (tensor<10xf32>) -> tensor<10xf32> + %zeros-like = "tf.ZerosLike"(%addn) : (tensor>>) -> tensor>> + // CHECK-NEXT: %[[ADDN2:.*]] = "tf.AddN"(%[[ADDN]], %[[ZEROS_LIKE]]) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32> + %addn2 = "tf.AddN"(%addn, %zeros-like) : (tensor>>, tensor>>) -> tensor>> + %stack = "tf.TensorListStack"(%addn2, %elem_shape) : (tensor>>, tensor<0xi32>) -> tensor<10xf32> + // CHECK-NEXT: %[[LEN:.*]] = "tf.Const"() <{value = dense<10> : tensor}> : () -> tensor + %length = "tf.TensorListLength"(%addn2) : (tensor>>) -> tensor + // CHECK-NEXT: return %[[ELEM]], %[[ADDN2]], %[[LEN]] : tensor, tensor<10xf32>, tensor + func.return %get, %stack, %length : tensor, tensor<10xf32>, tensor +} + +// ----- + +// Test get on a tensor list created from a tensor. + +// CHECK-LABEL: func @main +// CHECK-SAME: (%[[ARG0:.*]]: tensor, %[[ARG1:.*]]: tensor<10xf32>) -> tensor +func.func @main(%arg0: tensor, %arg1: tensor<10xf32>) -> tensor { + // CHECK-NEXT: "tf.Const"() <{value = dense<> : tensor<0xi32>}> + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NEXT: %[[BUFFER:.*]] = "tf.Identity"(%[[ARG1]]) : (tensor<10xf32>) -> tensor<10xf32> + // CHECK-NEXT: %[[SIZE:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> : () -> tensor<1xi32> + %tl = "tf.TensorListFromTensor"(%arg1, %elem_shape) : (tensor<10xf32>, tensor<0xi32>) -> tensor>> + // CHECK-NEXT: %[[SIZE_SHAPE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK-NEXT: %[[GET_INDEX:.*]] = "tf.Reshape"(%[[ARG0]], %[[SIZE_SHAPE]]) : (tensor, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %[[SLICE:.*]] = "tf.Slice"(%[[BUFFER]], %[[GET_INDEX]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK-NEXT: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK-NEXT: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + %get = "tf.TensorListGetItem"(%tl, %arg0, %elem_shape) : (tensor>>, tensor, tensor<0xi32>) -> tensor + // CHECK-NEXT: return %[[ELEM]] : tensor + func.return %get: tensor +} + +// ----- + +// Test tensor list element shape op. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<10x8x9xf32>) -> tensor<2xi64> { + %elem_shape = "tf.Const"() {value = dense<[8, 9]> : tensor<2xi32>} : () -> tensor<2xi32> + %tl = "tf.TensorListFromTensor"(%arg0, %elem_shape) : (tensor<10x8x9xf32>, tensor<2xi32>) -> tensor>> + // CHECK: %[[SHAPE:.*]] = "tf.Const"() <{value = dense<[8, 9]> : tensor<2xi64>}> : () -> tensor<2xi64> + %shape = "tf.TensorListElementShape"(%tl) : (tensor>>) -> tensor<2xi64> + // CHECK-NEXT: return %[[SHAPE]] : tensor<2xi64> + func.return %shape: tensor<2xi64> +} + +// ----- + +// Test tensor list gather op. + +// CHECK-LABEL: func @main +// CHECK-SAME: (%[[ARG0:.*]]: tensor<10x8x9xf32>, %[[ARG1:.*]]: tensor<3xi32>) -> tensor<3x8x9xf32> +func.func @main(%arg0: tensor<10x8x9xf32>, %arg1: tensor<3xi32>) -> tensor<3x8x9xf32> { + %elem_shape = "tf.Const"() {value = dense<[8, 9]> : tensor<2xi32>} : () -> tensor<2xi32> + // CHECK: %[[BUFFER:.*]] = "tf.Identity"(%[[ARG0]]) : (tensor<10x8x9xf32>) -> tensor<10x8x9xf32> + %tl = "tf.TensorListFromTensor"(%arg0, %elem_shape) : (tensor<10x8x9xf32>, tensor<2xi32>) -> tensor>> + // CHECK: %[[AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK: %[[GATHER:.*]] = "tf.GatherV2"(%[[BUFFER]], %[[ARG1]], %[[AXIS]]) : (tensor<10x8x9xf32>, tensor<3xi32>, tensor) -> tensor<3x8x9xf32> + %gather = "tf.TensorListGather"(%tl, %arg1, %elem_shape) : (tensor>>, tensor<3xi32>, tensor<2xi32>) -> tensor<3x8x9xf32> + // CHECK-NEXT: return %[[GATHER]] : tensor<3x8x9xf32> + func.return %gather: tensor<3x8x9xf32> +} + +// ----- + +// Test scatter into existing tensor list. + +// CHECK-LABEL: func @main +// CHECK-SAME: (%[[ARG0:.*]]: tensor<10x8x9xf32>, %[[ARG1:.*]]: tensor<5xi32>, %[[ARG2:.*]]: tensor<5x8x9xf32>) -> tensor<10x8x9xf32> +func.func @main(%arg0: tensor<10x8x9xf32>, %arg1: tensor<5xi32>, %arg2: tensor<5x8x9xf32>) -> tensor<10x8x9xf32> { + %elem_shape = "tf.Const"() {value = dense<[8, 9]> : tensor<2xi32>} : () -> tensor<2xi32> + // CHECK: %[[BUFFER:.*]] = "tf.Identity"(%[[ARG0]]) : (tensor<10x8x9xf32>) -> tensor<10x8x9xf32> + %tl = "tf.TensorListFromTensor"(%arg0, %elem_shape) : (tensor<10x8x9xf32>, tensor<2xi32>) -> tensor>> + // CHECK: %[[IND_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 1]> : tensor<2xi32>}> : () -> tensor<2xi32> + // CHECK: %[[IND_RESHPE:.*]] = "tf.Reshape"(%[[ARG1]], %[[IND_SHAPE]]) : (tensor<5xi32>, tensor<2xi32>) -> tensor<5x1xi32> + // CHECK: %[[SC:.*]] = "tf.TensorScatterUpdate"(%[[BUFFER]], %[[IND_RESHPE]], %[[ARG2]]) <{bad_indices_policy = ""}> : (tensor<10x8x9xf32>, tensor<5x1xi32>, tensor<5x8x9xf32>) -> tensor<10x8x9xf32> + %scatter = "tf.TensorListScatterIntoExistingList"(%tl, %arg2, %arg1) : (tensor>>, tensor<5x8x9xf32>, tensor<5xi32>) -> tensor>> + %stack = "tf.TensorListStack"(%scatter, %elem_shape) : (tensor>>, tensor<2xi32>) -> tensor<10x8x9xf32> + // CHECK: return %[[SC]] : tensor<10x8x9xf32> + func.return %stack : tensor<10x8x9xf32> +} + +// ----- + +// Tests while loop. + +// CHECK-LABEL: func @main +func.func @main() -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + %1:2 = "tf.While"(%tl, %max_size) { + body = @while_body, cond = @while_cond, device = "", is_stateless = false} + : (tensor>>, tensor) -> (tensor>>, tensor) + // CHECK: "tf.Slice" + %pop:2 = "tf.TensorListPopBack"(%1#0, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NOT: tf.TensorListPopBack + // CHECK: return + func.return +} +// CHECK: func @while_body(%[[BARG0:.*]]: tensor<10xf32>, %[[BARG1:.*]]: tensor, %[[BARG2:.*]]: tensor<1xi32>) +func.func @while_body(%arg0: tensor>>, %arg1: tensor) -> (tensor>>, tensor) { + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %const1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BARG1]], %[[CONST1]]) + %sub = "tf.Sub"(%arg1, %const1) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[BARG2]], %[[CONST1]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + // CHECK: return %[[UPDATE]], %[[SUB]], %[[ADD]] + func.return %push, %sub : tensor>>, tensor +} +// CHECK: func @while_cond(%[[CARG0:.*]]: tensor<10xf32>, %[[CARG1:.*]]: tensor, %[[CARG2:.*]]: tensor<1xi32>) +func.func @while_cond(%arg0: tensor>>, %arg1: tensor) -> tensor { + // CHECK-NEXT: return %[[CARG1]] + func.return %arg1 : tensor +} + +// ----- + +// Tests IfOp. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + %if_op = "tf.If"(%arg0, %tl) {then_branch = @if_then, else_branch = @if_else, is_stateless = false} + : (tensor, tensor>>) -> tensor>> + // CHECK: "tf.Slice" + %pop:2 = "tf.TensorListPopBack"(%if_op, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NOT: tf.TensorListPopBack + // CHECK: return + func.return +} +// CHECK: func @if_then(%[[TARG0:.*]]: tensor<10xf32>, %[[TARG1:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func @if_then(%arg0: tensor>>) -> tensor>> { + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[TARG1]], %[[CONST1]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + // CHECK: return %[[UPDATE]], %[[ADD]] + func.return %push : tensor>> +} +// CHECK: func @if_else(%[[EARG0:.*]]: tensor<10xf32>, %[[EARG1:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func @if_else(%arg0: tensor>>) -> tensor>> { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NOT: "tf.TensorListPopBack" + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[EARG0]]) + // CHECK: %[[CONST1_1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[EARG1]], %[[CONST1_1]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%arg0, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: return %[[COPY]], %[[SUB]] + func.return %pop#0 : tensor>> +} + +// ----- + +// Tests CaseOp. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + %case_op = "tf.Case"(%arg0, %tl) {branches = [@branch_0, @branch_1, @branch_2], is_stateless = false} + : (tensor, tensor>>) -> tensor>> + // CHECK: "tf.Slice" + %pop:2 = "tf.TensorListPopBack"(%case_op, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NOT: tf.TensorListPopBack + // CHECK: return + func.return +} +// CHECK: func @branch_0(%[[TARG0:.*]]: tensor<10xf32>, %[[TARG1:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func @branch_0(%arg0: tensor>>) -> tensor>> { + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[TARG1]], %[[CONST1]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + // CHECK: return %[[UPDATE]], %[[ADD]] + func.return %push : tensor>> +} +// CHECK: func @branch_1(%[[EARG0:.*]]: tensor<10xf32>, %[[EARG1:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func @branch_1(%arg0: tensor>>) -> tensor>> { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NOT: "tf.TensorListPopBack" + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[EARG0]]) + // CHECK: %[[CONST1_1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[EARG1]], %[[CONST1_1]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%arg0, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: return %[[COPY]], %[[SUB]] + func.return %pop#0 : tensor>> +} +// CHECK: func @branch_2(%[[EARG0:.*]]: tensor<10xf32>, %[[EARG1:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func @branch_2(%arg0: tensor>>) -> tensor>> { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // CHECK-NOT: "tf.TensorListPopBack" + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[EARG0]]) + // CHECK: %[[CONST1_1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[EARG1]], %[[CONST1_1]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) : (tensor<10xf32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xf32> + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) : (tensor<1xf32>, tensor<0xi32>) -> tensor + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%arg0, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: return %[[COPY]], %[[SUB]] + func.return %pop#0 : tensor>> +} + +// ----- + +// CHECK-LABEL: func @main +func.func @main() -> tensor { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %size) : (tensor<0xi32>, tensor) -> tensor>> + %while_op:2 = "tf.WhileRegion"(%tl, %size) <{is_stateless = false}> ({ + // CHECK: ^bb0(%[[CARG0:.*]]: tensor<10xf32>, %[[CARG1:.*]]: tensor, %[[CARG2:.*]]: tensor<1xi32>): + ^bb0(%arg0: tensor>>, %arg1: tensor): + // CHECK: %[[PRED:.*]] = "tf._SomeOp"() + // CHECK: "tf.Yield"(%[[PRED]]) + %pred = "tf._SomeOp"() : () -> tensor + "tf.Yield"(%pred) : (tensor) -> () + }, { + // CHECK: ^bb0(%[[CARG0:.*]]: tensor<10xf32>, %[[CARG1:.*]]: tensor, %[[CARG2:.*]]: tensor<1xi32>): + ^bb0(%arg0: tensor>>, %arg1: tensor): + // CHECK: %[[CST:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[CARG1]], %[[CST]]) + // CHECK: %[[ELEM:.*]] = "tf._SomeOp"() : () -> tensor + %cst = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %sub = "tf.Sub"(%arg1, %cst) : (tensor, tensor) -> tensor + %elem = "tf._SomeOp"() : () -> tensor + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice"(%[[CARG0]] + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[CARG2]], %[[ONE]]) + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: "tf.Yield"(%[[UPDATE]], %[[SUB]], %[[ADD]]) + // CHECK: }) + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + "tf.Yield"(%push, %sub) : (tensor>>, tensor) -> () + }) : (tensor>>, tensor) -> (tensor>>, tensor) + // CHECK: "tf.Slice" + // CHECK-NOT: tf.TensorListPopBack + %pop:2 = "tf.TensorListPopBack"(%while_op#0, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: return + func.return %pop#1 : tensor +} +// ----- + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + // CHECK: %[[ZERO:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK: %[[ZERO_F32:.*]] = "tf.Cast"(%[[ZERO]]) + // CHECK: %[[MAX_SIZE:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> + // CHECK: %[[BUFFER:.*]] = "tf.BroadcastTo"(%[[ZERO_F32]], %[[MAX_SIZE]]) + // CHECK: %[[BUFFER_SIZE:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> + // CHECK-NOT: tf.EmptyTensorList + %if_op = "tf.IfRegion"(%arg0) ({ + %elem = "tf._SomeOp"() : () -> tensor + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[BUFFER_SIZE]], %[[ONE]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%tl, %elem) : (tensor>>, tensor) -> tensor>> + "tf.Yield" (%push) : (tensor>>) -> () + }, { + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[BUFFER]]) + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BUFFER_SIZE]], %[[ONE]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%tl, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: "tf.Yield"(%[[COPY]], %[[SUB]]) + "tf.Yield" (%pop#0) : (tensor>>) -> () + }) + {is_stateless = false} + : (tensor) -> tensor>> + // CHECK: "tf.Slice" + // CHECK-NOT: tf.TensorListPopBack + %pop:2 = "tf.TensorListPopBack"(%if_op, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + func.return +} + +// ----- + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK: %[[ZERO:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK: %[[ZERO_F32:.*]] = "tf.Cast"(%[[ZERO]]) + // CHECK: %[[MAX_SIZE:.*]] = "tf.Const"() <{value = dense<10> : tensor<1xi32>}> + // CHECK: %[[BUFFER:.*]] = "tf.BroadcastTo"(%[[ZERO_F32]], %[[MAX_SIZE]]) + // CHECK: %[[BUFFER_SIZE:.*]] = "tf.Const"() <{value = dense<0> : tensor<1xi32>}> + // CHECK-NOT: tf.EmptyTensorList + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + %case_op = "tf.CaseRegion"(%arg0) ({ + %elem = "tf._SomeOp"() : () -> tensor + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[BUFFER_SIZE]], %[[ONE]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%tl, %elem) : (tensor>>, tensor) -> tensor>> + "tf.Yield" (%push) : (tensor>>) -> () + }, { + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[BUFFER]]) + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BUFFER_SIZE]], %[[ONE]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%tl, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: "tf.Yield"(%[[COPY]], %[[SUB]]) + "tf.Yield" (%pop#0) : (tensor>>) -> () + }, { + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[BUFFER]]) + // CHECK: %[[ONE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SUB:.*]] = "tf.Sub"(%[[BUFFER_SIZE]], %[[ONE]]) + // CHECK: %[[SLICE_SIZE:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> + // CHECK: %[[SLICE:.*]] = "tf.Slice"(%[[COPY]], %[[SUB]], %[[SLICE_SIZE]]) + // CHECK: %[[ELEM_SHAPE:.*]] = "tf.Const"() <{value = dense<> : tensor<0xi32>}> + // CHECK: %[[ELEM:.*]] = "tf.Reshape"(%[[SLICE]], %[[ELEM_SHAPE]]) + // CHECK-NOT: "tf.TensorListPopBack" + %pop:2 = "tf.TensorListPopBack"(%tl, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK: "tf.Yield"(%[[COPY]], %[[SUB]]) + "tf.Yield" (%pop#0) : (tensor>>) -> () + }) {is_stateless = false} + : (tensor) -> tensor>> + // CHECK: "tf.Slice" + // CHECK-NOT: tf.TensorListPopBack + %pop:2 = "tf.TensorListPopBack"(%case_op, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + func.return +} + +// ----- + +// Tests PartitionedCall/StatefulPartitionedCall. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + // CHECK: %[[INIT:.*]] = "tf.BroadcastTo" + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%[[INIT]], + // CHECK-SAME: f = @callee_tensorlist_decomposed + %call = "tf.StatefulPartitionedCall"(%tl, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor>>, tensor) -> tensor>> + // CHECK: %[[CALL2:.*]]:2 = "tf.PartitionedCall"(%[[INIT]], + // CHECK-SAME: f = @callee_tensorlist_decomposed + %call2 = "tf.PartitionedCall"(%tl, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor>>, tensor) -> tensor>> + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[CALL2]]#0) + // CHECK: "tf.Slice"(%[[COPY]], + %pop:2 = "tf.TensorListPopBack"(%call2, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NOT: tf.TensorListPopBack + // CHECK: return + func.return +} + +// CHECK: func @callee(%[[AARG0:.*]]: tensor>>, %[[AARG1:.*]]: tensor) -> tensor>> +func.func @callee(%arg0: tensor>>, %arg1: tensor) -> tensor>> { + %elem = "tf._SomeOp"(%arg1) : (tensor) -> tensor + // CHECK: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + func.return %push : tensor>> +} + +// CHECK: func private @callee_tensorlist_decomposed(%[[ARG0:.*]]: tensor<10xf32>, %[[ARG1:.*]]: tensor, %[[ARG2:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +// CHECK-NOT: "tf.TensorListPushBack" +// CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" +// CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> +// CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[ARG2]], %[[CONST1]]) +// CHECK-NOT: "tf.TensorListPushBack" +// CHECK: return %[[UPDATE]], %[[ADD]] + +// ----- + +// Tests PartitionedCall/StatefulPartitionedCall with private callee function. + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + // CHECK: %[[INIT:.*]] = "tf.BroadcastTo" + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%[[INIT]], + // CHECK-SAME: f = @callee + %call = "tf.StatefulPartitionedCall"(%tl, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor>>, tensor) -> tensor>> + // CHECK: %[[CALL2:.*]]:2 = "tf.PartitionedCall"(%[[INIT]], + // CHECK-SAME: f = @callee + %call2 = "tf.PartitionedCall"(%tl, %arg0) {f = @callee, config = "", config_proto = "", executor_type = ""} + : (tensor>>, tensor) -> tensor>> + // CHECK: %[[COPY:.*]] = "tf.Identity"(%[[CALL2]]#0) + // CHECK: "tf.Slice"(%[[COPY]], + %pop:2 = "tf.TensorListPopBack"(%call2, %elem_shape) : (tensor>>, tensor<0xi32>) -> (tensor>>, tensor) + // CHECK-NOT: tf.TensorListPopBack + // CHECK: return + func.return +} + +// CHECK: func private @callee(%[[ARG0:.*]]: tensor<10xf32>, %[[ARG1:.*]]: tensor, %[[ARG2:.*]]: tensor<1xi32>) -> (tensor<10xf32>, tensor<1xi32>) +func.func private @callee(%arg0: tensor>>, %arg1: tensor) -> tensor>> { + %elem = "tf._SomeOp"(%arg1) : (tensor) -> tensor + + // CHECK-NOT: "tf.TensorListPushBack" + // CHECK: %[[UPDATE:.*]] = "tf.XlaDynamicUpdateSlice" + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK: %[[ADD:.*]] = "tf.AddV2"(%[[ARG2]], %[[CONST1]]) + // CHECK-NOT: "tf.TensorListPushBack" + %push = "tf.TensorListPushBack"(%arg0, %elem) : (tensor>>, tensor) -> tensor>> + // CHECK: return %[[UPDATE]], %[[ADD]] + func.return %push : tensor>> +} + +// ----- + +// Tests PartitionedCall op with no signature change on callee. + +// CHECK-LABEL: func @main +func.func @main() { + "tf.PartitionedCall"() {f = @callee, config = "", config_proto = "", executor_type = ""} : () -> () + func.return +} +// CHECK: func private @callee() +func.func @callee() { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + // CHECK: "tf.BroadcastTo" + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<0xi32>, tensor) -> tensor>> + func.return +} + +// ----- + +// Tests that the pass uses the result type to infer element shape. + +func.func @main(%arg0 : tensor<*xi32>) -> () { + // 1-D element shape with dynamic size + %element_shape = "tf.Const"() {value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // CHECK-NOT: tf.EmptyTensorList + // CHECK: tf.BroadcastTo + // CHECK-SAME: tensor<10x16xf32> + %tl0 = "tf.EmptyTensorList"(%element_shape, %max_size) : (tensor<1xi32>, tensor) -> tensor>> + // CHECK-NOT: tf.TensorListReserve + // CHECK: tf.BroadcastTo + // CHECK-SAME: tensor<10x32xf32> + %tl1 = "tf.TensorListReserve"(%arg0, %max_size) : (tensor<*xi32>, tensor) -> tensor>> + func.return +} + +// ----- + +// Tests that the pass reports error on unknown maximum size. + +func.func @main(%arg0: tensor) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + // expected-error @+1 {{unknown max element count}} + %tl = "tf.EmptyTensorList"(%elem_shape, %arg0) : (tensor<0xi32>, tensor) -> tensor>> + func.return +} + +// ----- + +// Tests that the pass reports error on unknown element shape. + +func.func @main() -> () { + %elem_shape = "tf.Const"() {value = dense<[-1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // expected-error @+1 {{unknown tensor list element shape}} + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<2xi32>, tensor) -> tensor>> + func.return +} + +// ----- + +// Tests that the pass reports error on unknown element shape. + +func.func @main(%arg0: tensor<*xi32>) -> () { + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + // expected-error @+1 {{unknown tensor list element shape}} + %tl = "tf.EmptyTensorList"(%arg0, %max_size) : (tensor<*xi32>, tensor) -> tensor>> + func.return +} + +// ----- + +// Tests that the pass reports error on pushing elements to a fixed-size tenosr +// list. + +func.func @main(%arg0: tensor<*xi32>) -> () { + %elem_shape = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %num = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %tl = "tf.TensorListReserve"(%elem_shape, %num) : (tensor<0xi32>, tensor) -> tensor>> + %elem = "tf._SomeOp"() : () -> tensor + // expected-error @+1 {{cannot push on a fixed-size tensor list}} + %push = "tf.TensorListPushBack"(%tl, %elem) : (tensor>>, tensor) -> tensor>> + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-executor-to-functional.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-executor-to-functional.mlir new file mode 100644 index 0000000000000000000000000000000000000000..be8ec6bfee92492c1793237fc8533814c505977a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-executor-to-functional.mlir @@ -0,0 +1,110 @@ +// RUN: tf-opt -tf-executor-to-functional-conversion %s -split-input-file -verify-diagnostics | FileCheck %s + +func.func @unsupported_op() { + tf_executor.graph { + // expected-error@+1 {{'tf_executor.ControlTrigger' op is not supported for lifting out of tf_executor.graph, expected tf_executor.island}} + %control = tf_executor.ControlTrigger {} + tf_executor.fetch + } + func.return +} + +// ----- + +// CHECK-LABEL: func @empty_graph +// CHECK-NEXT: return +func.func @empty_graph() { + tf_executor.graph { + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @empty_island +// CHECK-NEXT: return +func.func @empty_island() { + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @island_forwarding_result +// CHECK-SAME: ([[ARG_0:%.*]]: tensor) +// CHECK-NEXT: return [[ARG_0]] +func.func @island_forwarding_result(%arg0: tensor) -> tensor { + %graph_result = tf_executor.graph { + %output, %control = tf_executor.island { + tf_executor.yield %arg0 : tensor + } + tf_executor.fetch %output : tensor + } + func.return %graph_result : tensor +} + +// CHECK-LABEL: func @transitive_data_dependencies +// CHECK-SAME: ([[ARG_0:%.*]]: tensor) +// CHECK-NEXT: [[A:%.*]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: [[B:%.*]] = "tf.opB"([[A]]) +// CHECK-NEXT: return [[B]] +func.func @transitive_data_dependencies(%arg0: tensor) -> tensor { + %graph_result = tf_executor.graph { + %output0, %control0 = tf_executor.island { + %a = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %a : tensor + } + %output1, %control1 = tf_executor.island { + %b = "tf.opB"(%output0) : (tensor) -> tensor + tf_executor.yield %b : tensor + } + tf_executor.fetch %output1 : tensor + } + func.return %graph_result : tensor +} + +// CHECK-LABEL: func @transitive_control_dependencies +// CHECK-SAME: ([[ARG_0:%.*]]: tensor) +// CHECK-NEXT: "tf.opA"([[ARG_0]]) +// CHECK-NEXT: [[B:%.*]] = "tf.opB"([[ARG_0]]) +// CHECK-NEXT: return [[B]] +func.func @transitive_control_dependencies(%arg0: tensor) -> tensor { + %graph_result = tf_executor.graph { + %output0, %control0 = tf_executor.island { + %a = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %a : tensor + } + %output1, %control1 = tf_executor.island(%control0) { + %b = "tf.opB"(%arg0) : (tensor) -> tensor + tf_executor.yield %b : tensor + } + tf_executor.fetch %output1 : tensor + } + func.return %graph_result : tensor +} + +// CHECK-LABEL: func @multiple_inner_ops +// CHECK-SAME: ([[ARG_0:%.*]]: tensor) +// CHECK-NEXT: [[A:%.*]] = "tf.opA"([[ARG_0]]) +// CHECK-NEXT: [[B:%.*]] = "tf.opB"([[A]]) +// CHECK-NEXT: [[C:%.*]] = "tf.opC"([[A]]) +// CHECK-NEXT: [[D:%.*]] = "tf.opD"([[C]]) +// CHECK-NEXT: return [[B]], [[D]] +func.func @multiple_inner_ops(%arg0: tensor) -> (tensor, tensor) { + %graph_result:2 = tf_executor.graph { + %output0_0, %output0_1, %control0 = tf_executor.island { + %a = "tf.opA"(%arg0) : (tensor) -> tensor + %b = "tf.opB"(%a) : (tensor) -> tensor + tf_executor.yield %a, %b : tensor, tensor + } + %output1_0, %output1_1, %control1 = tf_executor.island { + %c = "tf.opC"(%output0_0) : (tensor) -> tensor + %d = "tf.opD"(%c) : (tensor) -> tensor + tf_executor.yield %c, %d : tensor, tensor + } + tf_executor.fetch %output0_1, %output1_1 : tensor, tensor + } + func.return %graph_result#0, %graph_result#1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-functional-to-executor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-functional-to-executor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8b4086e1bf84ad36ef620db4ebfeafcc890373da --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-functional-to-executor.mlir @@ -0,0 +1,79 @@ +// RUN: tf-opt -tf-functional-to-executor-conversion %s | FileCheck %s + +func.func @multiple_return(%arg0 : tensor<*xi32>, %arg1 : tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %1 = "tf.Add"(%arg0, %arg1) {} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %2 = "tf.Add"(%1, %arg1) {} : (tensor<*xi32>, tensor) -> tensor<*xi32> + func.return %1, %2 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @multiple_return +// CHECK-SAME: (%[[ARG0:.*]]: tensor<*xi32>, %[[ARG1:.*]]: tensor) -> (tensor<*xi32>, tensor<*xi32>) { +// CHECK: %[[GRAPH_RESULT:.*]]:2 = tf_executor.graph { +// CHECK: %[[ISLAND_RESULT:.*]]:2, {{.*}} = tf_executor.island { +// CHECK: %[[ADD1:.*]] = "tf.Add"(%[[ARG0]], %[[ARG1]]) : (tensor<*xi32>, tensor) -> tensor<*xi32> +// CHECK: %[[ADD2:.*]] = "tf.Add"(%[[ADD1]], %[[ARG1]]) : (tensor<*xi32>, tensor) -> tensor<*xi32> +// CHECK: tf_executor.yield %[[ADD1]], %[[ADD2]] : tensor<*xi32>, tensor<*xi32> +// CHECK: } +// CHECK: tf_executor.fetch %[[ISLAND_RESULT]]#0, %[[ISLAND_RESULT]]#1 : tensor<*xi32>, tensor<*xi32> +// CHECK: } +// CHECK: return %[[GRAPH_RESULT]]#0, %[[GRAPH_RESULT]]#1 : tensor<*xi32>, tensor<*xi32> + +func.func @empty_graph() { + func.return +} + +// CHECK-LABEL: func @empty_graph +// CHECK: tf_executor.graph { +// CHECK: %[[CONTROL:.*]] = tf_executor.island { +// CHECK: tf_executor.yield +// CHECK: } +// CHECK: tf_executor.fetch %[[CONTROL]] : !tf_executor.control +// CHECK: } +// CHECK: return + +func.func @graph_already() { + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + func.return +} + + +// CHECK-LABEL: func @graph_already +// CHECK: tf_executor.graph { +// CHECK: %[[CONTROL:.*]] = tf_executor.island { +// CHECK: tf_executor.yield +// CHECK: } +// CHECK: tf_executor.fetch %[[CONTROL]] : !tf_executor.control +// CHECK: } +// CHECK: return + +func.func @graph_and_more(%arg0: tensor<*xi32>, %arg1: tensor) -> tensor<*xi32> { + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + %result = "tf.Add"(%arg0, %arg1) {} : (tensor<*xi32>, tensor) -> tensor<*xi32> + func.return %result : tensor<*xi32> +} + +// CHECK-LABEL: func @graph_and_more +// CHECK: %[[RESULT:.*]] = tf_executor.graph { +// CHECK: %[[ISLAND:.*]], %[[ISLAND_control:.*]] = tf_executor.island { +// CHECK: tf_executor.graph { +// CHECK: %[[ISLAND_INNER:.*]] = tf_executor.island { +// CHECK: tf_executor.yield +// CHECK: } +// CHECK: tf_executor.fetch %[[ISLAND_INNER]] : !tf_executor.control +// CHECK: } +// CHECK: %[[ADD:.*]] = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> +// CHECK: tf_executor.yield %[[ADD]] : tensor<*xi32> +// CHECK: } +// CHECK: tf_executor.fetch %[[ISLAND]] : tensor<*xi32> +// CHECK: } +// CHECK: return %[[RESULT]] : tensor<*xi32> diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2a2c30c41355e5253f719b0f228a35a85ba86c18 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir @@ -0,0 +1,5268 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics | FileCheck %s + +// Tests for TensorFlow ops with custom verifiers. + +// TODO(hinsu): Remove tests for ops without custom verifiers. These tests were +// added along with manual op definition and are obsolete now that the op +// definitions are auto-generated. + +// TODO(hinsu): Move attribute and type tests to types.mlir file. +//===--------------------------------------------------------------------===// +// Test TF TensorProto attributes +//===--------------------------------------------------------------------===// + +// CHECK-LABEL: func @tensorProtoAttr +func.func @tensorProtoAttr() -> () { +^bb0: +// CHECK: "tf.TensorProtoIntTensor"() {bar = #tf_type : tensor<2x1x4xi32>} : () -> () + "tf.TensorProtoIntTensor"(){bar = #tf_type : tensor<2x1x4xi32>} : () -> () +// CHECK: "tf.TensorProtoFloatTensor"() {bar = #tf_type : tensor<2x1x4xf32>} : () -> () + "tf.TensorProtoFloatTensor"(){bar = #tf_type : tensor<2x1x4xf32>} : () -> () +// CHECK: "tf.TensorProtoStringTensor"() {bar = #tf_type : tensor<2x1x4x!tf_type.string>} : () -> () + "tf.TensorProtoStringTensor"(){bar = #tf_type : tensor<2x1x4x!tf_type.string>} : () -> () +// CHECK: "tf.TensorProtoResourceTensor"() {bar = #tf_type : tensor<2x1x4x!tf_type.resource>} : () -> () + "tf.TensorProtoResourceTensor"(){bar = #tf_type : tensor<2x1x4x!tf_type.resource>} : () -> () + func.return +} + +//===--------------------------------------------------------------------===// +// Test TF placeholder attribute +//===--------------------------------------------------------------------===// + +// CHECK-LABEL: func @placeholderattr +func.func @placeholderattr() -> () +// CHECK: attributes {some_placeholder = #tf_type.placeholder<"foo">} { + attributes {some_placeholder = #tf_type.placeholder<"foo">} { + func.return +} +//===--------------------------------------------------------------------===// +// Test TF operations (tf.*) +//===--------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @testIdentity +func.func @testIdentity(%arg0: tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> { + %0 = "tf.Identity"(%arg0) : (tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + +// ----- + +// CHECK-LABEL: func @testBitcast +func.func @testBitcast(%arg0: tensor<3x4xui16>) -> tensor<3x4x!tf_type.quint16> { + %0 = "tf.Bitcast"(%arg0) : (tensor<3x4xui16>) -> tensor<3x4x!tf_type.quint16> + func.return %0 : tensor<3x4x!tf_type.quint16> +} + +// ----- + +// CHECK-LABEL: func @testBitcast_v2 +func.func @testBitcast_v2(%arg0: tensor<3x2xi16>) -> tensor<3xi32> { + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi16>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testBitcast_v3(%arg0: tensor<3x5xi16>) -> tensor<3xi32> { + // expected-error @+1 {{input rightmost dimension size is not equal to the divisor. the last dimension of input is expected to be 2}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x5xi16>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testBitcast_v4(%arg0: tensor<3x2xi16>) -> tensor<3x2xi32> { + // expected-error @+1 {{rank of input tensor is 2. rank of output tensor is expected to be 1, instead of 2.}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi16>) -> tensor<3x2xi32> + func.return %0 : tensor<3x2xi32> +} + +// ----- + +func.func @testBitcast_v5(%arg0: tensor<3x2xi16>) -> tensor<2xi32> { + // expected-error @+1 {{the 0th dim of output tensor is 2. It is not equal to the one in input tensor, which is 3}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi16>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +// CHECK-LABEL: func @testBitcast_v6 +func.func @testBitcast_v6(%arg0: tensor<3x2xi32>) -> tensor<3x2x2xi16> { + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi32>) -> tensor<3x2x2xi16> + func.return %0 : tensor<3x2x2xi16> +} + +// ----- + +func.func @testBitcast_v7(%arg0: tensor<3x2xi32>) -> tensor<3x2x3xi16> { + // expected-error @+1 {{output rightmost dimension size is not equal to the divisor. the last dimension of output is expected to be 2}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi32>) -> tensor<3x2x3xi16> + func.return %0 : tensor<3x2x3xi16> +} + +// ----- + +func.func @testBitcast_v8(%arg0: tensor<3x2xi32>) -> tensor<3x2xi16> { + // expected-error @+1 {{rank of input tensor is 2. rank of output tensor is expected to be 3, instead of 2.}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi32>) -> tensor<3x2xi16> + func.return %0 : tensor<3x2xi16> +} + +// ----- + +// CHECK-LABEL: func @testBitcast_v9 +func.func @testBitcast_v9(%arg0: tensor<3x2xi32>) -> tensor<3x2xf32> { + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi32>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +func.func @testBitcast_v10(%arg0: tensor<3x2xi32>) -> tensor<3x4xf32> { + // expected-error @+1 {{output tensor shape shall be equal to input tensor shape}} + %0 = "tf.Bitcast"(%arg0) : (tensor<3x2xi32>) -> tensor<3x4xf32> + func.return %0 : tensor<3x4xf32> +} + +// ----- + +// CHECK-LABEL: func @testBitcast_v11 +func.func @testBitcast_v11(%arg0: tensor<10x10x2xf16>) -> tensor<*xf32> { + %0 = "tf.Bitcast"(%arg0) : (tensor<10x10x2xf16>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// CHECK-LABEL: func @testReverseV2 +func.func @testReverseV2(%arg0: tensor<2x4x3xui8>, %arg1: tensor<1xi32>) -> tensor<2x4x3xui8> { + %0 = "tf.ReverseV2"(%arg0, %arg1) : (tensor<2x4x3xui8>, tensor<1xi32>) -> tensor<2x4x3xui8> + func.return %0 : tensor<2x4x3xui8> +} + +// ----- + +func.func @testIdentityWrongType(%arg0: tensor<4x2x!tf_type.string>) -> tensor<4x2x!tf_type.stringref> { + // expected-error @+1 {{all operands and results to have compatible element}} + %0 = "tf.Identity"(%arg0) : (tensor<4x2x!tf_type.string>) -> tensor<4x2x!tf_type.stringref> + func.return %0 : tensor<4x2x!tf_type.stringref> +} + +// ----- + +// TODO(hinsu): Move this to MLIR core once the test dialect have a custom type. + +// Check that broadcastable trait accepts TF specific element type +// CHECK-LABEL: func @testAdd +func.func @testAdd(%arg0: tensor<4x2x!tf_type.string>, %arg1: tensor<2x!tf_type.string>) -> tensor<4x2x!tf_type.string> { + %0 = "tf.Add"(%arg0, %arg1) : (tensor<4x2x!tf_type.string>, tensor<2x!tf_type.string>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + +// ----- + +// Valid BiasAdd operation. +func.func @testBiasAdd(%arg0: tensor<2x3x5x7xf32>, %arg1: tensor<7xf32>) -> tensor<2x3x5x7xf32> { + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NHWC"} : (tensor<2x3x5x7xf32>, tensor<7xf32>) -> tensor<2x3x5x7xf32> + func.return %0 : tensor<2x3x5x7xf32> +} + +// ----- + +func.func @testBiasAddNoDataFormatOk(tensor<1x32x32x16xf32>, tensor<16xf32>) -> tensor<1x32x32x16xf32> { +^bb0(%arg0: tensor<1x32x32x16xf32>, %arg1: tensor<16xf32>): + %0 = "tf.BiasAdd"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT"}: (tensor<1x32x32x16xf32>, tensor<16xf32>) -> tensor<1x32x32x16xf32> + func.return %0 : tensor<1x32x32x16xf32> +} + +// ----- + +func.func @testBiasAddWrongDataFormat(tensor<1x32x32x16xf32>, tensor<16xf32>) -> tensor<1x32x32x16xf32> { +^bb0(%arg0: tensor<1x32x32x16xf32>, %arg1: tensor<16xf32>): + // expected-error @+1 {{attribute 'data_format' failed to satisfy constraint: 'NHWC' or 'NCHW' convnet data format}} + %0 = "tf.BiasAdd"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", data_format = "HWCN"} : (tensor<1x32x32x16xf32>, tensor<16xf32>) -> tensor<1x32x32x16xf32> + func.return %0 : tensor<1x32x32x16xf32> +} + +// ----- + +func.func @testBiasAdd(%arg0: tensor<3xf32>, %arg1: tensor<3xf32>) -> tensor<3xf32> { + // expected-error @+1 {{requires value operand to have rank at least two with `NHWC` data format}} + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NHWC"} : (tensor<3xf32>, tensor<3xf32>) -> tensor<3xf32> + func.return %0 : tensor<3xf32> +} + +// ----- + +func.func @testBiasAdd(%arg0: tensor<2x3xf32>, %arg1: tensor<3xf32>) -> tensor<2x3xf32> { + // expected-error @+1 {{requires value operand to have rank at least three with `NCHW` data format}} + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NCHW"} : (tensor<2x3xf32>, tensor<3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> +} + +// ----- + +func.func @testBiasAdd(%arg0: tensor<2x3x5x7xf32>, %arg1: tensor<5x7xf32>) -> tensor<2x3x5x7xf32> { + // expected-error @+1 {{requires bias operand to have rank exactly one}} + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NHWC"} : (tensor<2x3x5x7xf32>, tensor<5x7xf32>) -> tensor<2x3x5x7xf32> + func.return %0 : tensor<2x3x5x7xf32> +} + +// ----- + +func.func @testBiasAdd(%arg0: tensor<2x3x5x7xf32>, %arg1: tensor<5xf32>) -> tensor<2x3x5x7xf32> { + // expected-error @+1 {{requires channel dimension and feature dimension to match; found 7 and 5, respectively}} + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NHWC"} : (tensor<2x3x5x7xf32>, tensor<5xf32>) -> tensor<2x3x5x7xf32> + func.return %0 : tensor<2x3x5x7xf32> +} + +// ----- + +// Valid BiasAddGrad operation. +func.func @testBiasAddGrad(%arg0: tensor<2x3x5x7xf32>) -> tensor<7xf32> { + %0 = "tf.BiasAddGrad"(%arg0) {data_format = "NHWC"} : (tensor<2x3x5x7xf32>) -> (tensor<7xf32>) + func.return %0 : tensor<7xf32> +} + +// ----- + +func.func @testBiasAddGrad(%arg0: tensor<3xf32>) -> tensor<3xf32> { + // expected-error @+1 {{requires out_backprop operand to have rank at least two with `NHWC` data format}} + %0 = "tf.BiasAddGrad"(%arg0) {data_format = "NHWC"} : (tensor<3xf32>) -> tensor<3xf32> + func.return %0 : tensor<3xf32> +} + +// ----- + +func.func @testBiasAddGrad(%arg0: tensor<2x3xf32>) -> tensor<3xf32> { + // expected-error @+1 {{requires out_backprop operand to have rank at least three with `NCHW` data format}} + %0 = "tf.BiasAddGrad"(%arg0) {data_format = "NCHW"} : (tensor<2x3xf32>) -> tensor<3xf32> + func.return %0 : tensor<3xf32> +} + +// ----- + +// Test valid tf.BroadcastGradientArgs +// CHECK-LABEL: func @testBroadcastGradientArgs +func.func @testBroadcastGradientArgs(%s0: tensor<4xi32>, %s1: tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) { + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<4xi32>, tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsIncompatibleInputType(%s0: tensor<4xi32>, %s1: tensor<4xi64>) -> (tensor<1xi32>, tensor<0xi32>) { + // expected-error @+1 {{requires the same element type for all operands and results}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<4xi32>, tensor<4xi64>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsIncompatibleBroadcastShape() -> (tensor<1xi32>, tensor<0xi32>) { + %s0 = "tf.Const"() {value = dense<[4, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %s1 = "tf.Const"() {value = dense<[2, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires broadcast compatible shape tensors for 's0' and 's1', but got dense<[4, 1]> : tensor<2xi32> and dense<[2, 4]> : tensor<2xi32>}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2xi32>, tensor<2xi32>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidS0Rank() -> (tensor<1xi32>, tensor<0xi32>) { + %s0 = "tf.Const"() {value = dense<[[4, 1], [2, 3]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + %s1 = "tf.Const"() {value = dense<[2, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{failed to verify that operand 0 is 1-D}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2x2xi32>, tensor<2xi32>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidS1Rank() -> (tensor<1xi32>, tensor<0xi32>) { + %s0 = "tf.Const"() {value = dense<[4, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %s1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // expected-error @+1 {{failed to verify that operand 1 is 1-D}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2xi32>, tensor) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidR0Rank() -> (tensor<2x2xi32>, tensor<0xi32>) { + %s0 = "tf.Const"() {value = dense<[4, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %s1 = "tf.Const"() {value = dense<[4, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{failed to verify that result 0 is 1-D}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2xi32>, tensor<2xi32>) -> (tensor<2x2xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<2x2xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidR1Rank(%s0: tensor<4xi32>, %s1: tensor<4xi32>) -> (tensor<1xi32>, tensor) { + // expected-error @+1 {{failed to verify that result 1 is 1-D}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<4xi32>, tensor<4xi32>) -> (tensor<1xi32>, tensor) + func.return %r0, %r1 : tensor<1xi32>, tensor +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidR0Size() -> (tensor<0xi32>, tensor<0xi32>) { + %s0 = "tf.Const"() {value = dense<[4, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + %s1 = "tf.Const"() {value = dense<[4, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires dimension 0 size of 'r0' to be 1 but got 0}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2xi32>, tensor<2xi32>) -> (tensor<0xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<0xi32>, tensor<0xi32> +} + +// ----- + +func.func @testBroadcastGradientArgsInvalidR1Size() -> (tensor<0xi32>, tensor<3xi32>) { + %s0 = "tf.Const"() {value = dense<[4, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + %s1 = "tf.Const"() {value = dense<[1, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires dimension 0 size of 'r1' to be 2 but got 3}} + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<2xi32>, tensor<2xi32>) -> (tensor<0xi32>, tensor<3xi32>) + func.return %r0, %r1 : tensor<0xi32>, tensor<3xi32> +} + +// ----- + +// Test valid tf.BroadcastTo +// CHECK-LABEL: func @testBroadcastTo(%arg0: tensor<16xf32>) +func.func @testBroadcastTo(%arg0: tensor<16xf32>) -> tensor<16x16x16x16xf32> { + %cst = arith.constant dense<16> : tensor<4xi32> + %0 = "tf.BroadcastTo"(%arg0, %cst) : (tensor<16xf32>, tensor<4xi32>) -> tensor<16x16x16x16xf32> + func.return %0 : tensor<16x16x16x16xf32> +} + +// ----- + +// Test valid tf.LeakyRelu +// CHECK-LABEL: func @testLeakyRelu(%arg0: tensor<16xf32>) +func.func @testLeakyRelu(tensor<16xf32>) -> tensor<16xf32> { +^bb0(%arg0: tensor<16xf32>): + %0 = "tf.LeakyRelu"(%arg0) {alpha = 0.2 : f32} : (tensor<16xf32>) -> tensor<16xf32> + func.return %0 : tensor<16xf32> +} + +// ----- +func.func @testLeakyWrongAlphaType(tensor<16xf32>) -> tensor<16xf32> { +^bb0(%arg0: tensor<16xf32>): + // expected-error @+1 {{attribute 'alpha' failed to satisfy constraint: 32-bit float}} + %0 = "tf.LeakyRelu"(%arg0) {alpha = 1: i32}: (tensor<16xf32>) -> tensor<16xf32> + func.return %0 : tensor<16xf32> +} + +// ----- + +// Test tf.Min with complex numbers. +// Previous versions of tensorflow said complex numbers were allowed with +// tf.Min even though it doesn't make sense. The legalization of tf to xla +// requires that complex types are not allowed in tf.Min, so we have an +// explicit unit here to make sure that invariant is enforced. +func.func @testMinComplex(%arg0: tensor<4x8xcomplex>) -> tensor<4x1xcomplex> { + %dimension = "tf.Const"() { value = dense<1> : tensor<1xi64> } : () -> tensor<1xi64> + // expected-error@below {{'tf.Min' op operand #0 must be tensor of}} + %0 = "tf.Min"(%arg0, %dimension) { keep_dims = true }: (tensor<4x8xcomplex>, tensor<1xi64>) -> tensor<4x1xcomplex> + func.return %0 : tensor<4x1xcomplex> +} + +// ----- + +// CHECK-LABEL: func @testMul +func.func @testMul(%arg0: tensor<2xui16>) -> (tensor<2xui16>) { + %0 = "tf.Mul"(%arg0, %arg0) {T = "tfdtype$DT_UINT16", device = "/device:CPU:0", name = "Mul"} : (tensor<2xui16>, tensor<2xui16>) -> tensor<2xui16> + func.return %0 : tensor<2xui16> +} + +// ----- + +// Test error message for incompatible element types. +func.func @testIncompatibleElementTypes(%arg0: tensor<3x2xf32>, %arg1: tensor<3x2xf64>) -> (tensor<3x2xf32>) { + // expected-error @+1 {{'tf.Mul' op requires compatible element types for all operands and results}} + %0 = "tf.Mul"(%arg0, %arg1) : (tensor<3x2xf32>, tensor<3x2xf64>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +// Test error message for incompatible element types. +func.func @testIncompatibleElementTypes(%arg0: tensor<3x2xf32>, %arg1: tensor<3x2xf32>) -> (tensor<3x2xf64>) { + // expected-error @+1 {{'tf.Mul' op requires compatible element types for all operands and results}} + %0 = "tf.Mul"(%arg0, %arg1) : (tensor<3x2xf32>, tensor<3x2xf32>) -> tensor<3x2xf64> + func.return %0 : tensor<3x2xf64> +} + +// ----- + +func.func @testPadRank1Paddings(%input: tensor<2xi64>) -> tensor<3xi64> { + %paddings = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi64>} : () -> tensor<2xi64> + // expected-error @+1 {{failed to verify that operand 1 is 2-D}} + %0 = "tf.Pad"(%input, %paddings) : (tensor<2xi64>, tensor<2xi64>) -> tensor<3xi64> + func.return %0 : tensor<3xi64> +} + +// ----- + +func.func @testPadV2Rank1Paddings(%input: tensor<2xi64>) -> tensor<3xi64> { + %constant = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %paddings = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi64>} : () -> tensor<2xi64> + // expected-error @+1 {{failed to verify that operand 1 is 2-D}} + %0 = "tf.PadV2"(%input, %paddings, %constant) : (tensor<2xi64>, tensor<2xi64>, tensor) -> tensor<3xi64> + func.return %0 : tensor<3xi64> +} + +// ----- + +func.func @testMirrorPadRank1Paddings(%input: tensor<2xi64>) -> tensor<3xi64> { + %paddings = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi64>} : () -> tensor<2xi64> + // expected-error @+1 {{failed to verify that operand 1 is 2-D}} + %0 = "tf.MirrorPad"(%input, %paddings) { mode = "SYMMETRIC" }: (tensor<2xi64>, tensor<2xi64>) -> tensor<3xi64> + func.return %0 : tensor<3xi64> +} + +// ----- + +// CHECK-LABEL: func @testReshape(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<10000xf32>, %arg3: tensor<*xi32>) +func.func @testReshape(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<10000xf32>, %arg3: tensor<*xi32>) -> (tensor<100x100xf32>, tensor<*xf32>, tensor<100x100xf32>, tensor<100x100xf32>, tensor<*xf32>, tensor<*xf32>) { + %shape1 = arith.constant dense<100> : tensor<2xi32> + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<*xf32>, tensor<2xi32>) -> tensor<100x100xf32> + %shape2 = "tf.Shape"(%arg0) : (tensor<*xf32>) -> tensor + %r2 = "tf.Reshape"(%arg1, %shape2) : (tensor<*xf32>, tensor) -> tensor<*xf32> + %r3 = "tf.Reshape"(%arg2, %shape1) : (tensor<10000xf32>, tensor<2xi32>) -> tensor<100x100xf32> + %shape3 = arith.constant dense<[-1, 100]> : tensor<2xi32> + %r4 = "tf.Reshape"(%arg2, %shape3) : (tensor<10000xf32>, tensor<2xi32>) -> tensor<100x100xf32> + %r5 = "tf.Reshape"(%arg0, %arg3) : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xf32> + %r6 = "tf.Reshape"(%arg2, %arg3) : (tensor<10000xf32>, tensor<*xi32>) -> tensor<*xf32> + func.return %r1, %r2, %r3, %r4, %r5, %r6: tensor<100x100xf32>, tensor<*xf32>, tensor<100x100xf32>, tensor<100x100xf32>, tensor<*xf32>, tensor<*xf32> +} + +// ----- +// tf.Reshape with incorrect type. +func.func @testReshape(tensor<*xf32>, tensor<*xf32>) -> (tensor<100x100xf32>) { +^bb0(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>): + %shape1 = arith.constant dense<100.> : tensor<2xf32> + // expected-error @+1 {{must be tensor of 32/64-bit signed integer values}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<*xf32>, tensor<2xf32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with incorrect element number. +func.func @testReshape(%arg0: tensor<10x10x10xf32>, %shape1: tensor<2xi32>) -> tensor<100x100xf32> { + // expected-error @+1 {{requires 'output' number of elements to match 'tensor' number of elements, but got 10000 and 1000}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with incorrect shape operand rank. +func.func @testReshape(%arg0: tensor<10x10x10xf32>, %shape1: tensor<2x2xi32>) -> tensor<*xf32> { + // expected-error @+1 {{requires 'shape' to be rank 1, but got 2}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10xf32>, tensor<2x2xi32>) -> tensor<*xf32> + func.return %r1 : tensor<*xf32> +} + +// ----- +// tf.Reshape with more than one -1 in the shape. +func.func @testReshape(%arg0: tensor<10x10x10x10xf32>) -> tensor<100x100xf32> { + %shape1 = arith.constant dense<-1> : tensor<2xi32> + // expected-error @+1 {{requires 'shape' to have at most one dynamic dimension, but got multiple dynamic dimensions at indices 0 and 1}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10x10xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with shape operand element < -1. +func.func @testReshape(%arg0: tensor<10x10x10x10xf32>) -> tensor<100x100xf32> { + %shape1 = arith.constant dense<[100, -2]> : tensor<2xi32> + // expected-error @+1 {{requires 'shape' to have dimensions greater than -1, but got -2 at index 1}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10x10xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with -1 in the shape can't infer the dimension. +func.func @testReshape(%arg0: tensor<10x10x10x10xf32>) -> tensor<100x100xf32> { + %shape1 = arith.constant dense<[101, -1]> : tensor<2xi32> + // expected-error @+1 {{requires 'tensor' number of elements be a multiple of 101, but got 10000}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10x10xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with incorrect output rank. +func.func @testReshape(%arg0: tensor<10x10xf32>) -> tensor { + %shape1 = arith.constant dense<[100]> : tensor<1xi32> + // expected-error @+1 {{requires 'output' type 'tensor' to be cast compatible with expected type 'tensor<100xf32>'}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10xf32>, tensor<1xi32>) -> tensor + func.return %r1 : tensor +} + +// ----- +// tf.Reshape with incorrect output dimension. +func.func @testReshape(%arg0: tensor<1000xf32>) -> tensor { + %shape1 = arith.constant dense<[10, 10, 10]> : tensor<3xi32> + // expected-error @+1 {{requires 'output' type 'tensor' to be cast compatible with expected type 'tensor<10x10x10xf32>'}} + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<1000xf32>, tensor<3xi32>) -> tensor + func.return %r1 : tensor +} + +// ----- +// tf.Reshape with a shape operand that has 0 for one of its elements. +func.func @testReshape(%arg0: tensor<10x10x10xf32>) -> tensor { + %shape1 = arith.constant dense<[-1, 0]> : tensor<2xi32> + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10xf32>, tensor<2xi32>) -> tensor + func.return %r1 : tensor +} + +// ----- +// tf.Reshape with a tensor operand that has 0 for one of its elements. +func.func @testReshape(%arg0: tensor<10x10x0xf32>) -> tensor { + %shape1 = arith.constant dense<[-1, 0]> : tensor<2xi32> + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x0xf32>, tensor<2xi32>) -> tensor + func.return %r1 : tensor +} + +// ----- +// tf.Reshape with a tensor operand that has non-static shape. +func.func @testReshape(%arg0: tensor<10x10x?xf32>) -> tensor<10x10xf32> { + %shape1 = arith.constant dense<[10, 10]> : tensor<2xi32> + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x?xf32>, tensor<2xi32>) -> tensor<10x10xf32> + func.return %r1 : tensor<10x10xf32> +} + +// ----- +// tf.Reshape with tensor operand that has non-static shape and shape operand +// with static shape. +func.func @testReshape(%arg0: tensor<10x10x?xf32>, %shape1: tensor<2xi32>) -> tensor<100x100xf32> { + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x?xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- +// tf.Reshape with tensor and shape operands with static shape. +func.func @testReshape(%arg0: tensor<10x10x10x10xf32>, %shape1: tensor<2xi32>) -> tensor<100x100xf32> { + %r1 = "tf.Reshape" (%arg0, %shape1) : (tensor<10x10x10x10xf32>, tensor<2xi32>) -> tensor<100x100xf32> + func.return %r1 : tensor<100x100xf32> +} + +// ----- + +// CHECK-LABEL: func @testValidAvgPool +func.func @testValidAvgPool(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +// CHECK-LABEL: func @testAvgPoolMissingDataFormatOk +func.func @testAvgPoolMissingDataFormatOk(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongDataType(tensor<1x7x7x16xi32>) -> tensor<1x1x1x16xi32> { +^bb0(%arg0: tensor<1x7x7x16xi32>): + // expected-error @+1 {{must be tensor of floating-point values}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_INT", data_format = "NHWC", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xi32>) -> tensor<1x1x1x16xi32> + func.return %0 : tensor<1x1x1x16xi32> +} + +// ----- + +func.func @testAvgPoolWrongDataFormat(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{attribute 'data_format' failed to satisfy constraint: 'NHWC' or 'NCHW' convnet data format}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", data_format = "HWCN", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolNoKsize(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{requires attribute 'ksize'}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongKsizeCount(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{attribute 'ksize' failed to satisfy constraint: 64-bit integer array attribute with at least 4 elements}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [7, 7, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongKsizeType(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{'ksize' failed to satisfy constraint: 64-bit integer array attribute with at least 4 elements}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7.5, 1], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- +func.func @testAvgPoolWrongKsizeIntType(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{'ksize' failed to satisfy constraint: 64-bit integer array attribute with at least 4 elements}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1 : i32, 7 : i32, 7 : i32, 1 : i32], padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolNoPadding(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{requires attribute 'padding'}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongPadding(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{attribute 'padding' failed to satisfy constraint: string attribute whose value is SAME, or VALID}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], padding = "MAGIC", strides = [1, 1, 1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolNoStrides(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{requires attribute 'strides'}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], padding = "VALID"} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongStridesCount(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{attribute 'strides' failed to satisfy constraint: 64-bit integer array attribute with at least 4 elements}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 1]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +func.func @testAvgPoolWrongStridesType(tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> { +^bb0(%arg0: tensor<1x7x7x16xf32>): + // expected-error @+1 {{attribute 'strides' failed to satisfy constraint: 64-bit integer array attribute with at least 4 elements}} + %0 = "tf.AvgPool"(%arg0) {T = "tfdtype$DT_FLOAT", ksize = [1, 7, 7, 1], padding = "VALID", strides = ["1", "1", "1", "1"]} : (tensor<1x7x7x16xf32>) -> tensor<1x1x1x16xf32> + func.return %0 : tensor<1x1x1x16xf32> +} + +// ----- + +// CHECK-LABEL: func @testValidConv2D +func.func @testValidConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> { + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +// CHECK-LABEL: func @testValidDynamicConv2D +func.func @testValidDynamicConv2D(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor { + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<*xf32>, tensor<*xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: func @testValidConv3D +func.func @testValidConv3D(%arg0: tensor<256x32x32x32x3xf32>, %arg1: tensor<3x3x3x3x16xf32>) -> tensor<256x32x32x32x16xf32> { + %0 = "tf.Conv3D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1, 1]} : (tensor<256x32x32x32x3xf32>, tensor<3x3x3x3x16xf32>) -> tensor<256x32x32x32x16xf32> + func.return %0 : tensor<256x32x32x32x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> { + // expected-error @+1 {{requires operands to be 4D tensor}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +func.func @testConv3D(%arg0: tensor<256x32x32x32x3xf32>, %arg1: tensor<3x3x3x3x16xf32>) -> tensor<256x32x32x16xf32> { + // expected-error @+2 {{'tf.Conv3D' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor<256x32x32x32x16xf32>' are incompatible with return type(s) of operation 'tensor<256x32x32x16xf32>'}} + %0 = "tf.Conv3D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1, 1]} : (tensor<256x32x32x32x3xf32>, tensor<3x3x3x3x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x2x16xf32>) -> tensor<256x32x32x16xf32> { + // expected-error @+1 {{requires the number of input channels to be divisible by the number of filter input channels; found 3 and 2, respectively}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x2x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires attribute 'explicit_paddings'}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "EXPLICIT", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires explicit_paddings attribute length to be 8}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "EXPLICIT", strides = [1, 1, 1, 1], explicit_paddings = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires non negative explicit paddings}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "EXPLICIT", strides = [1, 1, 1, 1], explicit_paddings = [0, 0, 1, -1, 1, -1, 0, 0]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires strides attribute length to be 4}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires positive strides}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [0, 1, 1, 0]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+2 {{'tf.Conv2D' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor<256x16x11x16xf32>' are incompatible with return type(s) of operation 'tensor<256x30x30x16xf32>'}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 2, 3, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x16x30x16xf32> { + // expected-error @+2 {{'tf.Conv2D' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor<256x16x11x16xf32>' are incompatible with return type(s) of operation 'tensor<256x16x30x16xf32>'}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 2, 3, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x16x30x16xf32> + func.return %0 : tensor<256x16x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> { + // expected-error @+2 {{'tf.Conv2D' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor<256x6x6x16xf32>' are incompatible with return type(s) of operation 'tensor<256x32x32x16xf32>'}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "EXPLICIT", dilations = [1, 2, 3, 4], explicit_paddings = [1, 2, 3, 4, 5, 6, 7, 8], strides = [5, 6, 7, 8]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> { + // expected-error @+2 {{'tf.Conv2D' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor<256x30x30x16xf32>' are incompatible with return type(s) of operation 'tensor<256x32x32x16xf32>'}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "VALID", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x32x32x16xf32> + func.return %0 : tensor<256x32x32x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires dilations attribute length to be 4}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1], dilations = [1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testConv2D(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> { + // expected-error @+1 {{requires positive dilations}} + %0 = "tf.Conv2D"(%arg0, %arg1) {padding = "SAME", strides = [1, 1, 1, 1], dilations = [1, 1, 0, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x16xf32>) -> tensor<256x30x30x16xf32> + func.return %0 : tensor<256x30x30x16xf32> +} + +// ----- + +func.func @testMaxPoolGrad(%orig_input: tensor, %orig_output: tensor<10x12x12x64xf32>, %grad: tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> { + // expected-error @+1 {{requires orig_input to be rank 4}} + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + ksize = [1, 2, 2, 1], + padding = "VALID", + strides = [1, 2, 2, 1] + } : (tensor, tensor<10x12x12x64xf32>, tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> + func.return %result : tensor<10x24x24x64xf32> +} + +// ----- + +func.func @testMaxPoolGrad(%orig_input: tensor<10x24x24x64xf32>, %orig_output: tensor<12x12x64xf32>, %grad: tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> { + // expected-error @+1 {{requires orig_output to be rank 4}} + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + ksize = [1, 2, 2, 1], + padding = "VALID", + strides = [1, 2, 2, 1] + } : (tensor<10x24x24x64xf32>, tensor<12x12x64xf32>, tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> + func.return %result : tensor<10x24x24x64xf32> +} + +// ----- + +func.func @testMaxPoolGrad(%orig_input: tensor<10x24x24x64xf32>, %orig_output: tensor<10x12x12x64xf32>, %grad: tensor<12x12x64xf32>) -> tensor<10x24x24x64xf32> { + // expected-error @+1 {{requires grad to be rank 4}} + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + ksize = [1, 2, 2, 1], + padding = "VALID", + strides = [1, 2, 2, 1] + } : (tensor<10x24x24x64xf32>, tensor<10x12x12x64xf32>, tensor<12x12x64xf32>) -> tensor<10x24x24x64xf32> + func.return %result : tensor<10x24x24x64xf32> +} + +// ----- + +// CHECK-LABEL: func @testValidDepthwiseConv2dNative +func.func @testValidDepthwiseConv2dNative(tensor<256x32x32x3xf32>, tensor<3x3x3x4xf32>) -> tensor<256x30x30x12xf32> { +^bb0(%arg0: tensor<256x32x32x3xf32>, %arg1: tensor<3x3x3x4xf32>) : + %0 = "tf.DepthwiseConv2dNative"(%arg0, %arg1) {device = "", name = "MobilenetV2/expanded_conv/depthwise/depthwise", T = "tfdtype$DT_FLOAT", data_format = "NHWC", dilations = [1, 1, 1, 1], padding = "SAME", strides = [1, 1, 1, 1]} : (tensor<256x32x32x3xf32>, tensor<3x3x3x4xf32>) -> tensor<256x30x30x12xf32> + func.return %0 : tensor<256x30x30x12xf32> +} + +// ----- + +// Test valid tf.FakeQuantWithMinMaxArgs +// CHECK-LABEL: func @testValidFakeQuantWithMinMaxArgs +func.func @testValidFakeQuantWithMinMaxArgs(tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>): + %0 = "tf.FakeQuantWithMinMaxArgs"(%arg0) {min = -1.0 : f32, max = 1.0 : f32, num_bits = 3} : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxArgs +func.func @testInvalidFakeQuantWithMinMaxArgsWrongAttr(tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>): + // expected-error @+1 {{requires num_bits to be between 2 and 16, inclusive}} + %0 = "tf.FakeQuantWithMinMaxArgs"(%arg0) {min = -1.0 : f32, max = 1.0 : f32, num_bits = 0} : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test valid tf.FakeQuantWithMinMaxVars +// CHECK-LABEL: func @testValidFakeQuantWithMinMaxVars +func.func @testValidFakeQuantWithMinMaxVars(tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor, %arg2: tensor): + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) : (tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVars +func.func @testInvalidFakeQuantWithMinMaxVarsWrongAttr(tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor, %arg2: tensor): + // expected-error @+1 {{requires num_bits to be between 2 and 16, inclusive}} + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) {min = -1.0 : f32, max = 1.0 : f32, num_bits = 0} : (tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVars +func.func @testInvalidFakeQuantWithMinMaxVarsWrongMinRank(tensor<8x8x8x8xf32>, tensor<1xf32>, tensor<2xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<1xf32>, %arg2: tensor<2xf32>): + // expected-error @+1 {{requires min to be a 0d float tensor}} + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) : (tensor<8x8x8x8xf32>, tensor<1xf32>, tensor<2xf32>) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVars +func.func @testInvalidFakeQuantWithMinMaxVarsWrongMaxRank(tensor<8x8x8x8xf32>, tensor, tensor<2xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor, %arg2: tensor<2xf32>): + // expected-error @+1 {{requires max to be a 0d float tensor}} + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) : (tensor<8x8x8x8xf32>, tensor, tensor<2xf32>) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVars +func.func @testInvalidFakeQuantWithMinMaxVarsWrongMinType(tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor, %arg2: tensor): + // expected-error @+1 {{op operand #1 must be tensor of 32-bit float values}} + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) : (tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVars +func.func @testInvalidFakeQuantWithMinMaxVarsWrongMaxType(tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor, %arg2: tensor): + // expected-error @+1 {{op operand #2 must be tensor of 32-bit float values}} + %0 = "tf.FakeQuantWithMinMaxVars"(%arg0, %arg1, %arg2) : (tensor<8x8x8x8xf32>, tensor, tensor) -> tensor<8x8x8x8xf32> + func.return %0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test valid tf.FakeQuantWithMinMaxVarsPerChannel +// CHECK-LABEL: func @FakeQuantWithMinMaxVarsPerChannel +func.func @FakeQuantWithMinMaxVarsPerChannel(tensor<1x2x3x8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<1x2x3x8xf32> { +^bb0(%arg0: tensor<1x2x3x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>): + %0 = "tf.FakeQuantWithMinMaxVarsPerChannel"(%arg0, %arg1, %arg2) : (tensor<1x2x3x8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<1x2x3x8xf32> + func.return %0 : tensor<1x2x3x8xf32> +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVarsPerChannel +func.func @FakeQuantWithMinMaxVarsPerChannel_ranked_inputs(tensor, tensor<8xf32>, tensor<8xf32>) -> tensor { +^bb0(%arg0: tensor, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>): + // expected-error @+1 {{requires inputs to be at least 1d float tensor}} + %0 = "tf.FakeQuantWithMinMaxVarsPerChannel"(%arg0, %arg1, %arg2) : (tensor, tensor<8xf32>, tensor<8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.FakeQuantWithMinMaxVarsPerChannel +func.func @FakeQuantWithMinMaxVarsPerChannel_mismatch_min_max(tensor<1x2x3x8xf32>, tensor<1xf32>, tensor<8xf32>) -> tensor<1x2x3x8xf32> { +^bb0(%arg0: tensor<1x2x3x8xf32>, %arg1: tensor<1xf32>, %arg2: tensor<8xf32>): + // expected-error @+1 {{requires min and max to have same size as last dimension of inputs}} + %0 = "tf.FakeQuantWithMinMaxVarsPerChannel"(%arg0, %arg1, %arg2) : (tensor<1x2x3x8xf32>, tensor<1xf32>, tensor<8xf32>) -> tensor<1x2x3x8xf32> + func.return %0 : tensor<1x2x3x8xf32> +} + +// ----- + +// Test invalid tf.Fill +func.func @testFill(tensor, tensor) -> tensor { +^bb0(%arg0: tensor, %arg1: tensor): + // expected-error @+1 {{requires dims to be a 1D tensor}} + %0 = "tf.Fill"(%arg0, %arg1) : (tensor, tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.Fill +func.func @testFill(tensor<2xi32>, tensor<1xf32>) -> tensor { +^bb0(%arg0: tensor<2xi32>, %arg1: tensor<1xf32>): + // expected-error @+1 {{requires value to be a scalar}} + %0 = "tf.Fill"(%arg0, %arg1) : (tensor<2xi32>, tensor<1xf32>) -> (tensor) + func.return %0 : tensor +} + +// ----- + +// Test valid tf.FusedBatchNorm +// CHECK-LABEL: func @testFusedBatchNorm +func.func @testFusedBatchNorm(tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>): + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FusedBatchNorm +func.func @testFusedBatchNormWrongXType(tensor<8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>): + // expected-error @+1 {{requires x to be a 4D float tensor}} + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8xf32> +} + +// ----- + +// Test invalid tf.FusedBatchNorm +func.func @testFusedBatchNormWrongScaleType(tensor<8x8x8x8xf32>, tensor<8xi32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xi32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>): + // expected-error @+1 {{operand #1 must be tensor of 32-bit float values}} + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xi32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xi32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// Test invalid tf.FusedBatchNorm +func.func @testFusedBatchNormWrongOffsetType(tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<2x8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<2x8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>): + // expected-error @+1 {{requires offset to be a 1D float tensor}} + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<2x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<2x8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- +// Test invalid tf.FusedBatchNorm +func.func @testFusedBatchNormWrongMeanType(tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor, tensor<8xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor, %arg4: tensor<8xf32>): + // expected-error @+1 {{requires mean to be a 1D float tensor}} + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- +// Test invalid tf.FusedBatchNorm +func.func @testFusedBatchNormWrongVarianceType(tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<10x2xf32>) -> tensor<8x8x8x8xf32> { +^bb0(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<10x2xf32>): + // expected-error @+1 {{requires variance to be a 1D float tensor}} + %0:5 = "tf.FusedBatchNorm"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<10x2xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<10x2xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- +func.func private @testIfThen(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIfElse(tensor<*xf32>) -> tensor<*xf32> + +// Test valid tf.If operation +// CHECK-LABEL: func @testValidIfOp +func.func @testValidIfOp(tensor, tensor<2xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<2xf32>): + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, else_branch = @testIfElse, is_stateless = false + } : (tensor, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(f32) -> f32 +func.func private @testIfElse(f32) -> f32 + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, f32) -> f32 { +^bb0(%arg0: tensor, %arg1: f32): + // expected-error @+1 {{operand #1 must be variadic of tensor of tf.dtype values}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, f32) -> f32 + + func.return %1 : f32 +} + +// ----- + +func.func private @testIfElse(tensor<2xf32>) -> tensor<2xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<2xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<2xf32>): + // expected-error @+1 {{requires attribute 'then_branch'}} + %1 = "tf.If"(%arg0, %arg1) { + else_branch = @testIfElse, is_stateless = false + } : (tensor, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> +func.func private @testIfElse(tensor<2xf32>) -> tensor<2xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<2xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<2xf32>): + // expected-error @+1 {{'tf.If' op 'then_branch' inputs (size = 2) should have the same number of values as inputs (size = 1)}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(tensor<2xf32>) -> (tensor<2xf32>, tensor<2xf32>) +func.func private @testIfElse(tensor<2xf32>) -> tensor<2xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<2xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<2xf32>): + // expected-error @+1 {{'tf.If' op 'then_branch' results (size = 2) should have the same number of values as results (size = 1)}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(tensor<*xf16>) -> tensor<*xf32> +func.func private @testIfElse(tensor<*xf32>) -> tensor<*xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<2xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<2xf32>): + // expected-error @+1 {{'tf.If' op 'then_branch' input type tensor<*xf16> is incompatible with input type tensor<2xf32> at index 0}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(tensor<2xf32>) -> tensor<*xf32> +func.func private @testIfElse(tensor<3xf32>) -> tensor<*xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<*xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<*xf32>): + // expected-error @+1 {{expects all branch input type(s) (tensor<2xf32>, tensor<3xf32>) at index 0 to be cast compatible}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, tensor<*xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +func.func private @testIfThen(tensor<*xf32>) -> tensor<*xf32> +func.func private @testIfElse(tensor<*xf32>) -> tensor<3xf32> + +// Test invalid tf.If operation +func.func @testInvalidIfOp(tensor, tensor<*xf32>) -> tensor<2xf32> { +^bb0(%arg0: tensor, %arg1: tensor<*xf32>): + // expected-error @+1 {{'tf.If' op 'else_branch' result type tensor<3xf32> is incompatible with result type tensor<2xf32> at index 0}} + %1 = "tf.If"(%arg0, %arg1) { + then_branch = @testIfThen, + else_branch = @testIfElse, + is_stateless = false + } : (tensor, tensor<*xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> +} + +// ----- + +// Test invalid tf.Yield operation (parent should be IfRegion) +func.func @testInvalidYieldOp(%arg0: f32) -> f32 { + // expected-error @+1 {{'tf.Yield' op expects parent op to be one of 'tf.CaseRegion, tf.IfRegion, tf.WhileRegion, tf.GeneratorDatasetRegion'}} + "tf.Yield"(%arg0) : (f32) -> () +} + +// ----- + +// Test valid tf.IfRegion operation +// CHECK-LABEL: func @testValidIfRegionOp +func.func @testValidIfRegionOp(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %neg = "tf.Neg"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%neg) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// Test valid tf.IfRegion operation with multiple results +// CHECK-LABEL: func @testValidIfRegionOpWithMultipleResults +func.func @testValidIfRegionOpWithMultipleResults(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %0, %1, %2 = "tf.IfRegion"(%arg0) ({ + %t0 = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + %t1 = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + %t2 = "tf.Acosh"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t0, %t1, %t2) : (tensor<2xf32>, tensor<2xf32>, tensor<2xf32>) -> () + }, { + %e0 = "tf.Neg"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + %e1 = "tf.Relu"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + %e2 = "tf.Sin"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e0, %e1, %e2) : (tensor<2xf32>, tensor<2xf32>, tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> (tensor<2xf32>, tensor<2xf32>, tensor<2xf32>) + + %3 = "tf.Add"(%0, %1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + %4 = "tf.Add"(%2, %3) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %4 : tensor<2xf32> +} + +// ----- + +// Test invalid type for operand #0 for tf.IfRegion operation +func.func @testInvalidIfRegionOpType0(%arg0: f32, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{operand #0 must be 0D tensor of 1-bit signless integer values, but got 'f32'}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (f32) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// tf.IfRegion operation should have 2 regions +func.func @testInvalidIfRegionOp1Region(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{op expected 2 regions}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testInvalidIfRegionOpNoRegions(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{op expected 2 regions}} + %0 = "tf.IfRegion"(%arg0) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testInvalidIfRegionOp3Regions(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{op expected 2 regions}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %te = "tf.Relu"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%te) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// tf.IfRegion regions should be terminated with a tf.Yield +func.func @testIfRegionThenTerminator(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+2 {{block with no terminator}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testIfRegionElseTerminator(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+5 {{block with no terminator}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// tf.Region yield number of results should match op number of results +func.func @testIfRegionThenResultCount(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.IfRegion' op along control flow edge from Operation tf.Yield to Operation tf.IfRegion: region branch point has 2 operands, but region successor needs 1 inputs}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + // expected-note @+1 {{region branch point}} + "tf.Yield"(%t, %t) : (tensor<2xf32>, tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testIfRegionElseResultCount(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.IfRegion' op along control flow edge from Operation tf.Yield to Operation tf.IfRegion: region branch point has 2 operands, but region successor needs 1 inputs}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + // expected-note @+1 {{region branch point}} + "tf.Yield"(%e, %e) : (tensor<2xf32>, tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// tf.IfRegion yield types should match op result types +func.func @testIfRegionOpYieldMismatchThen(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.IfRegion' op then result type tensor is incompatible with result type tensor<2xf32> at index 0}} + %0 = "tf.IfRegion"(%arg0) ({ + "tf.Yield"(%arg0) : (tensor) -> () + }, { + %e = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testIfRegionOpYieldMismatchElse(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.IfRegion' op else result type tensor is incompatible with result type tensor<2xf32> at index 0}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + "tf.Yield"(%arg0) : (tensor) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// value generated in one branch cannot be consumed in the other branch +func.func @testIfRegionElseConsumingThen(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + // expected-error @+1 {{use of undeclared SSA value name}} + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testIfRegionThenConsumingElse(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.IfRegion"(%arg0) ({ + // expected-error @+1 {{does not dominate this use}} + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + // expected-note @+1 {{operand defined here}} + %t = "tf.Acos"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// The regions for IfRegion themselves cannot have any arguments +func.func @testInvalidIfRegionThenArg(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %neg = "tf.Neg"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + // expected-error @+1 {{'tf.IfRegion' op region #0 should have no arguments}} + %0 = "tf.IfRegion"(%arg0) ({ + ^bb(%arg_bb: tensor<2xf32>): + %t = "tf.Abs"(%arg_bb) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + %e = "tf.Acos"(%neg) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testInvalidIfRegionElseArg(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + %neg = "tf.Neg"(%arg1) : (tensor<2xf32>) -> tensor<2xf32> + // expected-error @+1 {{'tf.IfRegion' op region #1 should have no arguments}} + %0 = "tf.IfRegion"(%arg0) ({ + %t = "tf.Abs"(%neg) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%t) : (tensor<2xf32>) -> () + }, { + ^bb(%arg_bb: tensor<2xf32>): + %e = "tf.Acos"(%arg_bb) : (tensor<2xf32>) -> tensor<2xf32> + "tf.Yield"(%e) : (tensor<2xf32>) -> () + }) { is_stateless = false} : (tensor) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> +} + +// ----- + +// Test valid tf.MatrixBandPart +// CHECK-LABEL: func @testValidMatrixBandPartOp +func.func @testValidMatrixBandPartOp(%arg0: tensor<64x64xbf16>, %arg1: tensor, %arg2: tensor) -> tensor<64x64xbf16> { + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64xbf16>, tensor, tensor) -> tensor<64x64xbf16> + func.return %0 : tensor<64x64xbf16> +} + +// ----- + +// Test valid tf.MatrixBandPart +// CHECK-LABEL: func @testValidMatrixBandPartOp3D +func.func @testValidMatrixBandPartOp3D(%arg0: tensor<64x64x64xbf16>, %arg1: tensor, %arg2: tensor) -> tensor<64x64x64xbf16> { + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64x64xbf16>, tensor, tensor) -> tensor<64x64x64xbf16> + func.return %0 : tensor<64x64x64xbf16> +} + +// ----- + +// Test valid tf.MatrixBandPart +// CHECK-LABEL: func @testValidMatrixBandPartOpUnranked +func.func @testValidMatrixBandPartOpUnranked(%arg0: tensor<*xbf16>, %arg1: tensor, %arg2: tensor) -> tensor<*xbf16> { + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<*xbf16>, tensor, tensor) -> tensor<*xbf16> + func.return %0 : tensor<*xbf16> +} + +// ----- + +// Test valid tf.MatrixBandPart +// CHECK-LABEL: func @testValidMatrixBandPartOpUnrankedBand +func.func @testValidMatrixBandPartOpUnrankedBand(%arg0: tensor<64x64x64xbf16>, %arg1: tensor, %arg2: tensor) -> tensor<*xbf16> { + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64x64xbf16>, tensor, tensor) -> tensor<*xbf16> + func.return %0 : tensor<*xbf16> +} + +// ----- + +// Test valid tf.MatrixBandPart +// CHECK-LABEL: func @testValidMatrixBandPartOpCompatibleDynamicShapes +func.func @testValidMatrixBandPartOpCompatibleDynamicShapes(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.MatrixBandPart +func.func @testInvalidMatrixBandPartOp(%arg0: tensor<64x64x64xbf16>, %arg1: tensor, %arg2: tensor) -> tensor<64x64xbf16> { + // expected-error @+1 {{op failed to verify that all of {input, band} have dynamically equal types}} + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64x64xbf16>, tensor, tensor) -> tensor<64x64xbf16> + func.return %0 : tensor<64x64xbf16> +} + +// ----- + +// Test invalid tf.MatrixBandPart +func.func @testInvalidMatrixBandPartOp(%arg0: tensor, %arg1: tensor<64x64xi64>, %arg2: tensor) -> tensor { + // expected-error @+1 {{op requires `input` to have rank of at least 2, but found 'tensor'}} + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor, tensor<64x64xi64>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.MatrixBandPart +func.func @testInvalidMatrixBandPartOp(%arg0: tensor<64x64xi64>, %arg1: tensor<32xi64>, %arg2: tensor) -> tensor<64x64xi64> { + // expected-error @+1 {{op requires `num_lower` to have 0 dimensions, but found 'tensor<32xi64>'}} + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64xi64>, tensor<32xi64>, tensor) -> tensor<64x64xi64> + func.return %0 : tensor<64x64xi64> +} + +// ----- + +// Test invalid tf.MatrixBandPart +func.func @testInvalidMatrixBandPartOp(%arg0: tensor<64x64xi64>, %arg1: tensor, %arg2: tensor<32xi64>) -> tensor<64x64xi64> { + // expected-error @+1 {{op requires `num_upper` to have 0 dimensions, but found 'tensor<32xi64>'}} + %0 = "tf.MatrixBandPart"(%arg0, %arg1, %arg2) : (tensor<64x64xi64>, tensor, tensor<32xi64>) -> tensor<64x64xi64> + func.return %0 : tensor<64x64xi64> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.{|Stateful}PartitionedCall +//===--------------------------------------------------------------------===// + +// Test valid tf.PartitionedCall +// CHECK-LABEL: func @testValidPartitionedCall +func.func @testValidPartitionedCall(%arg0: tensor) -> tensor { + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// Test invalid tf.PartitionedCall +func.func @testUndefinedPartitionedCall(%arg0: tensor) -> tensor { + // expected-error @+1 {{'f' attribute refers to an undefined function: @nonexistant_pcall_func}} + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @nonexistant_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.PartitionedCall +func.func @testInvalidPartitionedCall(%arg0: tensor) -> tensor { + // expected-error @+1 {{argument count mismatch: 'args' has 1 arguments, but '@pcall_func_2' expects 2}} + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_func_2} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @pcall_func_2(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// Test valid tf.StatefulPartitionedCall +// CHECK-LABEL: func @testValidStatefulPartitionedCall +func.func @testValidStatefulPartitionedCall(%arg0: tensor) -> tensor { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +func.func @testUndefinedCallee(%arg0: tensor) -> tensor { + // expected-error @+1 {{'f' attribute refers to an undefined function: @nonexistant_pcall_func}} + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @nonexistant_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +func.func @testArgMismatch(%arg0: tensor) -> tensor { + // expected-error @+1 {{argument count mismatch: 'args' has 1 arguments, but '@pcall_func_2' expects 2}} + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_func_2} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @pcall_func_2(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.Select +//===--------------------------------------------------------------------===// + +// Test valid tf.Select +// CHECK-LABEL: func @testSelect +func.func @testSelect(%arg0: tensor<3xi1>, %arg1: tensor<3x2xf16>, %arg2: tensor<3x2xf16>) -> tensor<3x2xf16> { + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<3xi1>, tensor<3x2xf16>, tensor<3x2xf16>) -> tensor<3x2xf16> + func.return %0: tensor<3x2xf16> +} + +// ----- + +func.func @testInvalidSelect(%arg0: tensor<3xi1>, %arg1: tensor<2x3xf16>, %arg2: tensor<2x3xf16>) -> tensor<2x3xf16> { + // expected-error @+1 {{requires that, when pred is a vector, the shape matches the first dimension of t and e}} + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<3xi1>, tensor<2x3xf16>, tensor<2x3xf16>) -> tensor<2x3xf16> + func.return %0: tensor<2x3xf16> +} + +// ----- + +// Test invalid tf.Select - broadcasting then/else parameters is not supported +func.func @selectBroadcastThen(%arg0: tensor, %arg1: tensor<8x1xi32>, %arg2: tensor<2x8x8xi32>) -> tensor<2x8x8xi32> { + // expected-error @+1 {{requires t and e have compatible shapes}} + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor, tensor<8x1xi32>, tensor<2x8x8xi32>) -> tensor<2x8x8xi32> + func.return %0: tensor<2x8x8xi32> +} + +// ----- + +func.func @invalidSelect(%arg0: tensor<2xi1>, %arg1: tensor, %arg2: tensor) -> tensor<2xi32> { + // expected-error @+1 {{requires that t and e are nonscalar when pred is a vector}} + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<2xi1>, tensor, tensor) -> tensor<2xi32> + func.return %0: tensor<2xi32> +} + +// ----- + +func.func @invalidSelect(%arg0: tensor<1x8xi1>, %arg1: tensor<1x8x8xi32>, %arg2: tensor<1x8x8xi32>) -> tensor<1x8x8xi32> { + // expected-error @+1 {{requires that pred is a scalar OR has the same rank as t and e OR is a vector}} + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<1x8xi1>, tensor<1x8x8xi32>, tensor<1x8x8xi32>) -> tensor<1x8x8xi32> + func.return %0: tensor<1x8x8xi32> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.SelectV2 +//===--------------------------------------------------------------------===// + +// Test valid tf.SelectV2 +// CHECK-LABEL: func @selectV2BroadcastThen +func.func @selectV2BroadcastThen(%arg0: tensor, %arg1: tensor<8x1xi32>, %arg2: tensor<2x8x8xi32>) -> tensor<2x8x8xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor, tensor<8x1xi32>, tensor<2x8x8xi32>) -> tensor<2x8x8xi32> + func.return %0: tensor<2x8x8xi32> +} + +// ----- + +// Test valid tf.SelectV2 +// CHECK-LABEL: func @selectV2BroadcastElse +func.func @selectV2BroadcastElse(%arg0: tensor, %arg1: tensor<2x8x8xi32>, %arg2: tensor<8x1xi32>) -> tensor<2x8x8xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor, tensor<2x8x8xi32>, tensor<8x1xi32>) -> tensor<2x8x8xi32> + func.return %0: tensor<2x8x8xi32> +} + +// ----- + +// Test valid tf.SelectV2 +// CHECK-LABEL: func @selectV2BroadcastPred +func.func @selectV2BroadcastPred(%arg0: tensor<1xi1>, %arg1: tensor<2x8x8xi32>, %arg2: tensor<2x8x8xi32>) -> tensor<2x8x8xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<1xi1>, tensor<2x8x8xi32>, tensor<2x8x8xi32>) -> tensor<2x8x8xi32> + func.return %0: tensor<2x8x8xi32> +} + +// ----- + +// CHECK-LABEL: func @selectV2BroadcastAll +func.func @selectV2BroadcastAll(%arg0: tensor<8x1x1xi1>, %arg1: tensor<1x8x1xi32>, %arg2: tensor<1x1x8xi32>) -> tensor<8x8x8xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<8x1x1xi1>, tensor<1x8x1xi32>, tensor<1x1x8xi32>) -> tensor<8x8x8xi32> + func.return %0: tensor<8x8x8xi32> +} + +// ----- + +// CHECK-LABEL: func @selectV2DynamicRanked +func.func @selectV2DynamicRanked(%arg0: tensor<1xi1>, %arg1: tensor<2x?x8xi32>, %arg2: tensor<2x8x8xi32>) -> tensor<2x?x8xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<1xi1>, tensor<2x?x8xi32>, tensor<2x8x8xi32>) -> tensor<2x?x8xi32> + func.return %0: tensor<2x?x8xi32> +} + +// ----- + +// CHECK-LABEL: func @selectV2Unranked +func.func @selectV2Unranked(%arg0: tensor<1xi1>, %arg1: tensor<2x8x8xi32>, %arg2: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<1xi1>, tensor<2x8x8xi32>, tensor<*xi32>) -> tensor<*xi32> + func.return %0: tensor<*xi32> +} + +// ----- + +// Test invalid tf.SelectV2: this is an invalid broadcast for the predicate +func.func @testInvalidSelectV2(%arg0: tensor<3xi1>, %arg1: tensor<3x2xf16>, %arg2: tensor<3x2xf16>) -> tensor<3x2xf16> { + // expected-error @+1 {{operands don't have broadcast-compatible shapes}} + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<3xi1>, tensor<3x2xf16>, tensor<3x2xf16>) -> tensor<3x2xf16> + func.return %0: tensor<3x2xf16> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.Softmax +//===--------------------------------------------------------------------===// + +// Test valid tf.Softmax +// CHECK-LABEL: func @testSoftmax +func.func @testSoftmax(tensor<8x16xf32>) -> tensor<8x16xf32> { +^bb0(%arg0: tensor<8x16xf32>): + %0 = "tf.Softmax"(%arg0) {T = "tfdtype$DT_FLOAT"} : (tensor<8x16xf32>) -> tensor<8x16xf32> + func.return %0 : tensor<8x16xf32> +} + +// ----- + +// Test invalid tf.Softmax +func.func @testSoftmax(%arg0 : tensor) -> tensor { + // expected-error @+1 {{requires operand to have rank at least 1}} + %0 = "tf.Softmax"(%arg0) {T = "tfdtype$DT_FLOAT"} : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test valid tf.SoftmaxCrossEntropyWithLogits +// CHECK-LABEL: func @testSoftmaxCrossEntropyWithLogits +func.func @testSoftmaxCrossEntropyWithLogits(%arg0: tensor<2x3xf32>, %arg1: tensor<3xf32>) -> (tensor<3xf32>, tensor<2x3xf32>) { + %0:2 = "tf.SoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<3xf32>) -> (tensor<3xf32>, tensor<2x3xf32>) + func.return %0#0, %0#1 : tensor<3xf32>, tensor<2x3xf32> +} + +// ----- + +func.func @testSoftmaxCrossEntropyWithLogits(%arg0: tensor<2xf32>, %arg1: tensor<3xf32>) -> (tensor<3xf32>, tensor<3xf32>) { + // expected-error @+1 {{requires features and labels to be broadcast compatible to rank two}} + %0:2 = "tf.SoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<2xf32>, tensor<3xf32>) -> (tensor<3xf32>, tensor<3xf32>) + func.return %0#0, %0#1 : tensor<3xf32>, tensor<3xf32> +} + +// ----- + +func.func @testSoftmaxCrossEntropyWithLogits(%arg0: tensor<3xf32>, %arg1: tensor<3xf32>) -> (tensor<3xf32>, tensor<3xf32>) { + // expected-error @+1 {{requires features and labels to be broadcast compatible to rank two}} + %0:2 = "tf.SoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<3xf32>, tensor<3xf32>) -> (tensor<3xf32>, tensor<3xf32>) + func.return %0#0, %0#1 : tensor<3xf32>, tensor<3xf32> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.SpaceToBatchND +//===--------------------------------------------------------------------===// + +// Test valid tf.SpaceToBatchND +// CHECK-LABEL: func @testSpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<2xi64>, %paddings: tensor<2x2xi64>) -> tensor { + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test valid tf.SpaceToBatchND +// CHECK-LABEL: func @testSpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>) -> tensor<36x2x3x10xf32> { + %block_shape = "tf.Const"() {value = dense<[4, 3]> : tensor<2xi64>} : () -> tensor<2xi64> + %paddings = "tf.Const"() {value = dense<[[1, 2], [1, 1]]> : tensor<2x2xi64>} : () -> tensor<2x2xi64> + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor<36x2x3x10xf32> + func.return %0 : tensor<36x2x3x10xf32> +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<2x2xi64>, %paddings: tensor<2x2xi64>) -> tensor { + // expected-error @+1 {{requires rank of block_shape = 1; got 2}} + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2x2xi64>, tensor<2x2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<2xi64>, %paddings: tensor<2xi64>) -> tensor { + // expected-error @+1 {{requires rank of paddings = 2; got 1}} + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<2xi64>, %paddings: tensor<2x10xi64>) -> tensor { + // expected-error @+1 {{requires paddings.shape[1] to be 2; got 10}} + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x10xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<4xi64>, %paddings: tensor<2x2xi64>) -> tensor { + // expected-error @+1 {{requires block_shape.shape[0] must equal paddings.shape[0]}} + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<4xi64>, tensor<2x2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5xf32>, %block_shape: tensor<2xi64>, %paddings: tensor<2x2xi64>) -> tensor { + // expected-error @+1 {{requires rank of input >= 1 + rank of block}} + %0 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %paddings: tensor<2x2xi64>) -> tensor { + %block_shape = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi64>} : () -> tensor<2xi64> + // expected-error @+1 {{requires all values of block_shape to be >= 1; failed for dimension 1}} + %1 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor + func.return %1 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>, %block_shape: tensor<2xi64>) -> tensor { + %paddings = "tf.Const"() {value = dense<[[1, 0], [-1, 0]]> : tensor<2x2xi64>} : () -> tensor<2x2xi64> + // expected-error @+1 {{requires all values of paddings to be >= 0; failed for dimension 1}} + %1 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor + func.return %1 : tensor +} + +// ----- + +// Test invalid tf.SpaceToBatchND +func.func @testSpaceToBatchND(%input: tensor<3x5x7x10xf32>) -> tensor<36x2x3x10xf32> { + %block_shape = "tf.Const"() {value = dense<[4, 3]> : tensor<2xi64>} : () -> tensor<2xi64> + %paddings = "tf.Const"() {value = dense<[[1, 2], [1, 2]]> : tensor<2x2xi64>} : () -> tensor<2x2xi64> + // expected-error @+1 {{requires block_shape[i] divides input_shape[i + 1] + paddings[i, 0] + paddings[i, 1]; failed for i=1}} + %1 = "tf.SpaceToBatchND"(%input, %block_shape, %paddings) : (tensor<3x5x7x10xf32>, tensor<2xi64>, tensor<2x2xi64>) -> tensor<36x2x3x10xf32> + func.return %1 : tensor<36x2x3x10xf32> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.SparseSoftmaxCrossEntropyWithLogits +//===--------------------------------------------------------------------===// + +// Test valid tf.SparseSoftmaxCrossEntropyWithLogits +// CHECK-LABEL: func @testSparseSoftmaxCrossEntropyWithLogits +func.func @testSparseSoftmaxCrossEntropyWithLogits(%arg0: tensor<2x3xf32>, %arg1: tensor<2xi32>) -> (tensor<3xf32>, tensor<2x3xf32>) { + %0:2 = "tf.SparseSoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<2xi32>) -> (tensor<3xf32>, tensor<2x3xf32>) + func.return %0#0, %0#1 : tensor<3xf32>, tensor<2x3xf32> +} + +// ----- + +func.func @testSparseSoftmaxCrossEntropyWithLogits(%arg0: tensor<3xf32>, %arg1: tensor<3xi32>) -> (tensor<3xf32>, tensor<2x3xf32>) { + // expected-error @+1 {{requires features operand of rank two}} + %0:2 = "tf.SparseSoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<3xf32>, tensor<3xi32>) -> (tensor<3xf32>, tensor<2x3xf32>) + func.return %0#0, %0#1 : tensor<3xf32>, tensor<2x3xf32> +} + +// ----- + +func.func @testSparseSoftmaxCrossEntropyWithLogits(%arg0: tensor<2x3xf32>, %arg1: tensor<2x3xi32>) -> (tensor<2xf32>, tensor<2x3xf32>) { + // expected-error @+1 {{requires labels operand of rank one}} + %0:2 = "tf.SparseSoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<2x3xi32>) -> (tensor<2xf32>, tensor<2x3xf32>) + func.return %0#0, %0#1 : tensor<2xf32>, tensor<2x3xf32> +} + +// ----- + +func.func @testSparseSoftmaxCrossEntropyWithLogits(%arg0: tensor<2x3xf32>, %arg1: tensor<3xi32>) -> (tensor<2xf32>, tensor<2x3xf32>) { + // expected-error @+1 {{requires features and labels with matching first dimension}} + %0:2 = "tf.SparseSoftmaxCrossEntropyWithLogits"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<3xi32>) -> (tensor<2xf32>, tensor<2x3xf32>) + func.return %0#0, %0#1 : tensor<2xf32>, tensor<2x3xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<*xf32>) -> (tensor) +func.func private @testWhileBody(tensor<*xf32>) -> (tensor<*xf32>) + +// Test valid 'While' operation +// CHECK-LABEL: func @testWhileResult +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- +func.func @testWhileUndefinedCond(%arg0: tensor, %arg1: tensor) -> tensor { + // expected-error @+1 {{cond refers to an undefined function : undefined_func}} + %0 = "tf.While"(%arg0, %arg1) {cond = @undefined_func, body = @body, is_stateless = false} : (tensor, tensor) -> (tensor) + func.return %0 : tensor +} + +func.func private @body(%arg0: tensor, %arg1: tensor) -> tensor + +// ----- +func.func @testWhileUndefinedBody(%arg0: tensor, %arg1: tensor) -> tensor { + // expected-error @+1 {{body refers to an undefined function : undefined_func}} + %0 = "tf.While"(%arg0, %arg1) {cond = @cond, body = @undefined_func, is_stateless = false} : (tensor, tensor) -> (tensor) + func.return %0 : tensor +} + +func.func private @cond(%arg0: tensor, %arg1: tensor) -> tensor + +// ----- + +func.func private @testWhileCond(tensor<*xf32>) -> () +func.func private @testWhileBody(tensor<*xf32>) -> (tensor<*xf32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{requires cond function to have exactly one result}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<*xf32>) -> (tensor) +func.func private @testWhileBody(tensor<*xf32>) -> (tensor<*xf32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xi32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{'tf.While' op input type tensor<*xf32> is incompatible with result type tensor<*xi32> at index 0}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xi32>) + + func.return %1 : tensor<*xi32> +} + +// ----- + +func.func private @testWhileCond(tensor<*xi32>) -> (tensor) +func.func private @testWhileBody(tensor<*xf32>) -> (tensor<*xf32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{'tf.While' op input type tensor<*xf32> is incompatible with condition input type tensor<*xi32> at index 0}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<*xf32>) -> (tensor) +func.func private @testWhileBody(tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{'tf.While' op inputs (size = 1) should have the same number of values as body inputs (size = 2)}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<*xf32>) -> (tensor) +func.func private @testWhileBody(tensor<*xf32>) -> (tensor<*xi32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{'tf.While' op result type tensor<*xf32> is incompatible with body result type tensor<*xi32> at index 0}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<3xf32>) -> (tensor) +func.func private @testWhileBody(tensor<4xf32>) -> (tensor<*xf32>) + +// Test invalid 'While' operation +func.func @testWhileResult(tensor<*xf32>) -> (tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{'tf.While' op condition input type tensor<3xf32> is incompatible with body input type tensor<4xf32> at index 0}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*xf32>) -> (tensor<*xf32>) + + func.return %1 : tensor<*xf32> +} + +// ----- + +func.func private @testWhileCond(tensor<*x!tf_type.resource>>) -> (tensor) +func.func private @testWhileBody(tensor<*x!tf_type.resource>>) -> (tensor>>) + +// Test invalid 'While' operation verifier that detects incompatible tf.resource +// subtypes. +func.func @testWhileResult(tensor<*x!tf_type.resource>>) -> (tensor>>) { +^bb0(%arg0: tensor<*x!tf_type.resource>>): + // expected-error @+1 {{'tf.While' op input type tensor<*x!tf_type.resource>> is incompatible with result type tensor>> at index 0}} + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*x!tf_type.resource>>) -> (tensor>>) + + func.return %1 : tensor>> +} + +// ----- + +func.func private @testWhileCond(tensor<*x!tf_type.resource>>) -> (tensor) +func.func private @testWhileBody(tensor<*x!tf_type.resource>>) -> (tensor>>) + +// Test 'While' operation verifier allows compatible tf.resource subtypes. +// CHECK-LABEL: func @testWhileResult +func.func @testWhileResult(tensor<*x!tf_type.resource>>) -> (tensor>>) { +^bb0(%arg0: tensor<*x!tf_type.resource>>): + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*x!tf_type.resource>>) -> (tensor>>) + + func.return %1 : tensor>> +} + +// ----- + +func.func private @testWhileCond(tensor<*x!tf_type.resource>>) -> (tensor) +func.func private @testWhileBody(tensor<*x!tf_type.resource>>) -> (tensor) + +// Test 'While' operation verifier treats tf.resource with subtype and without +// subtype as compatible types. +// CHECK-LABEL: func @testWhileResult +func.func @testWhileResult(tensor<*x!tf_type.resource>>) -> (tensor) { +^bb0(%arg0: tensor<*x!tf_type.resource>>): + %1 = "tf.While"(%arg0) { + cond = @testWhileCond, + body = @testWhileBody, + is_stateless = false + } : (tensor<*x!tf_type.resource>>) -> (tensor) + + func.return %1 : tensor +} + +// ----- + +func.func private @cond(tensor<1x?x3xf32>) -> tensor +func.func private @body(tensor<1x?x3xf32>) -> tensor<1x?x3xf32> + +// Test shape invariant 'While' operation verifier with different operand and +// result shapes. +// CHECK-LABEL: func @testShapeInvariantWhile +func.func @testShapeInvariantWhile(%arg0: tensor<1x2x3xf32>) -> tensor<1x8x3xf32> { + %0 = "tf.While"(%arg0) {cond = @cond, body = @body, is_stateless = false, shape_invariant} : (tensor<1x2x3xf32>) -> tensor<1x8x3xf32> + func.return %0 : tensor<1x8x3xf32> +} + +// ----- +// WhileRegion tests + +// Simple While region +// CHECK-LABEL: testValidWhileRegion +func.func @testValidWhileRegion(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %zero = arith.constant dense<0> : tensor + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%add, %sub) : (tensor<*xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + + func.return %0#0 : tensor<*xf32> +} + +// ----- + +// While region with no inputs (and hence no outputs) (infinite loop) +// CHECK-LABEL: testValidWhileRegionNoInputs +func.func private @printer(tensor) -> () +func.func @testValidWhileRegionNoInputs() -> () { + "tf.WhileRegion"() ( + { + %true = arith.constant dense<1> : tensor + "tf.Yield"(%true) : (tensor) -> () + }, + { + %one = arith.constant dense<1> : tensor + func.call @printer(%one) : (tensor) -> () + // TODO(b/159753381): tf.IfRegion implicit terminator not working + "tf.Yield"() : () -> () + } + ) { is_stateless = true } : () -> () + func.return +} + +// ----- + +// WhileRegion with a yield that passes arguments to the body. +// CHECK-LABEL: testWhileRegionWithFullConditionYield +func.func @testWhileRegionWithFullConditionYield(%arg0 : tensor<*xf32>, %arg1 : tensor) -> tensor<*xf32> { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%carg0: tensor<*xf32>, %carg1: tensor): + %cond = builtin.unrealized_conversion_cast to tensor + "tf.Yield"(%cond, %carg0, %carg1) : (tensor, tensor<*xf32>, tensor) -> () + }, { + ^bb0(%barg0: tensor<*xf32>, %barg1: tensor): + %add0 = "tf.Add"(%barg0, %barg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %add1 = "tf.Add"(%barg1, %barg1) : (tensor, tensor) -> tensor + "tf.Yield"(%add0, %add1) : (tensor<*xf32>, tensor) -> () + }) { is_stateless = false } : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor) + + func.return %0#0 : tensor<*xf32> +} + +// ----- +// Invalid while tests. There are 5 sets of type matching that is required +// I = input, O = output, BI, BO = body input/output, CI = cond input. +// [I, O], [I, CI], [I, BI], [BO, BI], [BO, O]. +// Each check can fail due to number or type mismatch. However, these +// conditions are not all independent. So we just check I->{CI, BI}, O->BO, and +// in addition I->O. BO->BI mismatch cannot be independently created without +// breaking one of these mismatches. That gives us 4x2 tests. In addition +// condition result needs to be tensor, for which we have 3 +// additional validation tests. All these tests are based on the following +// valid while + +func.func @testInvalidTestValidBase(%arg0 : tensor) -> (tensor) { + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @testInvalidWhileRegion_I_CI_CountMismatch(%arg0 : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op inputs (size = 1) should have the same number of values as condition inputs (size = 0)}} + %0 = "tf.WhileRegion"(%arg0) ( + { + //^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_I_CI_TypeMismatch(%arg0 : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op input type tensor is incompatible with condition input type tensor at index 0}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_I_BI_CountMismatch(%arg0 : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op inputs (size = 1) should have the same number of values as body inputs (size = 2)}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %true = arith.constant dense<1> : tensor + "tf.Yield"(%true) : (tensor) -> () + }, + { + ^bb0(%barg0: tensor, %barg1 : tensor): + "tf.Yield"(%barg0) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_I_BI_TypeMismatch(%arg0 : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op input type tensor is incompatible with body input type tensor at index 0}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %true = arith.constant dense<1> : tensor + "tf.Yield"(%true) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + %c = "tf.Cast"(%barg) : (tensor) -> tensor + "tf.Yield"(%c) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_O_BO_CountMismatch(%arg0 : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op results (size = 1) should have the same number of values as body results (size = 2)}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg, %barg) : (tensor, tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0#0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegionMismatch_O_BO_TypeMismatch(%arg0 : tensor, %arg1: tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op result type tensor is incompatible with body result type tensor at index 0}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%arg1) : (tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_I_O_CountMismatch(%arg0 : tensor) -> (tensor) { + // expected-error@+1 {{'tf.WhileRegion' op inputs (size = 1) should have the same number of values as results (size = 2)}} + %0:2 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg, %barg) : (tensor, tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor, tensor) + func.return %0#0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegion_I_O_TypeMismatch(%arg0: tensor, %arg1 : tensor) -> (tensor) { + // expected-error@+1 {{'tf.WhileRegion' op input type tensor is incompatible with result type tensor at index 0}} + %0 = "tf.WhileRegion"(%arg0) ( + { + ^bb0(%carg: tensor): + %false = arith.constant dense : tensor + "tf.Yield"(%false) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%arg1) : (tensor) -> () + } + ) { is_stateless = true } : (tensor) -> (tensor) + func.return %0 : tensor +} +// ----- + +func.func @testInvalidWhileRegionConditionOutputCount2(%arg : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op condition should yield a tensor and forward the arguments}} + %0 = "tf.WhileRegion"(%arg) ( + { + ^bb0(%carg: tensor): + %true = arith.constant dense<1> : tensor + "tf.Yield"(%true, %carg, %carg) : (tensor, tensor, tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegionForwarding(%arg : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op arguments on condition block aren't forwarded to yield}} + %0 = "tf.WhileRegion"(%arg) ( + { + ^bb0(%carg: tensor): + %true = arith.constant dense<1> : tensor + "tf.Yield"(%true, %true) : (tensor, tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegionConditionOutputCount0(%arg : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op condition should yield a tensor and forward the arguments}} + %0 = "tf.WhileRegion"(%arg) ( + { + ^bb0(%carg: tensor): + "tf.Yield"() : () -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidWhileRegionConditionOutputType(%arg : tensor) -> (tensor) { + // expected-error @+1 {{'tf.WhileRegion' op condition should have a single tensor result}} + %0 = "tf.WhileRegion"(%arg) ( + { + ^bb0(%carg: tensor): + "tf.Yield"(%carg) : (tensor) -> () + }, + { + ^bb0(%barg: tensor): + "tf.Yield"(%barg) : (tensor) -> () + } + ) {is_stateless = false} : (tensor) -> (tensor) + + func.return %0 : tensor +} + +// ----- + +// Test shape invariant 'WhileRegion' operation verifier with different operand +// and result shapes. +// CHECK-LABEL: func @testShapeInvariantWhileRegion +func.func @testShapeInvariantWhileRegion(%arg0: tensor<1x2x3xf32>) -> tensor<1x8x3xf32> { + %0 = "tf.WhileRegion"(%arg0) ({ + ^cond(%carg0: tensor<1x?x3xf32>): + %1 = "tf.SomeCondOp"(%carg0) : (tensor<1x?x3xf32>) -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^body(%barg0: tensor<1x?x3xf32>): + %1 = "tf.SomeBodyOp"(%barg0) : (tensor<1x?x3xf32>) -> tensor<1x?x3xf32> + "tf.Yield"(%1) : (tensor<1x?x3xf32>) -> () + }) {is_stateless = false, shape_invariant} : (tensor<1x2x3xf32>) -> tensor<1x8x3xf32> + func.return %0 : tensor<1x8x3xf32> +} + +// ----- + +// CHECK-LABEL: func @testValidShape +func.func @testValidShape(tensor<1x32x32x16xf32>, tensor<*xf32>) -> (tensor<4xi32>, tensor) { +^bb0(%arg0: tensor<1x32x32x16xf32>, %arg1: tensor<*xf32>): + %0 = "tf.Shape"(%arg0) {T = "tfdtype$DT_FLOAT", output = "tfdtype$DT_INT32"} : (tensor<1x32x32x16xf32>) -> tensor<4xi32> + %1 = "tf.Shape"(%arg1) {T = "tfdtype$DT_FLOAT", output = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor + func.return %0, %1 : tensor<4xi32>, tensor +} + +// ----- + +func.func @testShapeWrongResultElemType(%arg0: tensor<1x32x32x16xf32>) -> tensor<4xf32> { + // expected-error @+1 {{result #0 must be tensor of 32/64-bit signed integer values}} + %0 = "tf.Shape"(%arg0) : (tensor<1x32x32x16xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// ----- + +func.func @testShapeWrongResultDim(tensor<1x32x32x16xf32>) -> tensor<3x2xi32> { +^bb0(%arg0: tensor<1x32x32x16xf32>): + // expected-error @+1 {{requires 1D type for result}} + %0 = "tf.Shape"(%arg0) {T = "tfdtype$DT_FLOAT", output = "tfdtype$DT_INT32"} : (tensor<1x32x32x16xf32>) -> tensor<3x2xi32> + func.return %0 : tensor<3x2xi32> +} + +// ----- + +func.func @testShapeMismatchDim(tensor<1x32x32x16xf32>) -> tensor<2xi32> { +^bb0(%arg0: tensor<1x32x32x16xf32>): + // expected-error @+1 {{requires dimension size of result to match rank of operand}} + %0 = "tf.Shape"(%arg0) {T = "tfdtype$DT_FLOAT", output = "tfdtype$DT_INT32"} : (tensor<1x32x32x16xf32>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +func.func @testShapeWrongResultDimDynamic(tensor<*xf32>) -> tensor<2xi32> { +^bb0(%arg0: tensor<*xf32>): + // expected-warning @+1 {{has static shape result for unranked operand}} + %0 = "tf.Shape"(%arg0) {T = "tfdtype$DT_FLOAT", output = "tfdtype$DT_INT32"} : (tensor<*xf32>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +// CHECK-LABEL: func @testValidShapeN +func.func @testValidShapeN(%arg0 : tensor<1x32x32x16xf32>, %arg1 : tensor<*xf32>) -> (tensor<4xi32>, tensor) { + // CHECK-NEXT: "tf.ShapeN" + %0:2 = "tf.ShapeN"(%arg0, %arg1) : (tensor<1x32x32x16xf32>, tensor<*xf32>) -> (tensor<4xi32>, tensor) + func.return %0#0, %0#1 : tensor<4xi32>, tensor +} + +// ----- + +func.func @testShapeNWrongResultElemType(%arg0: tensor<1x32x32x16xf32>) -> tensor<4xf32> { + // expected-error @+1 {{result #1 must be variadic of tensor of 32/64-bit signed integer values}} + %0:2 = "tf.ShapeN"(%arg0, %arg0) : (tensor<1x32x32x16xf32>, tensor<1x32x32x16xf32>) -> (tensor<4xi32>, tensor<4xf32>) + func.return %0#1 : tensor<4xf32> +} + +// ----- + +func.func @testShapeNWrongResultDim(tensor<1x32x32x16xf32>) -> tensor<2x2xi32> { +^bb0(%arg0: tensor<1x32x32x16xf32>): + // expected-error @+1 {{requires 1D type for result #1}} + %0:2 = "tf.ShapeN"(%arg0, %arg0) : (tensor<1x32x32x16xf32>, tensor<1x32x32x16xf32>) -> (tensor<4xi32>, tensor<2x2xi32>) + func.return %0#1 : tensor<2x2xi32> +} + +// ----- + +func.func @testShapeNMismatchDim(tensor<1x32x32x16xf32>) -> tensor<2xi32> { +^bb0(%arg0: tensor<1x32x32x16xf32>): + // expected-error @+1 {{requires dimension size of result #1 to match rank of operand #1}} + %0:2 = "tf.ShapeN"(%arg0, %arg0) : (tensor<1x32x32x16xf32>, tensor<1x32x32x16xf32>) -> (tensor<4xi32>, tensor<2xi32>) + func.return %0#1 : tensor<2xi32> +} + +// ----- + +func.func @testShapeNWrongResultDimDynamic(tensor<*xf32>) -> tensor<2xi32> { +^bb0(%arg0: tensor<*xf32>): + // expected-warning @+1 {{has static shape result #1 for unranked operand #1}} + %0:2 = "tf.ShapeN"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> (tensor, tensor<2xi32>) + func.return %0#1 : tensor<2xi32> +} + +// ----- + +func.func @testShapeNWrongNumResults(tensor<*xf32>) { +^bb0(%arg0: tensor<*xf32>): + // expected-error @+1 {{requires 3 result(s), got 2 result(s)}} + %0:2 = "tf.ShapeN"(%arg0, %arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> (tensor, tensor) + func.return +} + +// ----- + +// CHECK-LABEL: func @testValidVariableShape +func.func @testValidVariableShape(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>) -> (tensor<4xi32>, tensor) { + %0 = "tf.VariableShape"(%arg0) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>) -> tensor<4xi32> + %1 = "tf.VariableShape"(%arg1) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>) -> tensor + func.return %0, %1 : tensor<4xi32>, tensor +} + +// ----- + +func.func @testVariableShapeMultipleSubtypes(%arg0: tensor<*x!tf_type.resource, tensor<1x32x32x16xf32>>>) { + // expected-error @+1 {{requires resource input type to have at most 1 subtype}} + %0 = "tf.VariableShape"(%arg0) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource, tensor<1x32x32x16xf32>>>) -> tensor<4xi32> + func.return +} + +// ----- + +func.func @testVariableShapeWrongResultElemType(%arg0: tensor<*x!tf_type.resource>>) -> tensor<4xf32> { + // expected-error @+1 {{result #0 must be tensor of 32/64-bit signed integer values}} + %0 = "tf.VariableShape"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// ----- + +func.func @testVariableShapeWrongResultDim(%arg0: tensor<*x!tf_type.resource>>) -> tensor<2x3xi32> { + // expected-error @+1 {{requires 1D type for result}} + %0 = "tf.VariableShape"(%arg0) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>) -> tensor<2x3xi32> + func.return %0 : tensor<2x3xi32> +} + +// ----- + +func.func @testVariableShapeMismatchDim(%arg0: tensor<*x!tf_type.resource>>) -> tensor<2xi32> { + // expected-error @+1 {{requires dimension size of result to match rank of operand}} + %0 = "tf.VariableShape"(%arg0) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +func.func @testVariableShapeWrongResultDimDynamic(%arg0: tensor<*x!tf_type.resource>>) -> tensor<2xi32> { + // expected-warning @+1 {{has static shape result for unranked operand}} + %0 = "tf.VariableShape"(%arg0) {output = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +func.func @testVariableShapeWrongNumResources(%arg0: tensor<1x2x!tf_type.resource>>) -> tensor<4xi32> { + // expected-error @+1 {{requires input to have one resource}} + %0 = "tf.VariableShape"(%arg0) : (tensor<1x2x!tf_type.resource>>) -> tensor<4xi32> + func.return %0 : tensor<4xi32> +} + +// ----- + +// Test invalid tf.Const +func.func @testConst() -> tensor { + // expected-error @+1 {{attribute 'value' failed to satisfy constraint: constant vector/tensor}} + %0 = "tf.Const"() {T = "tfdtype$DT_FLOAT", value = 1.0 : f32} : () -> tensor + func.return %0 : tensor +} + +// ----- + +// Test invalid tf.ToBool +func.func @testInvalidToBool(%arg0: tensor) -> tensor<1xi1> { + // expected-error @+2 {{'tf.ToBool' op failed to infer returned types}} + // expected-error @+1 {{op inferred type(s) 'tensor' are incompatible with return type(s) of operation 'tensor<1xi1>'}} + %0 = "tf.ToBool"(%arg0) : (tensor) -> tensor<1xi1> + func.return %0 : tensor<1xi1> +} + +// ----- + +// Test invalid tf.TPUPartitionedInputV2 with packing +func.func @testPackedTPUPartitionedInputV2(tensor<2x4xf32>, tensor<2x4xf32>) -> tensor<4x4xf32> { +^bb0(%arg0: tensor<2x4xf32>, %arg1: tensor<2x4xf32>): + // expected-error @+1 {{expected 1 inputs, got 2}} + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {partition_dims = [2, 1], is_packed = true} : (tensor<2x4xf32>, tensor<2x4xf32>) -> tensor<4x4xf32> + func.return %0 : tensor<4x4xf32> +} + +// ----- + +// Test invalid tf.TPUPartitionedInputV2 without packing +func.func @testUnpackedTPUPartitionedInputV2(tensor<2x4xf32>, tensor<2x4xf32>) -> tensor<4x4xf32> { +^bb0(%arg0: tensor<2x4xf32>, %arg1: tensor<2x4xf32>): + // expected-error @+1 {{expected 2 inputs, got 1}} + %0 = "tf.TPUPartitionedInputV2"(%arg0) {partition_dims = [2, 1], is_packed = false} : (tensor<2x4xf32>) -> tensor<4x4xf32> + func.return %0 : tensor<4x4xf32> +} + +// ----- + +// Test valid tf.Transpose +// CHECK-LABEL: testTranspose +func.func @testTranspose(tensor<2x3xf32>) -> tensor<3x2xf32> { +^bb0(%arg0: tensor<2x3xf32>): + %cst = arith.constant dense<[1, 0]> : tensor<2xi32> + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +// Test tf.Transpose with partial unknown shape +// CHECK-LABEL: testTranspose +func.func @testTranspose(tensor<2x?xf32>) -> tensor { +^bb0(%arg0: tensor<2x?xf32>): + %cst = arith.constant dense<[1, 0]> : tensor<2xi32> + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x?xf32>, tensor<2xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test tf.Transpose with different partial unknown shape +// CHECK-LABEL: testTranspose +func.func @testTranspose(tensor<2x?x?xf32>) -> tensor<3x?x2xf32> { +^bb0(%arg0: tensor<2x?x?xf32>): + %cst = arith.constant dense<[2, 1, 0]> : tensor<3xi32> + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x?x?xf32>, tensor<3xi32>) -> tensor<3x?x2xf32> + func.return %0 : tensor<3x?x2xf32> +} + +// ----- + +// Test tf.Transpose with invalid rank of perm +func.func @testTranspose(tensor<2x3xf32>, tensor<1x2xi32>) -> tensor<3x2xf32> { +^bb0(%arg0: tensor<2x3xf32>, %arg1: tensor<1x2xi32>): + // expected-error @+1 {{expected perm to be a 1-D Tensor, got perm of rank 2}} + %0 = "tf.Transpose"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3xf32>, tensor<1x2xi32>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +// Test tf.Transpose with invalid size of perm +func.func @testTranspose(tensor<2x3xf32>) -> tensor<3x2xf32> { +^bb0(%arg0: tensor<2x3xf32>): + %cst = arith.constant dense<[1, 0, 2]> : tensor<3xi32> + // expected-error @+1 {{expected perm to be a 1-D Tensor of size equal to the rank of x, got perm of size 3, and x of rank 2}} + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3xf32>, tensor<3xi32>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +// Test tf.Transpose with invalid index of perm +func.func @testTranspose(tensor<2x2xf32>) -> tensor<2x2xf32> { +^bb0(%arg0: tensor<2x2xf32>): + %cst = arith.constant dense<[1, -3]> : tensor<2xi32> + // expected-error @+1 {{'tf.Transpose' op perm[1]=-3 must be in range [-2, 2)}} + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x2xf32>, tensor<2xi32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// ----- + +// Test tf.Transpose with invalid rank of y +func.func @testTranspose(tensor<2x3xf32>) -> tensor<3x2x1xf32> { +^bb0(%arg0: tensor<2x3xf32>): + %cst = arith.constant dense<[1, 0]> : tensor<2xi32> + // expected-error @+1 {{x should be of the same rank with y, got x of rank 2, and y of rank 3}} + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<3x2x1xf32> + func.return %0 : tensor<3x2x1xf32> +} + +// ----- + +// Test tf.Transpose with invalid shape of y +func.func @testTranspose(tensor<2x3x4xf32>) -> tensor<3x2x4xf32> { +^bb0(%arg0: tensor<2x3x4xf32>): + %cst = arith.constant dense<[2, 0, 1]> : tensor<3xi32> + // expected-error @+1 {{requires y.shape[0] (3) to be equal to x.shape[perm[2]] (4)}} + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3x4xf32>, tensor<3xi32>) -> tensor<3x2x4xf32> + func.return %0 : tensor<3x2x4xf32> +} + +// ----- + +// Test invalid tf.Less +func.func @testLess(tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> { +^bb0(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>): + // expected-error @+1 {{op result #0 must be tensor of bool values}} + %0 = "tf.Less"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0 : tensor<4xi32> +} + +// ----- + +// Test valid tf.ConcatV2 +func.func @testConcatV2(%arg: tensor<8x16xf32>, %axis: tensor) -> tensor { + %0 = "tf.ConcatV2"(%arg, %arg, %axis) : (tensor<8x16xf32>, tensor<8x16xf32>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// tf.ConcatV2 with wrong 'axis' element type +func.func @testConcatV2(%arg: tensor<8x16xf32>, %axis: tensor) -> tensor { + // expected-error @+1 {{operand #2 must be tensor of 32/64-bit signed integer values}} + %0 = "tf.ConcatV2"(%arg, %arg, %axis) : (tensor<8x16xf32>, tensor<8x16xf32>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// tf.ConcatV2 missing required 'axis' operand +func.func @testConcatV2() -> tensor { + // expected-error @+1 {{expected 1 or more operands}} + %0 = "tf.ConcatV2"() : () -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: testAll +func.func @testAll(%arg0: tensor<2x2xi1>, %arg1: tensor) -> tensor { + %0 = "tf.All"(%arg0, %arg1) {keep_dims = false} : (tensor<2x2xi1>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: testAll64 +func.func @testAll64(%arg0: tensor<2x2xi1>, %arg1: tensor) -> tensor { + %0 = "tf.All"(%arg0, %arg1) {keep_dims = false} : (tensor<2x2xi1>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testAllFloat(%arg0: tensor<2x2xi1>, %arg1: tensor) -> tensor { + // expected-error @+1 {{'tf.All' op operand #1 must be tensor of 32/64-bit signed integer values}} + %0 = "tf.All"(%arg0, %arg1) {keep_dims = false} : (tensor<2x2xi1>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testAllI32(%arg0: tensor<2x2xi32>, %arg1: tensor) -> tensor { + // expected-error @+1 {{'tf.All' op operand #0 must be tensor of bool values}} + %0 = "tf.All"(%arg0, %arg1) {keep_dims = false} : (tensor<2x2xi32>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testEqualOpIncompatibleShapeTrue(%x: tensor<5xf32>, %y: tensor<4xf32>) -> tensor<5xi1> { + // expected-error @+1 {{operands don't have broadcast-compatible shapes}} + %0 = "tf.Equal"(%x, %y) {incompatible_shape_error = true} : (tensor<5xf32>, tensor<4xf32>) -> tensor<5xi1> + func.return %0 : tensor<5xi1> +} + +// ----- + +// CHECK-LABEL: testEqualOpIncompatibleShapeFalse +func.func @testEqualOpIncompatibleShapeFalse(%x: tensor<5xf32>, %y: tensor<4xf32>) -> tensor<*xi1> { + %0 = "tf.Equal"(%x, %y) {incompatible_shape_error = false} : (tensor<5xf32>, tensor<4xf32>) -> tensor<*xi1> + func.return %0 : tensor<*xi1> +} + +// ----- + +func.func @testNotEqualOpIncompatibleShapeTrue(%x: tensor<5xf32>, %y: tensor<4xf32>) -> tensor<5xi1> { + // expected-error @+1 {{operands don't have broadcast-compatible shapes}} + %0 = "tf.NotEqual"(%x, %y) {incompatible_shape_error = true} : (tensor<5xf32>, tensor<4xf32>) -> tensor<5xi1> + func.return %0 : tensor<5xi1> +} + +// ----- + +// CHECK-LABEL: testNotEqualOpIncompatibleShapeFalse +func.func @testNotEqualOpIncompatibleShapeFalse(%x: tensor<5xf32>, %y: tensor<4xf32>) -> tensor<*xi1> { + %0 = "tf.NotEqual"(%x, %y) {incompatible_shape_error = false} : (tensor<5xf32>, tensor<4xf32>) -> tensor<*xi1> + func.return %0 : tensor<*xi1> +} + +// ----- + +func.func @testConcatV2(%arg: tensor<8x16xf32>, %axis: tensor<1x1xi32>) -> tensor<*xf32> { // expected-error @+1 {{requires axis to be of scalar type (or vector type for older versions)}} + %0 = "tf.ConcatV2"(%arg, %arg, %axis) : (tensor<8x16xf32>, tensor<8x16xf32>, tensor<1x1xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testConcatV2(%arg: tensor<8x16xf32>, %axis: tensor<1x1xi32>) -> tensor<*xf32> { + // expected-error @+1 {{requires axis to be of scalar type (or vector type for older versions)}} + %0 = "tf.Concat"(%axis, %arg, %arg) : (tensor<1x1xi32>, tensor<8x16xf32>, tensor<8x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testConcatV2(%arg0: tensor<8x16xf32>, %arg1: tensor<8xf32>, %axis: tensor) -> tensor<*xf32> { + // expected-error @+1 {{operand type 'tensor<8xf32>' is not compatible with preceding operands; expected rank: 2}} + %0 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<8x16xf32>, tensor<8xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Valid Concat operation with concat axis 1 or -1. +func.func @testConcatV2(%arg0: tensor<8x16xf32>, %arg1: tensor<8x8xf32>, %axis: tensor) -> tensor<*xf32> { + %0 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<8x16xf32>, tensor<8x8xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testConcatV2(%arg0: tensor<8x16xf32>, %arg1: tensor<16x8xf32>, %axis: tensor) -> tensor<*xf32> { + // expected-error @+1 {{operand type 'tensor<16x8xf32>' is not compatible with preceding operands; expected dimension at index 1: 16}} + %0 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<8x16xf32>, tensor<16x8xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Valid Concat operation with concat axis 1 or -1. +func.func @testConcatV2(%arg0: tensor<8x8xf32>, %arg1: tensor, %arg2: tensor<*xf32>, %arg3: tensor<8x?xf32>, %axis: tensor) -> tensor<*xf32> { + %0 = "tf.ConcatV2"(%arg0, %arg1, %arg2, %arg3, %axis) : (tensor<8x8xf32>, tensor, tensor<*xf32>, tensor<8x?xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testInvalidInvertPermutationOp(%arg0: tensor<8x8xi32>) -> tensor<8x8xi32> { + // expected-error @+1 {{'tf.InvertPermutation' op requires input x to be 1-dimensional}} + %0 = "tf.InvertPermutation"(%arg0) : (tensor<8x8xi32>) -> tensor<8x8xi32> + func.return %0 : tensor<8x8xi32> +} + +// ----- + +// Valid Pack operation. +func.func @testPack(%arg0: tensor<4x8xf32>, %arg1: tensor<4x8xf32>) -> tensor<*xf32> { + %0 = "tf.Pack"(%arg0, %arg1) {axis = 1 : i64} : (tensor<4x8xf32>, tensor<4x8xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + + +// ----- + +func.func @testPack(%arg0: tensor<4x8xf32>, %arg1: tensor<4x2xf32>) -> tensor<*xf32> { + // expected-error @+1 {{operand type 'tensor<4x2xf32>' is not compatible with preceding operands; expected dimension at index 1: 8}} + %0 = "tf.Pack"(%arg0, %arg1) {axis = 1 : i64} : (tensor<4x8xf32>, tensor<4x2xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testPack(%arg0: tensor<4x8xf32>, %arg1: tensor<4x8xf32>, %axis: tensor) -> tensor<*xf32> { + // expected-error @+1 {{attribute 'axis' should be within range [-3, 3); actual value: 3}} + %0 = "tf.Pack"(%arg0, %arg1) {axis = 3 : i64} : (tensor<4x8xf32>, tensor<4x8xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// Valid slice operation. +func.func @testSlice(%arg0: tensor<3x4xi32>, %arg1: tensor<2xi64>) -> tensor<1x4xi32> { + %sizes = "tf.Const"() {value = dense<[1, 4]> : tensor<2xi64>} : () -> (tensor<2xi64>) + %0 = "tf.Slice"(%arg0, %arg1, %sizes) : (tensor<3x4xi32>, tensor<2xi64>, tensor<2xi64>) -> tensor<1x4xi32> + func.return %0 : tensor<1x4xi32> +} + +// ----- + +func.func @testSlice_begin_2d(%arg0: tensor<4xi32>, %begins: tensor<2x2xi64>) -> tensor<3xi32> { + %sizes = "tf.Const"() {value = dense<[1]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires begin operand to be 1D tensor}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<2x2xi64>, tensor<1xi64>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testSlice_size_two_much_elements(%arg0: tensor<4xi32>) -> tensor<3xi32> { + %begins = "tf.Const"() {value = dense<[1]> : tensor<1xi64>} : () -> (tensor<1xi64>) + %sizes = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi64>} : () -> (tensor<2xi64>) + // expected-error @+1 {{requires begin and size operands to have the same number of elements}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<2xi64>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testSlice_begin_negative(%arg0: tensor<4xi32>) -> tensor<2xi32> { + %begins = "tf.Const"() {value = dense<[-1]> : tensor<1xi64>} : () -> (tensor<1xi64>) + %sizes = "tf.Const"() {value = dense<[2]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires 0 <= begin[i] <= begin[i] + size[i] <= Di}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +func.func @testSlice_begin_out_of_bound(%arg0: tensor<4xi32>) -> tensor<2xi32> { + %begins = "tf.Const"() {value = dense<[4]> : tensor<1xi64>} : () -> (tensor<1xi64>) + %sizes = "tf.Const"() {value = dense<[2]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires 0 <= begin[i] <= begin[i] + size[i] <= Di}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor<2xi32> + func.return %0 : tensor<2xi32> +} + +// ----- + +func.func @testSlice_unknown_begin_out_of_bounds(%arg0: tensor<4xi32>, %begins: tensor<1xi64>) -> tensor<3xi32> { + %sizes = "tf.Const"() {value = dense<[5]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires size[i] <= Di, even if begin[i] is unknown at compile time}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testSlice_unknown_begin_in_bounds(%arg0: tensor<4xi32>, %begins: tensor<1xi64>) -> tensor<3xi32> { + %sizes = "tf.Const"() {value = dense<[4]> : tensor<1xi64>} : () -> (tensor<1xi64>) + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor<3xi32> + func.return %0 : tensor<3xi32> +} + +// ----- + +func.func @testSlice_unequal_output_input_rank(%arg0: tensor<4xi32>, %begins: tensor<1xi64>) -> tensor { + %sizes = "tf.Const"() {value = dense<[1]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires output to have the same rank as input, but got input rank 1 and output rank 0}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testSlice_wrong_output_size(%arg0: tensor<4xi32>) -> tensor<1xi32> { + %begins = "tf.Const"() {value = dense<[1]> : tensor<1xi64>} : () -> (tensor<1xi64>) + %sizes = "tf.Const"() {value = dense<[2]> : tensor<1xi64>} : () -> (tensor<1xi64>) + // expected-error @+1 {{requires output size to have the same size of slice, got slice size 2 and output size 1}} + %0 = "tf.Slice"(%arg0, %begins, %sizes) : (tensor<4xi32>, tensor<1xi64>, tensor<1xi64>) -> tensor<1xi32> + func.return %0 : tensor<1xi32> +} + +// ----- + +func.func @testSlice_wrong_type(%arg0: tensor<28x1x100xf32>, %arg1: tensor<3xi32>, %arg2: tensor<3xi32>) -> tensor<1x1x100xi32> { + // expected-error @+1 {{failed to verify that input and output must have same element type}} + %0 = "tf.Slice"(%arg0, %arg1, %arg2) : (tensor<28x1x100xf32>, tensor<3xi32>, tensor<3xi32>) -> tensor<1x1x100xi32> + func.return %0 : tensor<1x1x100xi32> +} + +// ----- + +// Valid StridedSlice operation. +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>) -> tensor { + %0 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor<2xi64>, tensor<2xi64>, tensor<2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor, %end: tensor, %strides: tensor) -> tensor { + // expected-error @+1 {{requires begin, end and strides to be 1D tensors}} + %0 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor<32xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>) -> tensor { + // expected-error @+1 {{with less than 32 elements}} + %0 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor<32xi64>, tensor<2xi64>, tensor<2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor, %end: tensor<3xi64>, %strides: tensor<2xi64>) -> tensor { + // expected-error @+1 {{to have the same number of elements}} + %0 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor, tensor<3xi64>, tensor<2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>) -> tensor { + %begin = "tf.Const"() { value = dense<[0, 0]> : tensor<2xi64> } : () -> tensor + %end = "tf.Const"() { value = dense<[5, 10]> : tensor<2xi64> } : () -> tensor + %strides = "tf.Const"() { value = dense<[2, 3, 4]> : tensor<3xi64> } : () -> tensor + + // expected-error @+1 {{to have the same number of elements}} + %1 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor, tensor, tensor) -> tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor<2xi32>, %end: tensor<2xi32>) -> tensor { + %strides = "tf.Const"() { value = dense<[2, 0]> : tensor<2xi32> } : () -> tensor<2xi32> + + // expected-error @+1 {{requires non-zero strides}} + %1 = "tf.StridedSlice"(%input, %begin, %end, %strides) : (tensor<4x8xf32>, tensor<2xi32>, tensor<2xi32>, tensor<2xi32>) -> tensor + func.return %1 : tensor +} + +// ----- + +func.func @testStridedSlice(%input: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>) -> tensor { + // expected-error @+1 {{cannot have multiple ellipses}} + %0 = "tf.StridedSlice"(%input, %begin, %end, %strides) {ellipsis_mask = 3}: (tensor<4x8xf32>, tensor<2xi64>, tensor<2xi64>, tensor<2xi64>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %depth: tensor, %on_value: tensor, %off_value: tensor) -> tensor<3x5xf32> { + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor, tensor, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %on_value: tensor, %off_value: tensor) -> tensor<3x5xf32> { + %depth = "tf.Const"() { value = dense<-5> : tensor } : () -> tensor + // expected-error @+1 {{depth must be non-negative}} + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor, tensor, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %depth: tensor<2xi32>, %on_value: tensor, %off_value: tensor) -> tensor<3x5xf32> { + // expected-error @+1 {{requires depth to be a scalar}} + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor<2xi32>, tensor, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %depth: tensor, %on_value: tensor<2xf32>, %off_value: tensor) -> tensor<3x5xf32> { + // expected-error @+1 {{requires on_value to be a scalar}} + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor, tensor<2xf32>, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %depth: tensor, %on_value: tensor, %off_value: tensor<2xf32>) -> tensor<3x5xf32> { + // expected-error @+1 {{requires off_value to be a scalar}} + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor, tensor, tensor<2xf32>) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testOneHot(%indices: tensor<3xi32>, %depth: tensor, %on_value: tensor, %off_value: tensor) -> tensor<3x5xf32> { + // expected-error @+1 {{expected axis (-2) to be -1 or between [0, 1]}} + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -2 : i64} : (tensor<3xi32>, tensor, tensor, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +// ----- + +func.func @testSplitNonConstSplitDim(%input: tensor<4x4xf32>, %split_dim: tensor) { + %0:2 = "tf.Split"(%split_dim, %input) : (tensor, tensor<4x4xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +func.func @testSplitUnknownRankSplitDim(%input: tensor<4x4xf32>, %split_dim: tensor<*xi32>) { + %0:2 = "tf.Split"(%split_dim, %input) : (tensor<*xi32>, tensor<4x4xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +func.func @testSplitUnknownRankInput(%input: tensor<*xf32>) { + %cst = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %0:2 = "tf.Split"(%cst, %input) : (tensor, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +func.func @testSplitUnknownDimInput(%input: tensor<4x?x4xf32>) { + %cst = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %0:2 = "tf.Split"(%cst, %input) : (tensor, tensor<4x?x4xf32>) -> (tensor<4x?x4xf32>, tensor<4x?x4xf32>) + func.return +} + +// ----- + +func.func @testSplitNonScalarSplitDim(%input: tensor<4x4xf32>, %split_dim: tensor<1xi32>) { + // expected-error @+1 {{split dimension should be an integer scalar tensor}} + %0:2 = "tf.Split"(%split_dim, %input) : (tensor<1xi32>, tensor<4x4xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitScalarInput(%input: tensor, %split_dim: tensor) { + // expected-error @+1 {{cannot split scalar input tensor}} + %0:2 = "tf.Split"(%split_dim, %input) : (tensor, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitLargeSplitDim(%input: tensor<4x8xf32>) { + %cst = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + // expected-error @+1 {{split dimension must be in range [-2, 2)}} + %0:2 = "tf.Split"(%cst, %input) : (tensor, tensor<4x8xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitSmallSplitDim(%input: tensor<4x8xf32>) { + %cst = "tf.Const"() {value = dense<-3> : tensor} : () -> tensor + // expected-error @+1 {{split dimension must be in range [-2, 2)}} + %0:2 = "tf.Split"(%cst, %input) : (tensor, tensor<4x8xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitSmallSplitDim(%input: tensor<4x8xf32>) { + %cst = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // expected-error @+1 {{dimension #0 not divisible by the number of result tensors}} + %0:3 = "tf.Split"(%cst, %input) : (tensor, tensor<4x8xf32>) -> (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSqueezeOutOfBounds(%arg0: tensor) -> tensor { + // expected-error @+1 {{squeeze dimension -4 not in [-3, 3)}} + %0 = "tf.Squeeze"(%arg0) { squeeze_dims = [-4] }: (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testNullaryEinsum(%arg0: tensor<2x3xf32>){ + // expected-error @+1 {{op must have 1 or 2 operands}} + "tf.Einsum"() {equation = "->"} : () -> (tensor) + func.return +} + +// ----- + +func.func @testTernaryEinsum(%arg0: tensor<2x3xf32>){ + // expected-error @+1 {{op must have 1 or 2 operands}} + %0 = "tf.Einsum"(%arg0, %arg0, %arg0) {equation = "ab,cd,ef->"} : (tensor<2x3xf32>, tensor<2x3xf32>, tensor<2x3xf32>) -> (tensor<*xf32>) + func.return +} + +// ----- + +func.func @testTopKV2WrongInputRank(%input: tensor, %k: tensor) { + // expected-error @+1 {{op requires input operand to have at least 1 dimension}} + %0:2 = "tf.TopKV2"(%input, %k) : (tensor, tensor) -> (tensor<*xf32>, tensor<*xi32>) + func.return +} + +// ----- + +func.func @testTopKV2WrongKRank(%input: tensor<8xf32>, %k: tensor<5xi32>) { + // expected-error @+1 {{op requires k operand to be 0D tensor}} + %0:2 = "tf.TopKV2"(%input, %k) : (tensor<8xf32>, tensor<5xi32>) -> (tensor<*xf32>, tensor<*xi32>) + func.return +} + +// ----- + +func.func @testSplitVScalarInput(%input: tensor, %split_sizes: tensor<2xi32>, %split_dim: tensor) { + // expected-error @+1 {{cannot split scalar input tensor}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVNonScalarSplitDim(%input: tensor<4x4xf32>, %split_sizes: tensor<2xi32>, %split_dim: tensor<1xi32>) { + // expected-error @+1 {{split dimension should be an integer scalar tensor}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor<1xi32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVSplitDimOutOfRange(%input: tensor<4x4xf32>, %split_sizes: tensor<2xi32>) { + %split_dim = "tf.Const"() {value = dense<100>: tensor} : () -> (tensor) + // expected-error @+1 {{split dimension must be in range [-2, 2)}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVWrongSplitSizesType(%input: tensor<4x4xf32>, %split_sizes: tensor<2x2xi32>, %split_dim: tensor) { + // expected-error @+1 {{op split sizes should be a 1D tensor of 2 elements}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2x2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVMultipleDynamicSizes(%input: tensor<4x4xf32>) { + %split_dim = "tf.Const"() {value = dense<1>: tensor} : () -> (tensor) + %split_sizes = "tf.Const"() {value = dense<[-1, -1]>: tensor<2xi32>} : () -> (tensor<2xi32>) + // expected-error @+1 {{cannot have more than one dynamic dimension in split sizes}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVSplitSizeOutOfRange(%input: tensor<4x4xf32>) { + %split_dim = "tf.Const"() {value = dense<1>: tensor} : () -> (tensor) + %split_sizes = "tf.Const"() {value = dense<[-1, 100]>: tensor<2xi32>} : () -> (tensor<2xi32>) + // expected-error @+1 {{split sizes must sum up to be less than or equal to the dimension size along split dimension, found 100 vs 4}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVSplitSizeOutOfRange(%input: tensor<4x4xf32>) { + %split_dim = "tf.Const"() {value = dense<1>: tensor} : () -> (tensor) + %split_sizes = "tf.Const"() {value = dense<[2, 3]>: tensor<2xi32>} : () -> (tensor<2xi32>) + // expected-error @+1 {{split sizes must sum up to the dimension size along split dimension, found 5 vs 4}} + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitV1(%input: tensor<4x4xf32>) { + %split_dim = "tf.Const"() {value = dense<1>: tensor} : () -> (tensor) + %split_sizes = "tf.Const"() {value = dense<[-1, 4]>: tensor<2xi32>} : () -> (tensor<2xi32>) + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +func.func @testSplitV2(%input: tensor<4x4xf32>) { + %split_dim = "tf.Const"() {value = dense<1>: tensor} : () -> (tensor) + %split_sizes = "tf.Const"() {value = dense<[3, 1]>: tensor<2xi32>} : () -> (tensor<2xi32>) + %0:2 = "tf.SplitV"(%input, %split_sizes, %split_dim) : (tensor<4x4xf32>, tensor<2xi32>, tensor) -> (tensor<*xf32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @testSplitVDynamic(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> (tensor, tensor) { + %0:2 = "tf.SplitV"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> (tensor, tensor) + func.return %0#0, %0#1 : tensor, tensor +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.All +//===--------------------------------------------------------------------===// + +func.func @testAllDimWrongRank(%input: tensor<4x6xi1>, %dims: tensor<2x2xi32>) { + // expected-error @+1 {{dimensions can only be 0D or 1D tensor}} + %0 = "tf.All"(%input, %dims) : (tensor<4x6xi1>, tensor<2x2xi32>) -> (tensor<*xi1>) + func.return +} + +// ----- + +func.func @testAllDimOutOfRange(%input: tensor<4x6xi1>) { + %dims = "tf.Const"() {value = dense<[-1, 5]> : tensor<2xi32>} : () -> (tensor<2xi32>) + // expected-error @+1 {{1-th dimension should be in the range of [-2, 2)}} + %0 = "tf.All"(%input, %dims) : (tensor<4x6xi1>, tensor<2xi32>) -> (tensor<*xi1>) + func.return +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.Any +//===--------------------------------------------------------------------===// + +func.func @testAnyDimWrongRank(%input: tensor<4x6xi1>, %dims: tensor<2x2xi32>) { + // expected-error @+1 {{dimensions can only be 0D or 1D tensor}} + %0 = "tf.Any"(%input, %dims) : (tensor<4x6xi1>, tensor<2x2xi32>) -> (tensor<*xi1>) + func.return +} + +// ----- + +func.func @testAnyDimOutOfRange(%input: tensor<4x6xi1>) { + %dims = "tf.Const"() {value = dense<[-1, 5]> : tensor<2xi32>} : () -> (tensor<2xi32>) + // expected-error @+1 {{1-th dimension should be in the range of [-2, 2)}} + %0 = "tf.Any"(%input, %dims) : (tensor<4x6xi1>, tensor<2xi32>) -> (tensor<*xi1>) + func.return +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.Unpack +//===--------------------------------------------------------------------===// + +func.func @testUnpackAxisOutOfRange(%input: tensor<2x6xf32>) { + // expected-error @+1 {{axis attribute must be in the range of [-2, 2)}} + %0:2 = "tf.Unpack"(%input) {axis = 5} : (tensor<2x6xf32>) -> (tensor<6xf32>, tensor<6xf32>) + func.return +} + +// ----- + +func.func @testAxisUnknownDim(%input: tensor) { + // CHECK: tf.Unpack + %0:2 = "tf.Unpack"(%input) {axis = 0} : (tensor) -> (tensor<6xf32>, tensor<6xf32>) + func.return +} + +// ----- + +func.func @testAxisDim(%input: tensor<2x6xf32>) { + // expected-error @+1 {{result count must be equal to 6}} + %0:2 = "tf.Unpack"(%input) {axis = -1} : (tensor<2x6xf32>) -> (tensor<6xf32>, tensor<6xf32>) + func.return +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.UnsortedSegment{Max|Min|Prod|Sum} +//===--------------------------------------------------------------------===// + +// CHECK-LABEL: unsortedSegmentReduction +func.func @unsortedSegmentReduction(%data: tensor, %segment_ids: tensor<7x?xi32>, %num_segments: tensor) { + // CHECK: tf.UnsortedSegmentMin + %0 = "tf.UnsortedSegmentMin"(%data, %segment_ids, %num_segments) : (tensor, tensor<7x?xi32>, tensor) -> (tensor) + func.return +} + +// ----- + +func.func @unsortedSegmentReduction(%data: tensor<7x10x8xf32>, %segment_ids: tensor<7x10xi32>, %num_segments: tensor<2x3xi32>) { + // expected-error @+1 {{number of segments should be a 0-D tensor}} + %0 = "tf.UnsortedSegmentMax"(%data, %segment_ids, %num_segments) : (tensor<7x10x8xf32>, tensor<7x10xi32>, tensor<2x3xi32>) -> (tensor) + func.return +} + +// ----- + +func.func @unsortedSegmentReduction(%data: tensor<7x10x8xf32>, %segment_ids: tensor<7x9xi32>, %num_segments: tensor) { + // expected-error @+1 {{requires segment ids shape to be a prefix of data shape, but dimension #1 differs: 9 vs. 10}} + %0 = "tf.UnsortedSegmentProd"(%data, %segment_ids, %num_segments) : (tensor<7x10x8xf32>, tensor<7x9xi32>, tensor) -> (tensor) + func.return +} + +// ----- + +func.func @unsortedSegmentReduction(%data: tensor<7x10x8xf32>, %segment_ids: tensor<7x10x8x1xi32>, %num_segments: tensor) { + // expected-error @+1 {{requires segment ids rank to be less than or equal to data's rank}} + %0 = "tf.UnsortedSegmentSum"(%data, %segment_ids, %num_segments) : (tensor<7x10x8xf32>, tensor<7x10x8x1xi32>, tensor) -> (tensor) + func.return +} + +// ----- + +func.func @unsortedSegmentReduction(%data: tensor<7x10x8xf32>, %segment_ids: tensor<7x10xi32>) { + %num_segments = "tf.Const"() {value = dense<-5> : tensor} : () -> (tensor) + // expected-error @+1 {{num of segments cannot be negative}} + %0 = "tf.UnsortedSegmentSum"(%data, %segment_ids, %num_segments) : (tensor<7x10x8xf32>, tensor<7x10xi32>, tensor) -> (tensor) + func.return +} + +// ----- + + +//===--------------------------------------------------------------------===// +// tf.GatherV2 +//===--------------------------------------------------------------------===// + +func.func @testGatherV2(%arg0: tensor<16x2x3xf32>, %arg1: tensor<16x5xi32>) -> tensor<16x2x5x3xf32> { + %0 = "tf.Const"() { value = dense<[-1]> : tensor<1xi32> } : () -> tensor<1xi32> + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = -1 : i64} : (tensor<16x2x3xf32>, tensor<16x5xi32>, tensor<1xi32>) -> tensor<16x2x5x3xf32> + func.return %1 : tensor<16x2x5x3xf32> +} + +// ----- + +// Verify that the batch_dims can be equal to the rank of the indices. +func.func @testGatherV2(%arg0: tensor<16x4xf32>, %arg1: tensor<16xi32>) -> tensor<16xf32> { + %0 = "tf.Const"() { value = dense<[1]> : tensor<1xi32> } : () -> tensor<1xi32> + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = 1 : i64} : (tensor<16x4xf32>, tensor<16xi32>, tensor<1xi32>) -> tensor<16xf32> + func.return %1 : tensor<16xf32> +} + +// ----- + +func.func @testGatherV2(%arg0: tensor<16x2x3xf32>, %arg1: tensor<16x5xi32>) -> tensor<16x2x5x3xf32> { + %0 = "tf.Const"() { value = dense<[-1]> : tensor<1xi32> } : () -> tensor<1xi32> + // expected-error @+1 {{batch_dims (-3) must be in range [-2, 3)}} + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = -3 : i64} : (tensor<16x2x3xf32>, tensor<16x5xi32>, tensor<1xi32>) -> tensor<16x2x5x3xf32> + func.return %1 : tensor<16x2x5x3xf32> +} + +// ----- + +func.func @testGatherV2(%arg0: tensor<16x2x3xf32>, %arg1: tensor<16x5xi32>) -> tensor<16x2x5x3xf32> { + %0 = "tf.Const"() { value = dense<[[-4]]> : tensor<1x1xi32> } : () -> tensor<1x1xi32> + // expected-error @+1 {{requires axis to have rank at most 1}} + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = -1 : i64} : (tensor<16x2x3xf32>, tensor<16x5xi32>, tensor<1x1xi32>) -> tensor<16x2x5x3xf32> + func.return %1 : tensor<16x2x5x3xf32> +} + +// ----- + +func.func @testGatherV2(%arg0: tensor<16x2x3xf32>, %arg1: tensor<16x5xi32>) -> tensor<16x2x5x3xf32> { + %0 = "tf.Const"() { value = dense<[-4]> : tensor<1xi32> } : () -> tensor<1xi32> + // expected-error @+1 {{axis (-4) must be in range [-3, 3)}} + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = -1 : i64} : (tensor<16x2x3xf32>, tensor<16x5xi32>, tensor<1xi32>) -> tensor<16x2x5x3xf32> + func.return %1 : tensor<16x2x5x3xf32> +} + +// ----- + +func.func @testGatherV2(%arg0: tensor<16x2x3xf32>, %arg1: tensor<16x5xi32>) -> tensor<16x2x5x3xf32> { + %0 = "tf.Const"() { value = dense<[0]> : tensor<1xi32> } : () -> tensor<1xi32> + // expected-error @+1 {{requires axis (0) to be greater than or equal to batch_dims (1)}} + %1 = "tf.GatherV2"(%arg0, %arg1, %0) {batch_dims = -1 : i64} : (tensor<16x2x3xf32>, tensor<16x5xi32>, tensor<1xi32>) -> tensor<16x2x5x3xf32> + func.return %1 : tensor<16x2x5x3xf32> +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.StridedSliceGrad +//===--------------------------------------------------------------------===// + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + // CHECK: tf.StridedSliceGrad + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor, %end: tensor<2xi64>, %strides: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + // expected-error @+1 {{requires begin, end and strides to be 1D tensors}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor, tensor<2xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor<32xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + // expected-error @+1 {{with less than 32 elements}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor<32xi64>, tensor<2xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor, %end: tensor<3xi64>, %strides: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + // expected-error @+1 {{have the same number of elements}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor, tensor<3xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %shape: tensor<2xi64>) -> tensor { + %begin = "tf.Const"() { value = dense<[0, 0]> : tensor<2xi64> } : () -> tensor + %end = "tf.Const"() { value = dense<[5, 10]> : tensor<2xi64> } : () -> tensor + %strides = "tf.Const"() { value = dense<[2, 3, 4]> : tensor<3xi64> } : () -> tensor + + // expected-error @+1 {{have the same number of elements}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor, tensor, tensor, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + %strides = "tf.Const"() { value = dense<[2, 0]> : tensor<2xi32> } : () -> tensor<2xi32> + + // expected-error @+1 {{requires non-zero strides}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<2xi32>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>, %shape: tensor<2xi64>) -> tensor { + // expected-error @+1 {{cannot have multiple ellipses}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) {ellipsis_mask = 3} : (tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @stridedSliceGrad(%dy: tensor<4x8xf32>, %begin: tensor<2xi64>, %end: tensor<2xi64>, %strides: tensor<2xi64>, %shape: tensor<1x2xi64>) -> tensor { + // expected-error @+1 {{'shape' operand must be 1D tensor, but got 2D tensor}} + %0 = "tf.StridedSliceGrad"(%shape, %begin, %end, %strides, %dy) : (tensor<1x2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<2xi64>, tensor<4x8xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2x2xf32>) -> tensor<2x2xf32> { + %indices = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<2x2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// ----- + +func.func @testDynamicStitch() -> tensor<2x2xf32> { + // expected-error @+1 {{requires attribute N with value >= 1}} + %0 = "tf.DynamicStitch"() : () -> (tensor<2x2xf32>) + func.return %0 : tensor<2x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2x2xf32>) -> tensor { + %indices = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires non scalar output}} + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<2x2xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2x2xf32>) -> tensor<2x2xf32> { + %indices = "tf.Const"() {value = dense<[-1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires non-negative index values; found -1}} + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<2x2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<3x2xf32>) -> tensor<2x2xf32> { + %indices = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{requires shape of data with type 'tensor<3x2xf32>' to have prefix matching with shape of the corresponding index type 'tensor<2xi32>'}} + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<3x2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2xf32>, %arg1: tensor<2x2x3xf32>) -> (tensor<5x2xf32>) { + %indices0 = "tf.Const"() {value = dense<4> : tensor} : () -> tensor + %indices1 = "tf.Const"() {value = dense<[[3, 2], [1, 0]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + + // expected-error @+1 {{inconsistent shaped data and index pairs; inferred item shapes [2] and [3] don't match}} + %0 = "tf.DynamicStitch"(%indices0, %indices1, %arg0, %arg1) : (tensor, tensor<2x2xi32>, tensor<2xf32>, tensor<2x2x3xf32>) -> tensor<5x2xf32> + func.return %0 : tensor<5x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2x2xf32>) -> tensor<2x2xf32> { + %indices = "tf.Const"() {value = dense<[2, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{missing index 1}} + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<2x2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor<2x2xf32>) -> tensor<3x2xf32> { + %indices = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + // expected-error @+1 {{has invalid output type; should be compatible with inferred type 'tensor<2x2xf32>'}} + %0 = "tf.DynamicStitch"(%indices, %arg0) : (tensor<2xi32>, tensor<2x2xf32>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor, %arg1: tensor) -> (tensor<*xf32>) { + // expected-error @+1 {{requires shape of data with type 'tensor' to have prefix matching with shape of the corresponding index type 'tensor'}} + %0 = "tf.DynamicStitch"(%arg0, %arg1) : (tensor, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testDynamicStitch(%arg0: tensor, %arg1: tensor<2x?xf32>) -> (tensor<2x3x2xf32>) { + %indices0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %indices1 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + + // expected-error @+1 {{has invalid output type; should be compatible with inferred type 'tensor<2x2x3xf32>'}} + %0 = "tf.DynamicStitch"(%indices0, %indices1, %arg0, %arg1) : (tensor, tensor, tensor, tensor<2x?xf32>) -> tensor<2x3x2xf32> + func.return %0 : tensor<2x3x2xf32> +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor, %shape0: tensor<3xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires N to be at least 2, got 1}} + %0 = "tf.ConcatOffset"(%concat_dim, %shape0) : (tensor, tensor<3xi32>) -> tensor<3xi32> + func.return +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor, %shape0: tensor<3xi32>, %shape1: tensor<3xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires sizes of shapes and offsets to be the same, got sizes 2 and 3}} + %0:3 = "tf.ConcatOffset"(%concat_dim, %shape0, %shape1) : (tensor, tensor<3xi32>, tensor<3xi32>) -> (tensor<3xi32>, tensor<3xi32>, tensor<3xi32>) + func.return +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor<1xi32>, %shape0: tensor<3xi32>, %shape1: tensor<3xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires concat_dim to be a scalar, got tensor of rank 1}} + %0:2 = "tf.ConcatOffset"(%concat_dim, %shape0, %shape1) : (tensor<1xi32>, tensor<3xi32>, tensor<3xi32>) -> (tensor<3xi32>, tensor<3xi32>) + func.return +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor, %shape0: tensor<3xi32>, %shape1: tensor<3xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires operand and result 1 to have compatible shapes}} + %0:2 = "tf.ConcatOffset"(%concat_dim, %shape0, %shape1) : (tensor, tensor<3xi32>, tensor<3xi32>) -> (tensor<3xi32>, tensor<8xi32>) + func.return +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor, %shape0: tensor<3xi32>, %shape1: tensor<3x3xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires shape tensor operand 1 to be of rank 1, got tensor of rank 2}} + %0:2 = "tf.ConcatOffset"(%concat_dim, %shape0, %shape1) : (tensor, tensor<3xi32>, tensor<3x3xi32>) -> (tensor<3xi32>, tensor<3x3xi32>) + func.return +} + +// ----- + +func.func @testConcatOffest(%concat_dim: tensor, %shape0: tensor<3xi32>, %shape1: tensor<8xi32>) { + // expected-error @+1 {{'tf.ConcatOffset' op requires shape tensor (rank 1) operand 1 to be of length 3, got tensor (rank 1) of length 8}} + %0:2 = "tf.ConcatOffset"(%concat_dim, %shape0, %shape1) : (tensor, tensor<3xi32>, tensor<8xi32>) -> (tensor<3xi32>, tensor<8xi32>) + func.return +} + +// ----- + +func.func @tensor_scatter_update(%tensor: tensor, %indices: tensor<4x2xi32>, %updates: tensor<4x4xf32>) -> tensor { + // expected-error @+1 {{op requires tensor operand to have at least 1 dimension}} + %0 = "tf.TensorScatterUpdate"(%tensor, %indices, %updates) : (tensor, tensor<4x2xi32>, tensor<4x4xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @tensor_scatter_update(%tensor: tensor<4x4x4xf32>, %indices: tensor, %updates: tensor<4x4xf32>) -> tensor<4x4x4xf32> { + // expected-error @+1 {{op requires indices operand to have at least 1 dimension}} + %0 = "tf.TensorScatterUpdate"(%tensor, %indices, %updates) : (tensor<4x4x4xf32>, tensor, tensor<4x4xf32>) -> tensor<4x4x4xf32> + func.return %0 : tensor<4x4x4xf32> +} + +// ----- + +func.func @tensor_scatter_update(%tensor: tensor<4x4x4xf32>, %indices: tensor<4x2xi32>, %updates: tensor) -> tensor<4x4x4xf32> { + // CHECK: TensorScatterUpdate + %0 = "tf.TensorScatterUpdate"(%tensor, %indices, %updates) : (tensor<4x4x4xf32>, tensor<4x2xi32>, tensor) -> tensor<4x4x4xf32> + func.return %0 : tensor<4x4x4xf32> +} + +// ----- + +func.func @tensor_scatter_update(%tensor: tensor<4xf32>, %indices: tensor<4x2xi32>, %updates: tensor<4x4xf32>) -> tensor<4x4x4xf32> { + // expected-error @+1 {{op requires tensor operand with rank greater than or equal to the indices operand's last dimensions}} + %0 = "tf.TensorScatterUpdate"(%tensor, %indices, %updates) : (tensor<4xf32>, tensor<4x2xi32>, tensor<4x4xf32>) -> tensor<4x4x4xf32> + func.return %0 : tensor<4x4x4xf32> +} + +// ----- + +// CHECK-LABEL: func @testParseExampleV2DenseOnlyValid +func.func @testParseExampleV2DenseOnlyValid(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %dense_keys : tensor<2x!tf_type.string>, %dense_default_0 : tensor, %dense_default_1 : tensor) -> (tensor<32xf32>) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %result:2 = "tf.ParseExampleV2"(%serialized, %names, %empty_str_vector, %dense_keys, %empty_str_vector, %dense_default_0, %dense_default_1) {dense_shapes = [#tf_type.shape<>, #tf_type.shape<>], num_sparse = 0 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor, tensor) -> (tensor<32xf32>, tensor<32xf32>) + func.return %result#0 : tensor<32xf32> +} + +// ----- + +func.func @testParseExampleV2DenseMismatchedInputOutput(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %dense_keys : tensor<2x!tf_type.string>, %dense_default_0 : tensor, %dense_default_1 : tensor) -> (tensor<32xf32>) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-error @+1 {{output 'dense_values' should have same length as attribute 'Tdense'}} + %result:3 = "tf.ParseExampleV2"(%serialized, %names, %empty_str_vector, %dense_keys, %empty_str_vector, %dense_default_0, %dense_default_1) {dense_shapes = [#tf_type.shape<>, #tf_type.shape<>], num_sparse = 0 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor, tensor) -> (tensor<32xf32>, tensor<32xf32>, tensor<32xi64>) + func.return %result#0 : tensor<32xf32> +} + +// ----- + +// CHECK-LABEL: func @testParseExampleV2SparseOnlyValid +func.func @testParseExampleV2SparseOnlyValid(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %sparse_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %result:6 = "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %empty_str_vector, %empty_str_vector) {dense_shapes = [], num_sparse = 2 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>) -> (tensor, tensor, tensor, tensor, tensor<2xi64>, tensor<2xi64>) + func.return %result#0 : tensor +} + +// ----- + +func.func @testParseExampleV2SparseInvalidNumSparse(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %sparse_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-error @+1 {{attribute 'num_sparse' should be the same as the length of attribute 'sparse_types'}} + %result:6 = "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %empty_str_vector, %empty_str_vector) {dense_shapes = [], num_sparse = 3 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>) -> (tensor, tensor, tensor, tensor, tensor<2xi64>, tensor<2xi64>) + func.return %result#0 : tensor +} + +// ----- + +func.func @testParseExampleV2SparseInvalidSparseIndicesOutput(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %sparse_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-error @+1 {{output 'sparse_indices' should have same length as attribute 'sparse_types'}} + %result:5 = "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %empty_str_vector, %empty_str_vector) {dense_shapes = [], num_sparse = 2 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>) -> (tensor, tensor, tensor, tensor<2xi64>, tensor<2xi64>) + func.return %result#0 : tensor +} + +// ----- + +func.func @testParseExampleV2SparseOnlyValid(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %sparse_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-error @+1 {{output 'sparse_shapes' should have same length as attribute 'sparse_types'}} + %result:5 = "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %empty_str_vector, %empty_str_vector) {dense_shapes = [], num_sparse = 2 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>) -> (tensor, tensor, tensor, tensor, tensor<2xi64>) + func.return %result#0 : tensor +} + +// ----- + +// CHECK-LABEL: func @testParseExampleV2RaggedOnlyValid +func.func @testParseExampleV2RaggedOnlyValid(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %ragged_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %result:4 = "tf.ParseExampleV2"(%serialized, %names, %empty_str_vector, %empty_str_vector, %ragged_keys) {dense_shapes = [], num_sparse = 0 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor, tensor, tensor, tensor) + func.return %result#0 : tensor +} + +// ----- + +func.func @testParseExampleV2RaggedMismatchedOutputLengths(%serialized: tensor<32x!tf_type.string>, %names : tensor<32x!tf_type.string>, %ragged_keys : tensor<2x!tf_type.string>) -> (tensor) { + %empty_str_vector = "tf.Const"() {dtype = !tf_type.string, value = #tf_type : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-error @+1 {{attribute 'ragged_value_types' should have same length as attribute 'ragged_split_types'}} + %result:3 = "tf.ParseExampleV2"(%serialized, %names, %empty_str_vector, %empty_str_vector, %ragged_keys) {dense_shapes = [], num_sparse = 0 : i64, resultSegmentSizes = array} : (tensor<32x!tf_type.string>, tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor, tensor, tensor) + func.return %result#0 : tensor +} + +// ----- + +// Legal BatchMatMul op. +func.func @testBatchMatMul(%lhs: tensor<2x?x2x?x3x5xf32>, %rhs: tensor<2x2x?x?x5x7xf32>) { + %0 = "tf.BatchMatMul"(%lhs, %rhs) : (tensor<2x?x2x?x3x5xf32>, tensor<2x2x?x?x5x7xf32>) -> tensor<2x?x?x?x3x7xf32> + func.return +} + +// ----- + +// Mismatching batch dimensions. +func.func @testBatchMatMul(%lhs: tensor<1x3x5xf32>, %rhs: tensor<2x5x7xf32>) { + // expected-error @+1 {{found mismatching batch dimensions for lhs shape 'tensor<1x3x5xf32>' and rhs shape 'tensor<2x5x7xf32>'}} + %0 = "tf.BatchMatMul"(%lhs, %rhs) : (tensor<1x3x5xf32>, tensor<2x5x7xf32>) -> tensor<2x3x7xf32> +} + +// ----- + +func.func @testBatchMatMulV2(%lhs: tensor, %rhs: tensor<10x10xf32>) { + // expected-error @+1 {{requires lhs operand to have rank at least two}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor, tensor<10x10xf32>) -> tensor<10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2(%lhs: tensor<10x10xf32>, %rhs: tensor) { + // expected-error @+1 {{requires rhs operand to have rank at least two}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x10xf32>, tensor) -> tensor<10x10xf32> +} + +// ----- + +// CHECK-LABEL: func @testBatchMatMulV2NoBatchDimension +func.func @testBatchMatMulV2NoBatchDimension(%lhs: tensor<5x10xf32>, %rhs: tensor<10x10xf32>) -> (tensor<5x10xf32>) { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<5x10xf32>, tensor<10x10xf32>) -> tensor<5x10xf32> + func.return %0 : tensor<5x10xf32> +} + +// ----- + +// CHECK-LABEL: func @testBatchMatMulV2ValidBroadcastingBatchDimension +func.func @testBatchMatMulV2ValidBroadcastingBatchDimension(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x10xf32>) -> (tensor<10x2x5x10xf32>) { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x10xf32>) -> tensor<10x2x5x10xf32> + func.return %0 : tensor<10x2x5x10xf32> +} + +// ----- + +// CHECK-LABEL: func @testBatchMatMulV2ValidMultiBatchDimension +func.func @testBatchMatMulV2ValidMultiBatchDimension(%lhs: tensor<4x5x1x3x2xf32>, %rhs: tensor<1x1x3x5xf32>) -> (tensor<4x5x1x2x5xf32>) { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) { adj_x = true } : (tensor<4x5x1x3x2xf32>, tensor<1x1x3x5xf32>) -> tensor<4x5x1x2x5xf32> + func.return %0 : tensor<4x5x1x2x5xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidBroadcastingBatchDimensionWithHigherXRank(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x10x10xf32>) { + // expected-error @+1 {{found incompatible broadcast batch dimensions for lhs shape 'tensor<10x2x5x10xf32>' and rhs shape 'tensor<10x10x10xf32>'}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x10x10xf32>) -> tensor<10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidBroadcastingBatchDimensionWithSameRank(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x10x10x10xf32>) { + // expected-error @+1 {{found incompatible broadcast batch dimensions for lhs shape 'tensor<10x2x5x10xf32>' and rhs shape 'tensor<10x10x10x10xf32>'}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x10x10x10xf32>) -> tensor<10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidBroadcastingBatchDimensionWithHigherYRank(%lhs: tensor<2x5x10xf32>, %rhs: tensor<10x10x10x10xf32>) { + // expected-error @+1 {{found incompatible broadcast batch dimensions for lhs shape 'tensor<2x5x10xf32>' and rhs shape 'tensor<10x10x10x10xf32>'}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<2x5x10xf32>, tensor<10x10x10x10xf32>) -> tensor<10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidOutputBatchDimension(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<2x10x10xf32>) { + // expected-error @+1 {{has mismatching input batch dimension 2 and output batch dimension 3}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<2x10x10xf32>) -> tensor<10x3x10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2DynamicInputBatchDimension(%lhs: tensor, %rhs: tensor) -> tensor<5x2x5x10xf32> { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor, tensor) -> tensor<5x2x5x10xf32> + func.return %0 : tensor<5x2x5x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2DynamicOutputBatchDimension(%lhs: tensor<1x2x5x10xf32>, %rhs: tensor<1x10xf32>) -> tensor { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<1x2x5x10xf32>, tensor<1x10xf32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testBatchMatMulV2InvalidOutputRank(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x1x10x10xf32>) { + // expected-error @+1 {{found invalid output rank, expected 4 but got 3}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x1x10x10xf32>) -> tensor<10x5x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidOutputRowDim(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x10xf32>) { + // expected-error @+1 {{found invalid output dimension on row, expected 5 but got 10}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x10xf32>) -> tensor<10x2x10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2AdjXInvalidOutputRowDim(%lhs: tensor<10x2x10x5xf32>, %rhs: tensor<10x10xf32>) { + // expected-error @+1 {{found invalid output dimension on row, expected 5 but got 10}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) { adj_x = true } : (tensor<10x2x10x5xf32>, tensor<10x10xf32>) -> tensor<10x2x10x10xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidOutputColDim(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<10x10xf32>) { + // expected-error @+1 {{found invalid output dimension on col, expected 10 but got 5}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<10x2x5x10xf32>, tensor<10x10xf32>) -> tensor<10x2x5x5xf32> +} + +// ----- + +func.func @testBatchMatMulV2AdjYInvalidOutputColDim(%lhs: tensor<10x2x5x10xf32>, %rhs: tensor<4x10xf32>) { + // expected-error @+1 {{found invalid output dimension on col, expected 4 but got 10}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) { adj_y = true } : (tensor<10x2x5x10xf32>, tensor<4x10xf32>) -> tensor<10x2x5x10xf32> +} + +// ----- + +// CHECK-LABEL: func @testBatchMatMulV2PartiallyKnownInputBatchDim +func.func @testBatchMatMulV2PartiallyKnownInputBatchDim(%lhs: tensor<4x5x?x3x2xf32>, %rhs: tensor<1x1x3x5xf32>) -> (tensor<4x5x?x2x5xf32>) { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) { adj_x = true } : (tensor<4x5x?x3x2xf32>, tensor<1x1x3x5xf32>) -> tensor<4x5x?x2x5xf32> + func.return %0 : tensor<4x5x?x2x5xf32> +} + +// ----- + +// CHECK-LABEL: func @testBatchMatMulV2PartiallyKnownMatmulDim +func.func @testBatchMatMulV2PartiallyKnownMatmulDim(%lhs: tensor<4x5x1x?x3xf32>, %rhs: tensor<1x1x3x5xf32>) -> (tensor<4x5x1x?x5xf32>) { + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<4x5x1x?x3xf32>, tensor<1x1x3x5xf32>) -> tensor<4x5x1x?x5xf32> + func.return %0 : tensor<4x5x1x?x5xf32> +} + +// ----- + +func.func @testBatchMatMulV2InvalidPartiallyKnownMatmulDim(%lhs: tensor<4x5x1x?x3xf32>, %rhs: tensor<1x1x3x5xf32>) -> (tensor<4x5x1x?x3xf32>) { + // expected-error @+1 {{found invalid output dimension on col, expected 5 but got 3}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) : (tensor<4x5x1x?x3xf32>, tensor<1x1x3x5xf32>) -> tensor<4x5x1x?x3xf32> + func.return %0 : tensor<4x5x1x?x3xf32> +} + +// ----- + +func.func @testBatchMatMulV2AdjXInvalidPartiallyKnownMatmulDim(%lhs: tensor<4x5x1x3x?xf32>, %rhs: tensor<1x1x3x5xf32>) -> (tensor<4x5x1x?x3xf32>) { + // expected-error @+1 {{found invalid output dimension on col, expected 5 but got 3}} + %0 = "tf.BatchMatMulV2"(%lhs, %rhs) { adj_x = true } : (tensor<4x5x1x3x?xf32>, tensor<1x1x3x5xf32>) -> tensor<4x5x1x?x3xf32> + func.return %0 : tensor<4x5x1x?x3xf32> +} + +// ----- + +func.func @testDataFormatVecPermuteInvalid1dInput(%x: tensor<5xi32>) { + // expected-error @+1 {{requires 1D input of size 4}} + %0 = "tf.DataFormatVecPermute"(%x): (tensor<5xi32>) -> tensor<5xi32> + func.return +} + +// ----- + +func.func @testDataFormatVecPermuteInvalid2dDim0Input(%x: tensor<5x2xi32>) { + // expected-error @+1 {{requires first dimensions of 2D input to be of size 4}} + %0 = "tf.DataFormatVecPermute"(%x): (tensor<5x2xi32>) -> tensor<5x2xi32> + func.return +} + +// ----- + +func.func @testDataFormatVecPermuteInvalid2dDim1Input(%x: tensor<4x3xi32>) { + // expected-error @+1 {{requires second dimensions of 2D input to be of size 2}} + %0 = "tf.DataFormatVecPermute"(%x): (tensor<4x3xi32>) -> tensor<4x3xi32> + func.return +} + +// ----- + +func.func @testDataFormatVecPermuteInvalid3dInput(%x: tensor<4x2x2xi32>) { + // expected-error @+1 {{requires input of rank 1 or 2}} + %0 = "tf.DataFormatVecPermute"(%x): (tensor<4x2x2xi32>) -> tensor<4x2x2xi32> + func.return +} + +// ----- + +func.func @testSendTPUEmbeddingGradients(%x: tensor<512x256xf32>) { + "tf.SendTPUEmbeddingGradients"(%x) {N = 1 : i64, NN = 0 : i64, config = "", operandSegmentSizes = array} : (tensor<512x256xf32>) -> () + func.return +} + +// ----- + +//===--------------------------------------------------------------------===// +// tf.BatchToSpace +//===--------------------------------------------------------------------===// + +func.func @testBatchToSpaceDynamic(%arg0: tensor<*xf32>, %arg1: tensor<*xi32>) { + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xf32> + func.return +} + +func.func @testBatchToSpaceRankedInput(%arg0: tensor, %arg1: tensor<*xi32>) { + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor, tensor<*xi32>) -> tensor<*xf32> + func.return +} + +func.func @testBatchToSpaceRankedCrops(%arg0: tensor<*xf32>, %arg1: tensor) { + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor) -> tensor<*xf32> + func.return +} + +func.func @testBatchToSpaceRankedOutput(%arg0: tensor<*xf32>, %arg1: tensor<*xi32>) { + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor + func.return +} + +func.func @testBatchToSpaceStatic(%arg0: tensor<36x8x8x8xf32>) { + %crops = "tf.Const"() {value = dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + %0 = "tf.BatchToSpace"(%arg0, %crops) {block_size = 3 : i64} : (tensor<36x8x8x8xf32>, tensor<2x2xi32>) -> tensor<4x21x17x8xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidInputRank(%arg0: tensor<8xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires input to be a 4D tensor, but got 'tensor<8xf32>'}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<8xf32>, tensor<*xi32>) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidInputBatch(%arg0: tensor<2x4x6x8xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires input batch (dimension 0) to be evenly divisible by (block_size * block_size), but got input batch 2 and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<2x4x6x8xf32>, tensor<*xi32>) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidCropsRank(%arg0: tensor<*xf32>, %arg1: tensor) { + // expected-error @+1 {{'tf.BatchToSpace' op requires crops to be a 2D tensor, but got 'tensor'}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidCropsFirstDim(%arg0: tensor<*xf32>, %arg1: tensor<3x?xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires crops to be a tensor<2x2>, but got 'tensor<3x?xi32>'}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor<3x?xi32>) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidCropsSecondDim(%arg0: tensor<*xf32>, %arg1: tensor) { + // expected-error @+1 {{'tf.BatchToSpace' op requires crops to be a tensor<2x2>, but got 'tensor'}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceBadCropValues(%arg0: tensor<*xf32>) { + %crops = "tf.Const"() {value = dense<[[-1, -2], [-3, -4]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + // expected-error @+1 {{'tf.BatchToSpace' op requires all crop values to be nonnegative, but got dense<[[-1, -2], [-3, -4]]> : tensor<2x2xi32>}} + %0 = "tf.BatchToSpace"(%arg0, %crops) {block_size = 2 : i64} : (tensor<*xf32>, tensor<2x2xi32>) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputRank(%arg0: tensor<*xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires output to be a 4D tensor, but got 'tensor<8xf32>'}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<8xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputBatch(%arg0: tensor<16x8x8x3xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires output batch (dimension 0) to be equal to input batch (dimension 0) / (block_size * block_size), but got output batch 8, input batch 16, and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<16x8x8x3xf32>, tensor<*xi32>) -> tensor<8x8x8x3xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputHeight(%arg0: tensor<16x8x8x3xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires output height (dimension 1) to be less than or equal to input height (dimension 1) * block_size, but got output height 17, input height 8, and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<16x8x8x3xf32>, tensor<*xi32>) -> tensor<4x17x8x3xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputHeightCrops(%arg0: tensor<16x8x8x3xf32>) { + %crops = "tf.Const"() {value = dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + // expected-error @+1 {{'tf.BatchToSpace' op requires output height (dimension 1) to be equal to input height (dimension 1) * block_size - crop_top - crop_bottom, but got output height 8, input height 8, crop_top 1, crop_bottom 2, and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %crops) {block_size = 2 : i64} : (tensor<16x8x8x3xf32>, tensor<2x2xi32>) -> tensor<4x8x9x3xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputWidth(%arg0: tensor<16x4x4x3xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires output width (dimension 2) to be less than or equal to input width (dimension 2) * block_size, but got output width 9, input width 4, and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<16x4x4x3xf32>, tensor<*xi32>) -> tensor<4x4x9x3xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputWidthCrops(%arg0: tensor<16x8x8x3xf32>) { + %crops = "tf.Const"() {value = dense<[[1, 2], [3, 4]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32> + // expected-error @+1 {{'tf.BatchToSpace' op requires output width (dimension 2) to be equal to input width (dimension 2) * block_size - crop_left - crop_right, but got output width 8, input width 8, crop_left 3, crop_right 4, and block_size 2}} + %0 = "tf.BatchToSpace"(%arg0, %crops) {block_size = 2 : i64} : (tensor<16x8x8x3xf32>, tensor<2x2xi32>) -> tensor<4x13x8x3xf32> + func.return +} + +// ----- + +func.func @testBatchToSpaceInvalidOutputDepth(%arg0: tensor<16x8x8x3xf32>, %arg1: tensor<*xi32>) { + // expected-error @+1 {{'tf.BatchToSpace' op requires output depth (dimension 3) to be equal to input depth (dimension 3), but got output depth 8 and input depth 3}} + %0 = "tf.BatchToSpace"(%arg0, %arg1) {block_size = 2 : i64} : (tensor<16x8x8x3xf32>, tensor<*xi32>) -> tensor<4x8x8x8xf32> + func.return +} + +// ----- + +func.func private @branch() + +func.func @testCaseBadBranchIndicesShape(%arg0: tensor<8xi32>) { + // expected-error @+1 {{expects 'branch_index' to be a scalar, but got 'tensor<8xi32>'}} + "tf.Case"(%arg0) {branches = [@branch], is_stateless = false} : (tensor<8xi32>) -> () + func.return +} + +// ----- + +func.func private @branch0(tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> +func.func private @branch1(tensor<2xf32>) -> tensor<2xf32> + +func.func @testCaseMismatchedNumOperands(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.Case' op branch #0 inputs (size = 2) should have the same number of values as inputs (size = 1)}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func private @branch0(tensor<2xf32>) -> (tensor<2xf32>, tensor<2xf32>) +func.func private @branch1(tensor<2xf32>) -> tensor<2xf32> + +func.func @testCaseMismatchedNumResults(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.Case' op branch #0 results (size = 2) should have the same number of values as results (size = 1)}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func private @branch0(tensor<*xf16>) -> tensor<*xf32> +func.func private @branch1(tensor<*xf32>) -> tensor<*xf32> + +func.func @testCaseOperandNotCastCompatible(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.Case' op branch #0 input type tensor<*xf16> is incompatible with input type tensor<2xf32> at index 0}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func private @branch0(tensor<2xf32>) -> tensor<*xf32> +func.func private @branch1(tensor<3xf32>) -> tensor<*xf32> + +func.func @testCaseBranchArgumentsNotCastCompatible(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<2xf32> { + // expected-error @+1 {{expects all branch input type(s) (tensor<2xf32>, tensor<3xf32>) at index 0 to be cast compatible}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<*xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func private @branch0(tensor<*xf32>) -> tensor<*xf32> +func.func private @branch1(tensor<*xf32>) -> tensor<3xf32> + +func.func @testCaseResultNotCastCompatible(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<2xf32> { + // expected-error @+1 {{'tf.Case' op branch #1 result type tensor<3xf32> is incompatible with result type tensor<2xf32> at index 0}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<*xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +func.func @testCaseRegionNoRegions(%arg0: tensor) { + // expected-error @+1 {{expects to have at least 1 region}} + "tf.CaseRegion"(%arg0) {is_stateless = false} : (tensor) -> () + func.return +} + +// ----- + +func.func @testCaseRegionBadBranchIndicesShape(%arg0: tensor<8xi32>) { + // expected-error @+1 {{expects 'branch_index' to be a scalar, but got 'tensor<8xi32>'}} + "tf.CaseRegion"(%arg0) ({ + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor<8xi32>) -> () + func.return +} + +// ----- + +func.func @testCaseRegionMismatchedNumResults(%arg0: tensor) { + // expected-error @+1 {{'tf.CaseRegion' op branch #0 results (size = 0) should have the same number of values as results (size = 1)}} + %1 = "tf.CaseRegion"(%arg0) ({ + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> tensor + func.return +} + +// ----- + +func.func @testCaseRegionMismatchedResultTypes(%arg0: tensor, %arg1: tensor) { + // expected-error @+1 {{'tf.CaseRegion' op branch #0 result type tensor is incompatible with result type tensor at index 0}} + %1 = "tf.CaseRegion"(%arg0) ({ + "tf.Yield"(%arg1) : (tensor) -> () + }) {is_stateless = false} : (tensor) -> tensor + func.return +} + +// ----- + +// Test valid tf.Cumsum +func.func @testCumsum(%arg: tensor<8x16xf32>, %axis: tensor) -> tensor<8x16xf32> { + %0 = "tf.Cumsum"(%arg, %axis) : (tensor<8x16xf32>, tensor) -> tensor<8x16xf32> + func.return %0 : tensor<8x16xf32> +} + +// ----- + +func.func @testCumprod(%arg: tensor<8x16xf32>, %axis: tensor<2xi32>) -> tensor<8x16xf32> { + // expected-error @+1 {{requires scalar axis operand}} + %0 = "tf.Cumprod"(%arg, %axis) : (tensor<8x16xf32>, tensor<2xi32>) -> tensor<8x16xf32> + func.return %0 : tensor<8x16xf32> +} + +// ----- + +func.func @testCumprod(%arg: tensor<8x16xf32>) -> tensor<8x16xf32> { + %axis = arith.constant dense<-3> : tensor + // expected-error @+1 {{axis operand should be within range [-2, 2)}} + %0 = "tf.Cumprod"(%arg, %axis) : (tensor<8x16xf32>, tensor) -> tensor<8x16xf32> + func.return %0 : tensor<8x16xf32> +} + +// ----- + +func.func @testTile(%arg0: tensor<2x3x?xf32>) { + %cst = arith.constant dense <[2, 3, 4]> : tensor<3xi32> + %0 = "tf.Tile"(%arg0, %cst) : (tensor<2x3x?xf32>, tensor<3xi32>) -> tensor<4x9x?xf32> + func.return +} + +// ----- + +func.func @testTileFold(%arg0: tensor<2x3x1xf32>, %arg1: tensor<2x3x20xf32>) -> tensor<2x3x20xf32> { + %cst = arith.constant dense <[1, 1, 20]> : tensor<3xi32> + %0 = "tf.Tile"(%arg0, %cst) : (tensor<2x3x1xf32>, tensor<3xi32>) -> tensor<2x3x20xf32> + %1 = "tf.AddV2"(%0, %arg1) {device = ""} : (tensor<2x3x20xf32>, tensor<2x3x20xf32>) -> tensor<2x3x20xf32> + func.return %1 : tensor<2x3x20xf32> +} + +// ----- + +func.func @testTileMultipleNotRank1(%arg0: tensor<2x3xf32>, %arg1: tensor<1x1xi32>) { + // expected-error @+1 {{expected multiples to be rank 1, got rank = 2}} + %0 = "tf.Tile"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<1x1xi32>) -> tensor<2x3xf32> + func.return +} + +// ----- + +func.func @testTileInputRankNotEqualToMultiplesSize(%arg0: tensor<2x3xf32>, %arg1: tensor<3xi32>) { + // expected-error @+1 {{expected size of multiples equal to rank of input, got multiples of size 3, and input of rank 2}} + %0 = "tf.Tile"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<3xi32>) -> tensor<2x3xf32> + func.return +} + +// ----- + +func.func @testTileInputRankNotEqualToOutputRank(%arg0: tensor<2x3xf32>, %arg1: tensor<2xi32>) { + // expected-error @+1 {{expected rank of input to equal to rank of output, got input of rank 2, and output of rank 3}} + %0 = "tf.Tile"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<2x3x1xf32> + func.return +} + +// ----- + +func.func @testTileNegativeMultiples(%arg0: tensor<2x3xf32>) { + %cst = arith.constant dense <[-1, 1]> : tensor<2xi32> + // expected-error @+1 {{expected multiples to be non-negative, got multiples[0] = -1}} + %0 = "tf.Tile"(%arg0, %cst) : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<2x3xf32> + func.return +} + +// ----- + +func.func @testTileInvalidOutputShape(%arg0: tensor<2x3xf32>) { + %cst = arith.constant dense <[2, 3]> : tensor<2xi32> + // expected-error @+1 {{requires input.shape[1] (3) * 3 to be equal to output.shape[1] (6)}} + %0 = "tf.Tile"(%arg0, %cst) : (tensor<2x3xf32>, tensor<2xi32>) -> tensor<4x6xf32> + func.return +} + +// ----- + +// Test reference variable support for some ops (no errors expected) + +// CHECK-LABEL: @testMaximumWithRef +func.func @testMaximumWithRef(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: tf.Maximum + %0 = "tf.Maximum"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: @testAddV2WithRef +func.func @testAddV2WithRef(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: tf.AddV2 + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: @testRealDivWithRef +func.func @testRealDivWithRef(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: tf.RealDivOp + %0 = "tf.RealDivOp"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: @testDivNoNanWithRef +func.func @testDivNoNanWithRef(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: tf.DivNoNanOp + %0 = "tf.DivNoNanOp"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: @testAddWithRef +func.func @testAddWithRef(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: tf.Add + %0 = "tf.Add"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testInvalidTPUExecuteAndUpdateVariables(%arg0: tensor>>, %arg1: tensor<3x!tf_type.string>) { + // expected-error@below {{requires 'device_var_reads_indices' to be the same size as number of resource handles in 'args' (1), but got 2}} + "tf.TPUExecuteAndUpdateVariables"(%arg0, %arg1) {device_var_reads_indices = [0, 1], device_var_updates_indices = [0]} : (tensor>>, tensor<3x!tf_type.string>) -> () + func.return +} + +// ----- + +func.func @testInvalidTPUExecuteAndUpdateVariables(%arg0: tensor>>, %arg1: tensor<3x!tf_type.string>) { + // expected-error@below {{requires 'device_var_updates_indices' to be the same size as number of resource handles in 'args' (1), but got 2}} + "tf.TPUExecuteAndUpdateVariables"(%arg0, %arg1) {device_var_reads_indices = [0], device_var_updates_indices = [0, 1]} : (tensor>>, tensor<3x!tf_type.string>) -> () + func.return +} + +// ----- + +func.func @testInvalidTPUExecuteAndUpdateVariables(%arg0: tensor>>, %arg1: tensor<3x!tf_type.string>) { + // expected-error@below {{requires 'device_var_reads_indices' to contain values of at least 0, but got -1 at index 0}} + "tf.TPUExecuteAndUpdateVariables"(%arg0, %arg1) {device_var_reads_indices = [-1], device_var_updates_indices = [0]} : (tensor>>, tensor<3x!tf_type.string>) -> () + func.return +} + +// ----- + +func.func @testInvalidTPUExecuteAndUpdateVariables(%arg0: tensor>>, %arg1: tensor<3x!tf_type.string>) { + // expected-error@below {{requires 'device_var_updates_indices' to contain values of at least -1, but got -2 at index 0}} + "tf.TPUExecuteAndUpdateVariables"(%arg0, %arg1) {device_var_reads_indices = [0], device_var_updates_indices = [-2]} : (tensor>>, tensor<3x!tf_type.string>) -> () + func.return +} + +// ----- + +// Valid VarHandleOp operation. +// CHECK-LABEL: func @testVarHandleOp +func.func @testVarHandleOp() -> tensor>> { + %0 = "tf.VarHandleOp"() { + container = "", + shared_name = "cd2c89b7-88b7-44c8-ad83-06c2a9158347" + } : () -> tensor>> + func.return %0 : tensor>> +} + +// ----- + +// VarHandleOp operation missing the required resource subtype. +func.func @testVarHandleOp() -> tensor<*x!tf_type.resource> { + // expected-error @+1 {{must have exactly one subtype in the result resource type}} + %0 = "tf.VarHandleOp"() { + container = "", + shared_name = "cd2c89b7-88b7-44c8-ad83-06c2a9158347" + } : () -> tensor<*x!tf_type.resource> + func.return %0 : tensor<*x!tf_type.resource> +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<2x3x5xi32>, %arg1: tensor<5x2xi32>) -> () { + %0 = "tf.Const"() {value = dense<2> : tensor<1xi64>} : () -> tensor<1xi64> + // expected-error @+2 {{'tf.XlaBroadcastHelper' op failed to infer returned types}} + // expected-error @+1 {{broadcast_dims must have size equal to the smaller argument rank}} + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<2x3x5xi32>, tensor<5x2xi32>, tensor<1xi64>) -> (tensor<2x3x5xi32>, tensor<2x1x5xi32>) + func.return +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<2x3x5xi32>, %arg1: tensor<5x2xi32>) -> () { + %0 = "tf.Const"() {value = dense<> : tensor<0xi64>} : () -> tensor<0xi64> + // expected-error @+2 {{'tf.XlaBroadcastHelper' op failed to infer returned types}} + // expected-error @+1 {{if broadcast_dims is empty, both arguments must have equal rank or at least one argument must be a scalar}} + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<2x3x5xi32>, tensor<5x2xi32>, tensor<0xi64>) -> (tensor<2x3x5xi32>, tensor<2x1x5xi32>) + func.return +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<5x2xi32>, %arg1: tensor<2x3x5xi32>) -> () { + %0 = "tf.Const"() {value = dense<0> : tensor<2xi64>} : () -> tensor<2xi64> + // expected-error @+2 {{'tf.XlaBroadcastHelper' op failed to infer returned types}} + // expected-error @+1 {{broadcast_dims has duplicates}} + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<5x2xi32>, tensor<2x3x5xi32>, tensor<2xi64>) -> (tensor<2x1x5xi32>, tensor<2x3x5xi32>) + func.return +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<2xi32>, %arg1: tensor) -> () { + %0 = "tf.Const"() {value = dense<> : tensor<0xi64>} : () -> tensor<0xi64> + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<2xi32>, tensor, tensor<0xi64>) -> (tensor<2xi32>, tensor) + func.return +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<5x2xi32>, %arg1: tensor<2x3x5xi32>) -> () { + %0 = "tf.Const"() {value = dense<[2, 0]> : tensor<2xi64>} : () -> tensor<2xi64> + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<5x2xi32>, tensor<2x3x5xi32>, tensor<2xi64>) -> (tensor<2x1x5xi32>, tensor<2x3x5xi32>) + func.return +} + +// ----- + +func.func @testXlaBroadcastHelper(%arg0: tensor<2x3x5xi32>, %arg1: tensor<5x2xi32>) -> () { + %0 = "tf.Const"() {value = dense<[2, 0]> : tensor<2xi64>} : () -> tensor<2xi64> + %lhs_output, %rhs_output = "tf.XlaBroadcastHelper"(%arg0, %arg1, %0) : (tensor<2x3x5xi32>, tensor<5x2xi32>, tensor<2xi64>) -> (tensor<2x3x5xi32>, tensor<2x1x5xi32>) + func.return +} + +// ----- + +func.func @testXlaConvV2InvalidFeatureGroupCount(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32>) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor<2xi32>} : () -> tensor<2xi32> + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // / expected-error@+1 {{'tf.XlaConvV2' op expects feature_group_count to be a scalar}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor<2xi32>) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> +} + +// ----- + +func.func @testXlaConvV2InvalidLhsDilation(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32>) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %lhs_dilation = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error@+1 {{'tf.XlaConvV2' op expects lhs_dilation to be a vecotr}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> +} + +// ----- + +func.func @testXlaConvV2InvalidRhsDilation(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32>) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %lhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error@+1 {{'tf.XlaConvV2' op expects rhs_dilation to be a vecotr}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x2xi32>, tensor<3xi32>, tensor, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> +} + +// ----- + +func.func @testXlaConvV2InvalidWindowStrides(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32>) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %lhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x2xi32>} : () -> tensor<3x2xi32> + %strides = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // expected-error@+1 {{'tf.XlaConvV2' op expects window_stride to be a vector}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor, tensor<3x2xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> +} + +// ----- + +func.func @testXlaConvV2InvalidPadding(%lhs: tensor<8x4x16x16x16xf32>, %rhs: tensor<4x3x3x16x16xf32>) -> (tensor<8x4x14x14x16xf32>) { + %feature_group_count = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %lhs_dilation = "tf.Const"() {value = dense<[4, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + %rhs_dilation = "tf.Const"() {value = dense<1> : tensor<3xi32>} : () -> tensor<3xi32> + %padding = "tf.Const"() {value = dense<0> : tensor<3x3xi32>} : () -> tensor<3x3xi32> + %strides = "tf.Const"() {value = dense<[3, 1, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error@+1 {{'tf.XlaConvV2' op expects padding to be a matrix with minor dimension 2}} + %0 = "tf.XlaConvV2"(%lhs, %rhs, %strides, %padding, %lhs_dilation, %rhs_dilation, %feature_group_count) {dimension_numbers = "\18\03 \042\03\00\01\02@\04P\04Z\03\01\02\03b\03\01\02\03", precision_config = ""} : (tensor<8x4x16x16x16xf32>, tensor<4x3x3x16x16xf32>, tensor<3xi32>, tensor<3x3xi32>, tensor<3xi32>, tensor<3xi32>, tensor) -> tensor<8x4x14x14x16xf32> + func.return %0 : tensor<8x4x14x14x16xf32> +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + "tf._XlaHostComputeMlir"(%arg0) {send_key="", recv_key="", host_mlir_module=""} : (tensor<2xf32>) -> () + func.return +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + "tf._XlaHostComputeMlir"(%arg0) {send_key="", recv_key="", host_mlir_module="module {\0A func.func @host_func(%arg0: tensor<*xf32>) -> tensor<*xf32> {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A func.return %0 : tensor<*xf32> \0A } \0A} \0A"} : (tensor<2xf32>) -> (tensor<2xf32>) + func.return +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + // expected-error @+1 {{can not be deserialized}} + "tf._XlaHostComputeMlir"(%arg0) {send_key="", recv_key="", host_mlir_module="bad_module"} : (tensor<2xf32>) -> () + func.return +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + // expected-error @+1 {{'host_mlir_module' does not contain 'host_func' function}} + "tf._XlaHostComputeMlir"(%arg0) {send_key="", recv_key="", host_mlir_module="module {\0A func.func @bad_func(%arg0: tensor<*xf32>) -> tensor<*xf32> {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A func.return %0 : tensor<*xf32> \0A } \0A} \0A"} : (tensor<2xf32>) -> () + func.return +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + // expected-error @+1 {{Number of operands/inputs should be the same}} + "tf._XlaHostComputeMlir"() {send_key="", recv_key="", host_mlir_module="module {\0A func.func @host_func(%arg0: tensor<*xf32>) -> tensor<*xf32> {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A func.return %0 : tensor<*xf32> \0A } \0A} \0A"} : () -> () + func.return +} + +// ----- + +func.func @testXlaHostComputeMlir(%arg0: tensor<2xf32>) -> () { + // expected-error @+1 {{Number of results should be the same}} + "tf._XlaHostComputeMlir"(%arg0) {send_key="", recv_key="", host_mlir_module="module {\0A func.func @host_func(%arg0: tensor<*xf32>) -> tensor<*xf32> {\0A %0 = \22tf.Identity\22(%arg0) {_xla_outside_compilation = \22cluster1\22} : (tensor<*xf32>) -> tensor<*xf32> \0A func.return %0 : tensor<*xf32> \0A } \0A} \0A"} : (tensor<2xf32>) -> () + func.return +} + +// ----- +func.func @testXlaSelectAndScatterAttrs(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1]> : tensor<3xi32>} : () -> tensor<3xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op expects the size of window_dimensionsto be equal to the input rank (3 vs. 4)}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @no_select} : (tensor<4x5x1x1xbf16>, tensor<3xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaSelectAndScatterPadding(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2x3xi32>} : () -> tensor<4x2x3xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op expects padding to be a matrix with minor dimension 2, got 4, 2, 3}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @no_select} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2x3xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaSelectAndScatterSelect(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op has no select function specified}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @no_select} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaSelectAndScatterSelectNumArgs(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op expects select function to take 2 parameters, but has 3 parameter(s)}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @xla_select_and_scatter_select_3_args} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +func.func private @xla_select_and_scatter_select_3_args(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + %0 = "tf.GreaterEqual"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaSelectAndScatterSelectReturnType(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op expects select function to return a single boolean result but got 'tensor<4xi32>'}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @xla_select_and_scatter_select_return_int32_vector} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +func.func private @xla_select_and_scatter_select_return_int32_vector(%arg0: tensor, %arg1: tensor) -> tensor<4xi32> { + %0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + func.return %0 : tensor<4xi32> +} + +// ----- + +func.func @testXlaSelectAndScatterScatter(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op has no scatter function specified}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @no_scatter, select = @xla_select_and_scatter_select1} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +func.func private @xla_select_and_scatter_select1(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.GreaterEqual"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaSelectAndScatterSelectNumArgs(%arg0: tensor<4x5x1x1xbf16>, %arg1: tensor<2x2x1x1xbf16>, %arg2: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %cst_0 = "tf.Const"() {value = dense<[2, 2, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_1 = "tf.Const"() {value = dense<[2, 3, 1, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + // expected-error @+1 {{'tf.XlaSelectAndScatter' op expects scatter function to take 2 parameters, but has 3 parameter(s)}} + %0 = "tf.XlaSelectAndScatter"(%arg0, %cst_1, %cst_0, %cst, %arg1, %arg2) {scatter = @xla_select_and_scatter_scatter, select = @xla_select_and_scatter_select2} : (tensor<4x5x1x1xbf16>, tensor<4xi32>, tensor<4xi32>, tensor<4x2xi32>, tensor<2x2x1x1xbf16>, tensor) -> tensor + func.return %0 : tensor +} + +func.func private @xla_select_and_scatter_select2(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.GreaterEqual"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func private @xla_select_and_scatter_scatter(%arg0: tensor<*xbf16>, %arg1: tensor<*xbf16>, %arg2: tensor<*xbf16>) -> tensor<*xbf16> { + %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor<*xbf16>, tensor<*xbf16>) -> tensor<*xbf16> + func.return %0 : tensor<*xbf16> +} + +// ----- + +func.func @testXlaReduceWindowAttrs(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-error @+1 {{tf.XlaReduceWindow' op expects the size of base_dilations to be equal to the input rank (2 vs. 1)}} + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @no_reducer} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<2xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaReduceWindowPadding(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<1x3xi32>} : () -> tensor<1x3xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-error @+1 {{'tf.XlaReduceWindow' op expects padding to be a matrix with minor dimension 2, got 1, 3}} + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @no_reducer} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x3xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaReduceWindowComputation(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-error @+1 {{'tf.XlaReduceWindow' op has no reduction function specified}} + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @no_reducer} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testXlaReduceWindowComputationNumArgs(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<0> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-error @+1 {{'tf.XlaReduceWindow' op expects reduction function to take 2 parameters, but has 3 parameter(s)}} + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @xla_reduce_window_op_reducer} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor + func.return %0 : tensor +} + +func.func private @xla_reduce_window_op_reducer(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testLogStaticShapeInputAndDynamicShapeOutput(%arg0: tensor<8x16xf32>) -> tensor<*xf32> { + %0 = "tf.Log"(%arg0) : (tensor<8x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testReluStaticShapeInputAndDynamicShapeOutput(%arg0: tensor<8x16xf32>) -> tensor<*xf32> { + %0 = "tf.Relu"(%arg0) : (tensor<8x16xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} +// ----- + +func.func @set_dynamic_dimension_size(%input: tensor<4xf32>, %size: tensor) -> tensor { + %dimension = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + // expected-error @+2 {{'tf.XlaSetDynamicDimensionSize' op failed to infer returned types}} + // expected-error @+1 {{dim_index (1) is out of range [0, 1)}} + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4xf32>, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @testSetStaticDimensionBounds(%arg0: f32, %arg1: tensor) -> tensor { + // expected-error @below {{'tf.SetStaticDimensionBounds' op operand #0 must be tensor of tf.dtype values, but got 'f32'}} + %dyn_arg0 = "tf.SetStaticDimensionBounds" (%arg0, %arg1) :(f32, tensor) -> tensor + func.return %dyn_arg0 : tensor +} + +// ----- + +func.func @testSetStaticDimensionBounds(%arg0: tensor, %arg1: tensor) -> tensor { + // expected-error @below {{'tf.SetStaticDimensionBounds' op was used with an input tensor with rank > 2, only tensors of rank 1,2 are supported}} + %dyn_arg0 = "tf.SetStaticDimensionBounds" (%arg0, %arg1) :(tensor, tensor) -> tensor + func.return %dyn_arg0 : tensor +} + +// ----- + +func.func @testSetStaticDimensionBounds(%arg0: tensor, %arg1: tensor) -> tensor { + // expected-error @below {{'tf.SetStaticDimensionBounds' op static shape must be of rank 1 (vector)}} + %dyn_arg0 = "tf.SetStaticDimensionBounds" (%arg0, %arg1) :(tensor, tensor) -> tensor + func.return %dyn_arg0 : tensor +} + +// ----- + +func.func @testSetStaticDimensionBounds(%arg0: tensor, %arg1: tensor<4xi32>) -> tensor { + // expected-error @below {{'tf.SetStaticDimensionBounds' op static shape must have num_elements == rank of input tensor}} + %dyn_arg0 = "tf.SetStaticDimensionBounds" (%arg0, %arg1) :(tensor, tensor<4xi32>) -> tensor + func.return %dyn_arg0 : tensor +} + +// ----- + +func.func @testUniformQuantizedDotHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedDotHybrid' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformQuantizedDotHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + rhs_quantization_axis = -1 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedDotHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor, %rhs_zps: tensor<2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedDotHybrid' op quantization_axis is -1, zero_points must have 0 rank.}} + %0 = "tf.UniformQuantizedDotHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + rhs_quantization_axis = -1 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor, tensor<2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedDotHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2x2xf32>, %rhs_zps: tensor<2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedDotHybrid' op quantization_axis is not -1, scales must have 1 rank.}} + %0 = "tf.UniformQuantizedDotHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2x2xf32>, tensor<2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedDotHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor<2x2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedDotHybrid' op quantization_axis is not -1, zero_points must have 1 rank.}} + %0 = "tf.UniformQuantizedDotHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<2x2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedDotHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor<3xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedDotHybrid' op scales and zero points must have same number of elements.}} + %0 = "tf.UniformQuantizedDotHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<3xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedConvolutionHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedConvolutionHybrid' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformQuantizedConvolutionHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = -1 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedConvolutionHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor, %rhs_zps: tensor<2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedConvolutionHybrid' op quantization_axis is -1, zero_points must have 0 rank.}} + %0 = "tf.UniformQuantizedConvolutionHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = -1 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor, tensor<2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedConvolutionHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2x2xf32>, %rhs_zps: tensor<2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedConvolutionHybrid' op quantization_axis is not -1, scales must have 1 rank.}} + %0 = "tf.UniformQuantizedConvolutionHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2x2xf32>, tensor<2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedConvolutionHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor<2x2xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedConvolutionHybrid' op quantization_axis is not -1, zero_points must have 1 rank.}} + %0 = "tf.UniformQuantizedConvolutionHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<2x2xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedConvolutionHybrid(%lhs: tensor<*xf32>, %rhs: tensor<2x2x!tf_type.qint8>, %rhs_scales: tensor<2xf32>, %rhs_zps: tensor<3xi32>) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformQuantizedConvolutionHybrid' op scales and zero points must have same number of elements.}} + %0 = "tf.UniformQuantizedConvolutionHybrid"(%lhs, %rhs, %rhs_scales, %rhs_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = 0 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<3xi32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantize(%arg0: tensor<*xf32>, %scales: tensor<2xf32>, %zps: tensor) -> tensor<*x!tf_type.qint8> { + // expected-error @below {{'tf.UniformQuantize' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<*xf32>, tensor<2xf32>, tensor) -> tensor<*x!tf_type.qint8> + func.return %0 : tensor<*x!tf_type.qint8> +} + +// ----- + +func.func @testUniformRequantize( + %arg0: tensor<*x!tf_type.qint8>, + %scales_0: tensor<2xf32>, %zps_0: tensor, + %scales_1: tensor, %zps_1: tensor) -> tensor<*x!tf_type.qint8> { + // expected-error @below {{'tf.UniformRequantize' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformRequantize"(%arg0, %scales_0, %zps_0, %scales_1, %zps_1) { + input_quantization_axis = -1 : i64, input_quantization_min_val = -2147483648 : i64, input_quantization_max_val = 2147483647 : i64, + output_quantization_axis = -1 : i64, output_quantization_min_val = -128 : i64, output_quantization_max_val = 127 : i64 + } : (tensor<*x!tf_type.qint8>, tensor<2xf32>, tensor, tensor, tensor) -> tensor<*x!tf_type.qint8> + func.return %0 : tensor<*x!tf_type.qint8> +} + +// ----- + +func.func @testUniformRequantize( + %arg0: tensor<*x!tf_type.qint8>, + %scales_0: tensor, %zps_0: tensor, + %scales_1: tensor<2xf32>, %zps_1: tensor) -> tensor<*x!tf_type.qint8> { + // expected-error @below {{'tf.UniformRequantize' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformRequantize"(%arg0, %scales_0, %zps_0, %scales_1, %zps_1) { + input_quantization_axis = -1 : i64, input_quantization_min_val = -2147483648 : i64, input_quantization_max_val = 2147483647 : i64, + output_quantization_axis = -1 : i64, output_quantization_min_val = -128 : i64, output_quantization_max_val = 127 : i64 + } : (tensor<*x!tf_type.qint8>, tensor, tensor, tensor<2xf32>, tensor) -> tensor<*x!tf_type.qint8> + func.return %0 : tensor<*x!tf_type.qint8> +} + +// ----- + +func.func @testUniformDequantize(%arg0: tensor<*x!tf_type.qint8>, %scales: tensor<2xf32>, %zps: tensor) -> tensor<*xf32> { + // expected-error @below {{'tf.UniformDequantize' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformDequantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<*x!tf_type.qint8>, tensor<2xf32>, tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @testUniformQuantizedDot( + %input: tensor<*x!tf_type.qint8>, %weight: tensor<2x2x!tf_type.qint8>, + %input_scales: tensor<2xf32>, %input_zps: tensor, + %weight_scales: tensor, %weight_zps: tensor, + %output_scales: tensor, %output_zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedDot' op quantization_axis is -1, scales must have 0 rank.}} + %1 = "tf.UniformQuantizedDot"( + %input, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<*x!tf_type.qint8>, tensor<2x2x!tf_type.qint8>, + tensor<2xf32>, tensor, + tensor, tensor, + tensor, tensor) -> tensor<*x!tf_type.qint32> + func.return +} + +// ----- + +func.func @testUniformQuantizedDot( + %input: tensor<*x!tf_type.qint8>, %weight: tensor<2x2x!tf_type.qint8>, + %input_scales: tensor, %input_zps: tensor, + %weight_scales: tensor<2xf32>, %weight_zps: tensor, + %output_scales: tensor, %output_zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedDot' op quantization_axis is -1, scales must have 0 rank.}} + %1 = "tf.UniformQuantizedDot"( + %input, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<*x!tf_type.qint8>, tensor<2x2x!tf_type.qint8>, + tensor, tensor, + tensor<2xf32>, tensor, + tensor, tensor) -> tensor<*x!tf_type.qint32> + func.return +} + +// ----- + +func.func @testUniformQuantizedDot( + %input: tensor<*x!tf_type.qint8>, %weight: tensor<2x2x!tf_type.qint8>, + %input_scales: tensor, %input_zps: tensor, + %weight_scales: tensor, %weight_zps: tensor, + %output_scales: tensor<2xf32>, %output_zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedDot' op quantization_axis is -1, scales must have 0 rank.}} + %1 = "tf.UniformQuantizedDot"( + %input, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<*x!tf_type.qint8>, tensor<2x2x!tf_type.qint8>, + tensor, tensor, + tensor, tensor, + tensor<2xf32>, tensor) -> tensor<*x!tf_type.qint32> + func.return +} + +// ----- + +func.func @testUniformQuantizedConvolution( + %input: tensor<*x!tf_type.qint8>, %weight: tensor<2x2x!tf_type.qint8>, + %input_scales: tensor, %input_zps: tensor, + %weight_scales: tensor<2xf32>, %weight_zps: tensor, + %output_scales: tensor, %output_zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedConvolution' op quantization_axis is -1, scales must have 0 rank.}} + %1 = "tf.UniformQuantizedConvolution"( + %input, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<*x!tf_type.qint8>, tensor<2x2x!tf_type.qint8>, + tensor, tensor, + tensor<2xf32>, tensor, + tensor, tensor) -> tensor<*x!tf_type.qint32> + func.return +} + +// ----- + +func.func @testUniformQuantizedAdd( + %input: tensor<2x2x!tf_type.qint32>, %bias: tensor<2x!tf_type.qint32>, + %input_scales: tensor, %input_zps: tensor, + %bias_scales: tensor, %bias_zps: tensor, + %output_scales: tensor<2xf32>, %output_zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedAdd' op quantization_axis is -1, scales must have 0 rank.}} + %1 = "tf.UniformQuantizedAdd"( + %input, %bias, + %input_scales, %input_zps, + %bias_scales, %bias_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -2147483648 : i64, + lhs_quantization_max_val = 2147483647 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -2147483648 : i64, + rhs_quantization_max_val = 2147483647 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<2x2x!tf_type.qint32>, tensor<2x!tf_type.qint32>, + tensor, tensor, + tensor, tensor, + tensor<2xf32>, tensor) -> tensor<2x2x!tf_type.qint32> + func.return +} + +// ----- + +func.func @testUniformQuantizedClipByValue( + %operand: tensor<*x!tf_type.qint32>, %min: tensor, %max: tensor, + %scales: tensor<2xf32>, %zps: tensor) -> () { + // expected-error @below {{'tf.UniformQuantizedClipByValue' op quantization_axis is -1, scales must have 0 rank.}} + %0 = "tf.UniformQuantizedClipByValue"(%operand, %min, %max, %scales, %zps) { + quantization_axis = -1 : i64, + quantization_min_val = -2147483648 : i64, + quantization_max_val = 2147483647 : i64 + } : ( + tensor<*x!tf_type.qint32>, tensor, tensor, + tensor<2xf32>, tensor + ) -> tensor<*x!tf_type.qint32> + func.return +} + +// ----- + +// CHECK-LABEL: func @testValidFusedConv2DBiasActivation +func.func @testValidFusedConv2DBiasActivation( + %conv_input: tensor<*xf32>, + %filter: tensor<*xf32>, + %bias: tensor<*xf32>, + %side_input: tensor<*xf32>, + %conv_input_scale: tensor<*xf32>, + %side_input_scale: tensor<*xf32>) -> tensor<*xf32> { + %0 ="tf.FusedConv2DBiasActivation"(%conv_input, %filter, %bias, %side_input, %conv_input_scale, %side_input_scale) { + T = f32, Tbias = f32, + activation_mode = "Relu", + data_format = "NHWC", + dilations = [1, 1, 1, 1], + filter_format = "HWIO", + padding = "SAME", + strides = [1, 2, 2, 1] + } : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>) + func.return %0 : tensor<*xf32> +} + + +// Following tests are for LegacyCall symbol use verifier. + +// ----- + +// Tests that valid symbol use does not produce any error. +func.func @valid_symbol_use(%arg0: tensor) -> () { + "tf.LegacyCall"(%arg0) {f = @call_func} : (tensor) -> (tensor) + func.return +} + +func.func @call_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// Tests that undefined call function produces error. +func.func @test_undefined_function() -> () { + // expected-error @below {{'f' attribute refers to an undefined function: undefined_func}} + "tf.LegacyCall"() {f = @undefined_func} : () -> () + func.return +} + +// ----- + +// Tests that argument count mismatch produces error. +func.func @test_arg_count_mismatch(%arg0: tensor) -> () { + // expected-error @below {{argument count mismatch: 'args' has 1 argument(s), but 'call_func' expects 2}} + "tf.LegacyCall"(%arg0) {f = @call_func} : (tensor) -> tensor + func.return +} + +func.func @call_func(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +func.func @test_batch_function_with_valid_symbol(%arg0: tensor<1x3xf32>, %arg1: tensor>>) -> () { + "tf.BatchFunction"(%arg0, %arg1) {batch_timeout_micros = 100000 : i64, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return +} + +func.func private @batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<1x3xf32> { + %0 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %0 : tensor<1x3xf32> +} + +// ----- + +func.func @test_batch_function_with_invalid_symbol(%arg0: tensor<1x3xf32>, %arg1: tensor>>) -> () { + // expected-error @below {{'f' attribute refers to an undefined function: undefined_function}} + "tf.BatchFunction"(%arg0, %arg1) {batch_timeout_micros = 100000 : i64, f = @undefined_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return +} + +// ----- + +func.func @test_xla_call_module_with_invalid_symbol() { + // expected-error @below {{refers to an undefined function: @undefined_function}} + "tf.XlaCallModule"() {Sout = [], device = "", dim_args_spec = [], function_list = [@undefined_function], module = "", platforms = [], version = 4 : i64} : () -> () + func.return +} + + +// ----- + +func.func @init(%arg0: tensor<4xf32>) -> tensor<7xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<7xf32> + return %0 : tensor<7xf32> +} + +func.func @next(%arg0: tensor<7xf32>, %arg1: tensor<3xf32>) -> tensor<6xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<6xf32> + return %0 : tensor<6xf32> +} + +func.func @finalize(%arg0: tensor<6xf32>, %arg1: tensor<2xf32>) -> tensor<5xf32> { + %0 = builtin.unrealized_conversion_cast to tensor<5xf32> + return %0 : tensor<5xf32> +} + +// CHECK-LABEL: func @testGeneratorDataset +func.func @testGeneratorDataset(%arg0: tensor<4xf32>, + %arg1: tensor<3xf32>, + %arg2: tensor, + %arg3: tensor<2xf32>) -> tensor { + %0 = "tf.GeneratorDataset"(%arg0, %arg1, %arg2, %arg3) { + device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0", + finalize_func = @finalize, + init_func = @init, + next_func = @next, + operandSegmentSizes = array, + output_shapes = [#tf_type.shape<>], + output_types = [!tf_type.string], + metadata = ""} : ( + tensor<4xf32>, + tensor<3xf32>, + tensor, + tensor<2xf32>) -> tensor + return %0 : tensor +} + +func.func @testDebugIdentity(%arg0: tensor) -> tensor { + %0 = "tf.DebugIdentity"(%arg0) {debug_urls = ["file:///tmp/foo"], device_name = "CPU", tensor_name = "test_tensor"} : (tensor) -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-reduce-identity.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-reduce-identity.mlir new file mode 100644 index 0000000000000000000000000000000000000000..82d404bff19a5423c8b77a28aeaba7cf8a0616b5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf-reduce-identity.mlir @@ -0,0 +1,17 @@ +// RUN: tf-reduce %s -reduction-tree='traversal-mode=0 test=%S/reducer/unsupported-op-test.sh' | FileCheck %s + +// CHECK: @target_function +func.func @target_function() -> tensor { + %0 = "tf_device.cluster"() ({ + // CHECK: tf.UnsupportedOp + %1 = "tf.UnsupportedOp"() {value = dense<1> : tensor} : () -> tensor + // CHECK: tf.Identity + %2 = "tf.Identity"(%1) : (tensor) -> tensor + // CHECK-NOT: tf.Identity + %3 = "tf.Identity"(%2) : (tensor) -> tensor + %4 = "tf.Identity"(%3) : (tensor) -> tensor + // CHECK: tf_device.return + tf_device.return %4 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_map_and_batch.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_map_and_batch.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a885ca71b8d1e689c1a5eda32643e3cfb4ba053f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_map_and_batch.mlir @@ -0,0 +1,34 @@ +// RUN: tf-opt -tf-standard-pipeline -tf-data-optimization %s -o %t && FileCheck %s < %t + +module { +// CHECK-LABEL: fuse_map_and_batch +func.func @fuse_map_and_batch() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "BatchDatasetV2"}} { + %0 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<[0, 1, 2]> : tensor<3xi32>} : () -> tensor<3xi32> + // CHECK: %[[NPC:.*]] = "tf.Const"() <{value = dense<1> : tensor}> + // CHECK: %[[TSLICE:.*]] = "tf.TensorSliceDataset" + %3 = "tf.TensorSliceDataset"(%2) {device = "", output_shapes = [#tf_type.shape<>], metadata = ""} : (tensor<3xi32>) -> tensor<*x!tf_type.variant> + // CHECK: "tf.MapAndBatchDataset"(%[[TSLICE]], %[[BSIZE:.*]], %[[NPC]] + // CHECK-SAME: f = @"__inference_Dataset_map__80", + %4 = "tf.MapDataset"(%3) {device = "", + f = @"__inference_Dataset_map__80", + output_shapes = [#tf_type.shape<>], output_types = [i32], + preserve_cardinality = false, sloppy = false, + use_inter_op_parallelism = true, + metadata = ""} : (tensor<*x!tf_type.variant>) -> tensor + %5 = "tf.BatchDatasetV2"(%4, %0, %1) {device = "", + output_shapes = [#tf_type.shape<>], + output_types = [i32], + parallel_copy = false, + metadata = ""} : (tensor, tensor, tensor) -> tensor + func.return %5 : tensor +} + +func.func @"__inference_Dataset_map__80"(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %1 = "tf.Mul"(%arg0, %0) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %2 = "tf.Identity"(%1) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + func.return %2 : tensor<*xi32> +} +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_pmap_and_batch.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_pmap_and_batch.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3b7d68c07af5a95eb10fc6937c1037e47cc8621f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_data_fuse_pmap_and_batch.mlir @@ -0,0 +1,34 @@ +// RUN: tf-opt -tf-standard-pipeline -tf-data-optimization %s -o %t && FileCheck %s < %t + +module { +// CHECK-LABEL: fuse_pmap_and_batch +func.func @fuse_pmap_and_batch() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "BatchDatasetV2"}} { + %0 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<[0, 1, 2]> : tensor<3xi32>} : () -> tensor<3xi32> + %3 = "tf.Const"() {value = dense<12> : tensor} : () -> tensor + // CHECK: %[[TSLICE:.*]] = "tf.TensorSliceDataset" + %4 = "tf.TensorSliceDataset"(%2) {device = "", output_shapes = [#tf_type.shape<>], metadata = ""} : (tensor<3xi32>) -> tensor<*x!tf_type.variant> + // CHECK: "tf.MapAndBatchDataset"(%[[TSLICE]], + // CHECK-SAME: f = @"__inference_Dataset_map__80", + %5 = "tf.ParallelMapDataset"(%4, %3) {device = "", + f = @"__inference_Dataset_map__80", + output_shapes = [#tf_type.shape<>], output_types = [i32], + preserve_cardinality = false, sloppy = false, + use_inter_op_parallelism = true, + metadata = ""} : (tensor<*x!tf_type.variant>, tensor) -> tensor + %6 = "tf.BatchDatasetV2"(%5, %0, %1) {device = "", + output_shapes = [#tf_type.shape<>], + output_types = [i32], + parallel_copy = false, + metadata = ""} : (tensor, tensor, tensor) -> tensor + func.return %6 : tensor +} + +func.func @"__inference_Dataset_map__80"(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %1 = "tf.Mul"(%arg0, %0) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %2 = "tf.Identity"(%1) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + func.return %2 : tensor<*xi32> +} +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_index_selector.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_index_selector.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a5840e88bc9f783af6ac015b6317e2bca59479d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_index_selector.mlir @@ -0,0 +1,25 @@ +// Test DeviceIndex selector. + +// RUN: tf-opt --tf-device-index-selector %s | FileCheck %s + +// CHECK-LABEL: func @select +func.func @select(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: %[[first:.*]] = "tf.DeviceIndex" + // CHECK: constant dense<2> + // CHECK: return %[[first]], + %0 = "tf.DeviceIndex"() {device = "", device_names = ["CPU", "GPU"]} : () -> tensor + %1 = "tf.DeviceIndex"() {device = "", device_names = ["CPU", "GPU"]} : () -> tensor + %4 = "tf.Case"(%1, %arg0, %arg1) {branches = [@sub, @add], output_shapes = [#tf_type.shape<>], is_stateless = false} : (tensor, tensor, tensor) -> tensor + + func.return %0, %4 : tensor, tensor +} + +func.func @add(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.Add"(%arg0, %arg1): (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +func.func @sub(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.Sub"(%arg0, %arg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..cc2e79b890ac133ea9947034e055655f6dd9c836 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops.mlir @@ -0,0 +1,236 @@ +// RUN: tf-opt --verify-diagnostics --split-input-file %s | FileCheck %s + +// CHECK-LABEL: func @return_no_operands +func.func @return_no_operands() { + "tf_device.launch"() ({ +// CHECK: tf_device.return + tf_device.return + }) {device = "device"} : () -> () + func.return +} + +// ----- + +// CHECK-LABEL: func @return_one_operand +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>) +func.func @return_one_operand(%arg_0: tensor<*xf32>) { + %result = "tf_device.launch"() ({ +// CHECK: tf_device.return %[[ARG_0]] : tensor<*xf32> + tf_device.return %arg_0 : tensor<*xf32> + }) {device = "device"} : () -> tensor<*xf32> + func.return +} + +// ----- + +// CHECK-LABEL: func @return_multiple_operands +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<*xi32>) +func.func @return_multiple_operands(%arg_0: tensor<*xf32>, %arg_1: tensor<*xi32>) { + %result:2 = "tf_device.launch"() ({ +// CHECK: tf_device.return %[[ARG_0]], %[[ARG_1]] : tensor<*xf32>, tensor<*xi32> + tf_device.return %arg_0, %arg_1 : tensor<*xf32>, tensor<*xi32> + }) {device = "device"} : () -> (tensor<*xf32>, tensor) + func.return +} + +// ----- + +// CHECK-LABEL: func @empty_replicate +func.func @empty_replicate() { + tf_device.replicate {n = 2 : i32} { + } + func.return + +// CHECK: tf_device.replicate +// CHECK-SAME: n = 2 +// CHECK-NEXT: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @no_operand_replicate +func.func @no_operand_replicate() { + tf_device.replicate {n = 2 : i32} { + %0 = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + %1 = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + tf_device.return %0, %1 : tensor, tensor + } + func.return + // CHECK: tf_device.replicate + // CHECK-SAME: n = 2 + // CHECK: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @replicate_with_multiple_operands +func.func @replicate_with_multiple_operands() { + %0 = "tf.opA"() : () -> tensor<*xi1> + %1 = "tf.opB"() : () -> tensor<*xi1> + %2 = "tf.opC"() : () -> tensor<*xi1> + %3 = "tf.opD"() : () -> tensor<*xi32> + %4 = "tf.opE"() : () -> tensor<*xi32> + %5 = "tf.opF"() : () -> tensor<*xi32> + %6 = "tf.opG"() : () -> tensor<*xf32> + %7 = "tf.opH"() : () -> tensor<*xf32> + %8 = "tf.opI"() : () -> tensor<*xf32> + %9 = "tf.opJ"() : () -> tensor<*xi8> + %10 = "tf.opK"() : () -> tensor<*xi16> + %11 = "tf.opL"() : () -> tensor<*xi64> + tf_device.replicate([%0, %1, %2] as %input0: tensor<*xi1>, %9 as %input1: tensor<*xi8>, %10 as %input2: tensor<*xi16>, [%3, %4, %5] as %input3: tensor<*xi32>, [%6, %7, %8] as %input4: tensor<*xf32>, %11 as %input5: tensor<*xi64>) {n = 3 : i32} { + tf_device.return + } + func.return + +// CHECK: %[[OP_A:[a-z0-9]*]] = "tf.opA" +// CHECK: %[[OP_B:[a-z0-9]*]] = "tf.opB" +// CHECK: %[[OP_C:[a-z0-9]*]] = "tf.opC" +// CHECK: %[[OP_D:[a-z0-9]*]] = "tf.opD" +// CHECK: %[[OP_E:[a-z0-9]*]] = "tf.opE" +// CHECK: %[[OP_F:[a-z0-9]*]] = "tf.opF" +// CHECK: %[[OP_G:[a-z0-9]*]] = "tf.opG" +// CHECK: %[[OP_H:[a-z0-9]*]] = "tf.opH" +// CHECK: %[[OP_I:[a-z0-9]*]] = "tf.opI" +// CHECK: %[[OP_J:[a-z0-9]*]] = "tf.opJ" +// CHECK: %[[OP_K:[a-z0-9]*]] = "tf.opK" +// CHECK: %[[OP_L:[a-z0-9]*]] = "tf.opL" +// CHECK: tf_device.replicate +// CHECK-SAME: [%[[OP_A]], %[[OP_B]], %[[OP_C]]] as %{{[a-z0-9]*}}: tensor<*xi1> +// CHECK-SAME: [%[[OP_D]], %[[OP_E]], %[[OP_F]]] as %{{[a-z0-9]*}}: tensor<*xi32> +// CHECK-SAME: [%[[OP_G]], %[[OP_H]], %[[OP_I]]] as %{{[a-z0-9]*}}: tensor<*xf32> +// CHECK-SAME: %[[OP_J]] as %{{[a-z0-9]*}}: tensor<*xi8> +// CHECK-SAME: %[[OP_K]] as %{{[a-z0-9]*}}: tensor<*xi16> +// CHECK-SAME: %[[OP_L]] as %{{[a-z0-9]*}}: tensor<*xi64> +// CHECK-SAME: n = 3 +// CHECK-NEXT: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @replicate_derived_operandSegmentSizes +func.func @replicate_derived_operandSegmentSizes() { + tf_device.replicate {n = 2 : i32, operandSegmentSizes = array} { + } + func.return + +// CHECK: tf_device.replicate +// CHECK-SAME: n = 2 +// CHECK-NOT: operandSegmentSizes +// CHECK-NEXT: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @replicate_with_return +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<*xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>) +func.func @replicate_with_return(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xi32>) { + %result:4 = tf_device.replicate([%arg0, %arg1] as %input0: tensor<*xf32>) {n = 2 : i32} { + tf_device.return %input0, %arg2 : tensor<*xf32>, tensor<*xi32> + } + func.return + +// CHECK: tf_device.replicate +// CHECK-SAME: ([%[[ARG_0]], %[[ARG_1]]] as %[[INPUT_0:[a-z0-9]*]]: tensor<*xf32>) +// CHECK-SAME: n = 2 +// CHECK-NEXT: tf_device.return %[[INPUT_0]], %[[ARG_2]] +} + +// ----- + +// CHECK-LABEL: func @replicate_with_devices +func.func @replicate_with_devices() { + tf_device.replicate() {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"]}} { + tf_device.return + } + func.return + +// CHECK: tf_device.replicate +// CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"]} +// CHECK-SAME: n = 2 +// CHECK-NEXT: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @replicate_with_multiple_devices +func.func @replicate_with_multiple_devices() { + tf_device.replicate() {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"], TPU_REPLICATED_CORE_1 = ["/DEVICE:2", "/DEVICE:3"]}} { + tf_device.return + } + func.return + +// CHECK: tf_device.replicate +// CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"], TPU_REPLICATED_CORE_1 = ["/DEVICE:2", "/DEVICE:3"]} +// CHECK-SAME: n = 2 +// CHECK-NEXT: tf_device.return +} + +// ----- + +// CHECK-LABEL: func @replicate_with_inner_ops +func.func @replicate_with_inner_ops() { + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi1>) + %2 = "tf.opC"() : () -> (tensor<*xi32>) + %3 = "tf.opD"() : () -> (tensor<*xi32>) + %4 = "tf.opE"() : () -> (tensor<*xf32>) + %result:4 = tf_device.replicate([%0, %1] as %input0: tensor<*xi1>, [%2, %3] as %input1: tensor<*xi32>) {n = 2 : i32} { + %5 = "tf.opF"(%input0, %4) : (tensor<*xi1>, tensor<*xf32>) -> (tensor<*xi1>) + %6 = "tf.opG"(%input1, %4) : (tensor<*xi32>, tensor<*xf32>) -> (tensor<*xi32>) + tf_device.return %5, %6 : tensor<*xi1>, tensor<*xi32> + } + func.return +} + +// ----- + +// CHECK-LABEL: func @parallel_execute_two_regions +func.func @parallel_execute_two_regions() { + "tf_device.parallel_execute"() ({ + tf_device.return + }, + { + tf_device.return + }) {} : () -> () + func.return +} + +// ----- + +// CHECK-LABEL: func @parallel_execute_two_regions_with_ops +func.func @parallel_execute_two_regions_with_ops() { + "tf_device.parallel_execute"() ({ + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi32>) + tf_device.return %0, %1 : tensor<*xi1>, tensor<*xi32> + }, + { + %2 = "tf.opC"() : () -> (tensor<*xi1>) + tf_device.return + }) {} : () -> (tensor<*xi1>, tensor<*xi32>) + func.return +} + +// ----- + +// CHECK-LABEL: func @parallel_execute_regions_with_data_results +func.func @parallel_execute_regions_with_data_results() { + "tf_device.parallel_execute"() ({ + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi32>) + tf_device.return %0, %1 : tensor<*xi1>, tensor<*xi32> + }, + { + %2 = "tf.opC"() : () -> (tensor<*xf32>) + tf_device.return %2 : tensor<*xf32> + }) {} : () -> (tensor<*xi1>, tensor<*xi32>, tensor<*xf32>) + func.return +} + +// ----- + +func.func @parallel_execute_regions_with_data_results(%arg0: tensor) -> tensor { + // expected-error @+1 {{'func' attribute refers to an undefined function: undefined_func}} + %0 = "tf_device.cluster_func"(%arg0) {func = @undefined_func} : (tensor) -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops_invalid.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops_invalid.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4b7516df1ddb9c68fc986468920ad0b629b805d6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_device_ops_invalid.mlir @@ -0,0 +1,271 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics + +// Check that a replicate with 'n' attribute that is less than 2 is invalid. +func.func @parser_replicate_n(%arg0: tensor<*xf32>) { + tf_device.replicate([%arg0] as %input0: tensor<*xf32>) {n = 1 : i32} { +// expected-error@-1 {{'tf_device.replicate' expects 'n' to be at least 2, got 1}} + } + func.return +} + +// ----- + +// Check that a replicate replicated inputs where operand sizes do not match +// 'n' is invalid. +func.func @parser_replicate_operand_count(%arg0: tensor<*xf32>) { + tf_device.replicate([%arg0, %arg0, %arg0] as %input0: tensor<*xf32>) {n = 2 : i32} { +// expected-error@-1 {{'tf_device.replicate' expects number of operands for replicated input 0 to be 'n' (2), got 3}} + } + func.return +} + +// ----- + +// Check that a replicate with incompatible operands and block argument type is +// invalid. +func.func @parser_replicate_operand_type(%arg0: tensor<*xi32>) { +// expected-note@-1 {{prior use here}} + tf_device.replicate([%arg0, %arg0] as %input0: tensor<*xf32>) {n = 2 : i32} { +// expected-error@-1 {{use of value '%arg0' expects different type than prior uses: 'tensor<*xf32>' vs 'tensor<*xi32>'}} + } + func.return +} + +// ----- + +// Check that a replicate with multiple blocks in its region is invalid. +func.func @parser_replicate_region() { + tf_device.replicate() {n = 2 : i32} { +// expected-error@-1 {{custom op 'tf_device.replicate' expects a single block region}} + cf.br ^bb + ^bb: + tf_device.return + } + func.return +} + +// ----- + +// Check that a replicate with a bad terminator is invalid. +func.func @parser_replicate_terminator() { + tf_device.replicate() {n = 2 : i32} { +// expected-error@-1 {{custom op 'tf_device.replicate' expects a tf_device.return terminator}} + func.return + } + func.return +} + +// ----- + +// Check that an empty replicate is invalid (replicate needs a region). +func.func @verifier_replicate_no_block() { + "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op region #0 ('body') failed to verify constraint: region with 1 blocks}} + }) {n = 2 : i32, operandSegmentSizes = array} : () -> () + func.return +} + +// ----- + +// Check that an empty replicate block is invalid. +func.func @verifier_replicate_empty_block() { + "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op expects a non-empty block}} + ^entry: + }) {n = 2 : i32, operandSegmentSizes = array} : () -> () + func.return +} + +// ----- + +// Check that a replicate with a bad terminator is invalid. +func.func @verifier_replicate_terminator() { + "tf_device.replicate" () ({ +// expected-error@+2 {{'func.return' op expects parent op 'func.func'}} + ^entry: + func.return + }) {n = 2 : i32, operandSegmentSizes = array} : () -> () + func.return +} + +// ----- + +// Check that a replicate with 'n' attribute that is less than 2 is invalid. +func.func @verifier_replicate_n() { + "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op attribute 'n' failed to satisfy constraint: 32-bit signless integer attribute whose minimum value is 2}} + ^entry: + tf_device.return + }) {n = 1 : i32, operandSegmentSizes = array} : () -> () +} + +// ----- + +// Check that a replicate with mismatched 'n' attribute and device count is +// invalid. +func.func @verifier_replicate_n_device() { + "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op expects number of devices (2) to be equal to 'n' (3)}} + ^entry: + tf_device.return + }) {devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"]}, n = 3 : i32, operandSegmentSizes = array} : () -> () +} + +// ----- + +// Check that replicate op's 'devices' attribute must consist of dictionary +// with values as list with size equal to 'n' attribute. +func.func @verifier_replicate_n_device_multiple_alias() { + "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op expects number of devices (2) to be equal to 'n' (3)}} + ^entry: + tf_device.return + }) {devices = {TPU_REPLICATED_CORE_0 = ["/DEVICE:0", "/DEVICE:1"], TPU_REPLICATED_CORE_1 = ["/DEVICE:2"]}, n = 3 : i32, operandSegmentSizes = array} : () -> () +} + +// ----- + +// Check number of replicated inputs is evenly divisible by 'n'. +func.func @verifier_replicate_bad_operandSegmentSizes(%arg0: tensor<*xi32>) { + "tf_device.replicate" (%arg0, %arg0, %arg0, %arg0) ({ +// expected-error@-1 {{'tf_device.replicate' op expects number of replicated inputs (4) to be evenly divisible by 'n' (3)}} + ^entry(%input0: tensor<*xi32>, %input1: tensor<*xi32>): + tf_device.return + }) {n = 3 : i32, operandSegmentSizes = array} : (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) -> () +} + +// ----- + +// Check number of replicated inputs / 'n' + number of packed inputs matches the +// number of block arguments. +func.func @verifier_replicate_num_block_args(%arg0: tensor<*xi32>) { + "tf_device.replicate" (%arg0, %arg0, %arg0, %arg0, %arg0) ({ +// expected-error@-1 {{'tf_device.replicate' op expects number of block arguments (2) to be equal to number of replicated inputs (3) / 'n' (3) + number of packed inputs (2)}} + ^entry(%input0: tensor<*xi32>, %input1: tensor<*xi32>): + tf_device.return + }) {n = 3 : i32, operandSegmentSizes = array} : (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) -> () +} + +// ----- + +// Check that a replicate with incompatible replicated operand and block +// argument type is invalid. +func.func @verifier_replicate_replicated_operand_block_arg_type(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.replicate" (%arg0, %arg1) ({ +// expected-error@-1 {{'tf_device.replicate' op expects operand 1 ('tensor<*xi1>') and block argument 0 ('tensor<*xi32>') to have compatible types}} + ^entry(%input0: tensor<*xi32>): + tf_device.return + }) {n = 2 : i32, operandSegmentSizes = array} : (tensor<*xi32>, tensor<*xi1>) -> () +} + +// ----- + +// Check that a replicate with incompatible packed operand and block argument +// type is invalid. +func.func @verifier_replicate_packed_operand_block_arg_type(%arg0: tensor<*xi1>) { + "tf_device.replicate" (%arg0) ({ +// expected-error@-1 {{'tf_device.replicate' op expects operand 0 ('tensor<*xi1>') and block argument 0 ('tensor<*xi32>') to have compatible types}} + ^entry(%input0: tensor<*xi32>): + tf_device.return + }) {n = 2 : i32, operandSegmentSizes = array} : (tensor<*xi1>) -> () +} + +// ----- + +// Check that a replicate with mismatched result and terminator operand counts +// is invalid. +func.func @verifier_replicate_result_return_operand_count(%arg0: tensor<*xi32>) { + %result:3 = "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op expects number of results (3) to be equal to 'n' * number of terminator operands (2 * 1)}} + ^entry: + tf_device.return %arg0 : tensor<*xi32> + }) {n = 2 : i32, operandSegmentSizes = array} : () -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) +} + +// ----- + +// Check that a replicate with incompatible result and terminator operand type +// is invalid. +func.func @verifier_replicate_result_return_operand_type(%arg0: tensor<*xi32>) { + %result:2 = "tf_device.replicate" () ({ +// expected-error@-1 {{'tf_device.replicate' op incompatible types for result 1 and terminator operand 0}} + ^entry: + tf_device.return %arg0 : tensor<*xi32> + }) {n = 2 : i32, operandSegmentSizes = array} : () -> (tensor<*xi32>, tensor<*xi1>) +} + +// ----- + +// Check that a parallel_execute op with a single region is not allowed. +func.func @parallel_execute_single_region() { + "tf_device.parallel_execute"() {} : () -> () +// expected-error@-1 {{'tf_device.parallel_execute' op must have at least one region.}} + func.return +} + +// ----- + +// Check that a parallel_execute op with empty regions are not allowed. +func.func @parallel_execute_empty_region() { + "tf_device.parallel_execute"() ({ +// expected-error@-1 {{'tf_device.parallel_execute' op region #0 ('regions') failed to verify constraint: region with 1 blocks}} + }, + { + tf_device.return + }) {} : () -> () + func.return +} + +// ----- + +// Check that a parallel_execute ops with invalid number of output types are +// not allowed. +func.func @parallel_execute_invalid_output_type_numbers() { + "tf_device.parallel_execute"() ({ +// expected-error@-1 {{'tf_device.parallel_execute' op number of output types (3) must match the total number of outputs from all regions (2).}} + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi32>) + tf_device.return %0, %1 : tensor<*xi1>, tensor<*xi32> + }, + { + %2 = "tf.opC"() : () -> (tensor<*xi1>) + tf_device.return + }) {} : () -> (tensor<*xi1>, tensor<*xi32>, tensor<*xi32>) + func.return +} + +// ----- + +// Check that a parallel_execute ops with mismatching output types are not +// allowed. +func.func @parallel_execute_mismatched_output_types() { + "tf_device.parallel_execute"() ({ +// expected-error@-1 {{'tf_device.parallel_execute' op output types must be a concatenated list of output types for each regions.}} + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi32>) + tf_device.return %0, %1 : tensor<*xi1>, tensor<*xi32> + }, + { + %2 = "tf.opC"() : () -> (tensor<*xi1>) + tf_device.return + }) {} : () -> (tensor<*xi1>, tensor<*xi1>) + func.return +} + +// ----- + +// Check that a parallel_execute ops with unmatching output types for +// multiple regions with data outputs are not allowed. +func.func @parallel_execute_regions_with_invalid_data_results() { + "tf_device.parallel_execute"() ({ +// expected-error@-1 {{'tf_device.parallel_execute' op output types must be a concatenated list of output types for each regions.}} + %0 = "tf.opA"() : () -> (tensor<*xi1>) + %1 = "tf.opB"() : () -> (tensor<*xi32>) + tf_device.return %0, %1 : tensor<*xi1>, tensor<*xi32> + }, + { + %2 = "tf.opC"() : () -> (tensor<*xf32>) + tf_device.return %2 : tensor<*xf32> + }) {} : () -> (tensor<*xi1>, tensor<*xi32>, tensor<*xi1>) + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_dialect_to_executor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_dialect_to_executor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..771cd2809ebdded33ed48b1a8ac64fd19f60ecd9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_dialect_to_executor.mlir @@ -0,0 +1,37 @@ +// RUN: tf-opt %s -tf-replicate-invariant-op-hoisting -tf-dialect-to-executor-v2 | FileCheck %s + +// CHECK-LABEL: @concat_that_uses_replicated_args +// CHECK: Concat +// CHECK-NOT: Concat +func.func @concat_that_uses_replicated_args( + %arg0: tensor<128x10xf32>, + %arg1: tensor<128x10xf32>, + %arg2: tensor<*xi32>, + %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) { + %0:4 = tf_device.replicate([%arg0, %arg1] as %arg4: tensor<128x10xf32>, + [%arg2, %arg3] as %arg5: tensor<*xi32>) {n = 2 : i32} { + %6 = "tf.Something"(%arg4) : (tensor<128x10xf32>) -> tensor<*xi32> + %cst_0 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %5 = "tf.Concat"(%cst_0, %arg5, %6) : (tensor, tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + tf_device.return %5, %6 : tensor<*xi32>, tensor<*xi32> + } + return %0#0, %0#0, %0#2, %0#3 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: @concat_that_does_not_use_replicated_args +// CHECK: Concat +// CHECK-NOT: Concat +func.func @concat_that_does_not_use_replicated_args( + %arg0: tensor<128x10xf32>, + %arg1: tensor<128x10xf32>, + %arg2: tensor<*xi32>, + %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) { + %0:4 = tf_device.replicate([%arg0, %arg1] as %arg4: tensor<128x10xf32>, + [%arg2, %arg3] as %arg5: tensor<*xi32>) {n = 2 : i32} { + %6 = "tf.Something"(%arg4) : (tensor<128x10xf32>) -> tensor<*xi32> + %cst_0 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %5 = "tf.Concat"(%cst_0, %arg2, %arg3) : (tensor, tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + tf_device.return %5, %6 : tensor<*xi32>, tensor<*xi32> + } + return %0#0, %0#1, %0#2, %0#3 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0d91245a92d30d42d6713572129443e50f49f546 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir @@ -0,0 +1,554 @@ +// RUN: tf-opt %s | tf-opt | FileCheck %s + +// CHECK-LABEL: func private @control_type() -> !tf_executor.control +func.func private @control_type() -> !tf_executor.control + +// CHECK-LABEL: func private @token_type() -> !tf_executor.token +func.func private @token_type() -> !tf_executor.token + +// CHECK-LABEL: func @empty_graph +func.func @empty_graph() { + tf_executor.graph { + } + func.return + +// CHECK: tf_executor.graph { +// CHECK-NEXT: tf_executor.fetch +// CHECK-NEXT: } + +} + +// CHECK-LABEL: func @graph_with_fetch(%{{.*}}: tensor<*xf32>) +func.func @graph_with_fetch(%0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + tf_executor.fetch %0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @graph_with_fetch_attributes(%{{.*}}: tensor<*xf32>) +func.func @graph_with_fetch_attributes(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + tf_executor.fetch %arg0 : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + } {attr2 = 32 : i64, tf_executor.attr1 = "value1"} + +// CHECK: tf_executor.graph { +// CHECK-NEXT: tf_executor.fetch %{{.*}} : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} +// CHECK-NEXT: } {attr2 = 32 : i64, tf_executor.attr1 = "value1"} + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @simpleIsland(%{{.*}}: tensor<*xf32>) +func.func @simpleIsland(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + tf_executor.yield %arg0 : tensor<*xf32> + } + tf_executor.fetch %1#0 : tensor<*xf32> + } +// CHECK: tf_executor.island { +// CHECK: tf_executor.yield %{{.*}} : tensor<*xf32> +// CHECK: tf_executor.fetch {{.*}} : tensor<*xf32> + + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @simpleIsland_with_attributes(%{{.*}}: tensor<*xf32>) +func.func @simpleIsland_with_attributes(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + tf_executor.yield %arg0 : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + } {attr2 = 32 : i64, tf_executor.attr1 = "value1"} + +// CHECK: tf_executor.yield %{{.*}} : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} +// CHECK-NEXT: } {attr2 = 32 : i64, tf_executor.attr1 = "value1"} + + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @simpleIsland_with_multiple_control_inputs(%{{.*}}: tensor<*xf32>) +func.func @simpleIsland_with_multiple_control_inputs(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1 = tf_executor.island { + tf_executor.yield + } + %2 = tf_executor.island { + tf_executor.yield + } + %3:2 = tf_executor.island(%1, %2) { + tf_executor.yield %arg0 : tensor<*xf32> + } + tf_executor.fetch %3#0 : tensor<*xf32> + } +// CHECK: %[[ISLAND0:[a-z_0-9]*]] = tf_executor.island { +// CHECK-NEXT: tf_executor.yield +// CHECK: %[[ISLAND1:[a-z_0-9]*]] = tf_executor.island { +// CHECK-NEXT: tf_executor.yield +// CHECK: %[[ISLAND2:.*]], %[[ISLAND2_control:.*]] = tf_executor.island(%[[ISLAND0]], %[[ISLAND1]]) { +// CHECK: tf_executor.fetch %[[ISLAND2]] : tensor<*xf32> + + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @fetchWithControlDep(%{{.*}}: tensor<*xf32>) +func.func @fetchWithControlDep(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + %val, %ctl_id = tf_executor.island { + %val = arith.addf %arg0, %arg0 : tensor<*xf32> + tf_executor.yield %arg0 : tensor<*xf32> + } +// CHECK: tf_executor.fetch %{{.*}}, %{{.*}} : tensor<*xf32>, !tf_executor.control + tf_executor.fetch %val, %ctl_id : tensor<*xf32>, !tf_executor.control + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @testAddWithControlDependency(%{{.*}}: tensor<*xf32>) +func.func @testAddWithControlDependency(%0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + // This identity operation is unused, but the control dependency on the + // island is consumed by the add right after. + %unused, %ctl_id = tf_executor.island { + %id_in = "tf.Identity"(%0) : (tensor<*xf32>) -> tensor<*xf32> + tf_executor.yield %id_in : tensor<*xf32> + } + + // The control dependency is held by the operand. +// CHECK: tf_executor.island(%{{.*}}) + %add, %clt_add = tf_executor.island(%ctl_id) { + %add_in= "tf.Add"(%0, %0) : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>) + tf_executor.yield %add_in : tensor<*xf32> + } + + tf_executor.fetch %add : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : tensor<*xf32> + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_broadcast(%{{.*}}: tensor<2xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_with_broadcast(%arg0: tensor<2xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : (tensor<2xf32>, tensor) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : (tensor<2xf32>, tensor) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_broadcast_one_output(%{{.*}}: tensor<2xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_with_broadcast_one_output(%arg0: tensor<2xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : (tensor<2xf32>, tensor) -> (tensor<*xf32>, tensor<2xf32>, !tf_executor.control) + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : (tensor<2xf32>, tensor) -> (tensor<*xf32>, tensor<2xf32>, !tf_executor.control) + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_broadcast_output_only(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_with_broadcast_output_only(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor<2xf32>, !tf_executor.control) + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor<2xf32>, !tf_executor.control) + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_attributes(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_with_attributes(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_unranked_pred(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor<*xi1>) -> tensor<*xf32> { +func.func @switch_with_unranked_pred(%arg0: tensor<*xf32>, %arg1: tensor<*xi1>) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : (tensor<*xf32>, tensor<*xi1>) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : (tensor<*xf32>, tensor<*xi1>) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_with_control_inputs( +func.func @switch_with_control_inputs(%arg0: tensor, %arg1: !tf_executor.control, %arg2: !tf_executor.control) -> tensor { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{[^%]*}}, %{{[^%]*}}, %{{[^%]*}}, %{{[^%]*}} : tensor + %1:3 = tf_executor.Switch %arg0, %arg0, %arg1, %arg2 : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %result : tensor +} + +// CHECK-LABEL: func @switch_with_control_inputs_functional( +func.func @switch_with_control_inputs_functional(%arg0: tensor, %arg1: !tf_executor.control, %arg2: !tf_executor.control) -> tensor { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{[^%]*}}, %{{[^%]*}}, %{{[^%]*}}, %{{[^%]*}} : tensor + %1:3 = tf_executor.Switch %arg0, %arg0, %arg1, %arg2 : (tensor, tensor, !tf_executor.control, !tf_executor.control) -> (tensor, tensor, !tf_executor.control) + tf_executor.fetch %1#0 : tensor + } + func.return %result : tensor +} + +// CHECK-LABEL: func @switchN( +func.func @switchN(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %fetches = tf_executor.graph { + +// CHECK: tf_executor._SwitchN %{{.*}}, %{{.*}} of 5 : tensor<*xf32> + %1:6 = tf_executor._SwitchN %arg1, %arg0 of 5 : tensor<*xf32> + +// CHECK: tf_executor._SwitchN %{{.*}}, %{{.*}} of 12 (%{{.*}}) : tensor<*xf32> + %2:13 = tf_executor._SwitchN %arg1, %arg0 of 12 (%1#5) : tensor<*xf32> + + tf_executor.fetch %2#0 : tensor<*xf32> + } + func.return %fetches : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_merge(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_merge(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : tensor<*xf32> + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + +// CHECK: tf_executor.Merge %{{.*}}, %{{.*}} : tensor<*xf32> + %value, %idx, %ctlMerge = tf_executor.Merge %true, %false : tensor<*xf32> + + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_merge_with_ctl(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_merge_with_ctl(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : tensor<*xf32> + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + +// CHECK: tf_executor.Merge %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : tensor<*xf32> + %value, %idx, %ctlMerge = tf_executor.Merge %true, %false, %ctlSwitch, %ctlSwitch, %ctlSwitch : tensor<*xf32> + + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @switch_merge_with_attributes(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @switch_merge_with_attributes(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Switch %{{.*}}, %{{.*}} : tensor<*xf32> + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + +// CHECK: tf_executor.Merge %{{.*}}, %{{.*}} : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %value, %idx, %ctlMerge = tf_executor.Merge %true, %false : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// Verify that long form printing is used when operand types do not match the +// result type and then it can be parsed again correctly. +// CHECK-LABEL: func @merge_different_operand_types +func.func @merge_different_operand_types(%arg0: tensor<*xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor<*xf32>, tensor<4xf32>) -> (tensor<4xf32>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = tf_executor.Merge %arg0, %arg1 : (tensor<*xf32>, tensor<4xf32>) -> (tensor<4xf32>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor<4xf32> + } + func.return %result : tensor<4xf32> +} + +// Verify that long form printing is used when there is only one data operand +// and then it can be parsed again correctly. +// CHECK-LABEL: func @merge_one_data_operand +func.func @merge_one_data_operand(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor<*xf32>) -> (tensor<*xf32>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = tf_executor.Merge %arg0 : (tensor<*xf32>) -> (tensor<*xf32>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @merge_with_variant_type +func.func @merge_with_variant_type(%arg0: tensor, %arg1: tensor>>) -> tensor>> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor, tensor>>) -> (tensor>>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor, tensor>>) -> (tensor>>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor>> + } + func.return %result : tensor>> +} + +// CHECK-LABEL: func @merge_with_resource_type +func.func @merge_with_resource_type(%arg0: tensor, %arg1: tensor>>) -> tensor>> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor, tensor>>) -> (tensor>>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor, tensor>>) -> (tensor>>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor>> + } + func.return %result : tensor>> +} + +// CHECK-LABEL: func @merge_with_ref_type +func.func @merge_with_ref_type(%arg0: tensor<4x!tf_type.f32ref>, %arg1: tensor<4xf32>) -> tensor<4xf32> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor<4x!tf_type.f32ref>, tensor<4xf32>) -> (tensor<4xf32>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<4x!tf_type.f32ref>, tensor<4xf32>) -> (tensor<4xf32>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor<4xf32> + } + func.return %result : tensor<4xf32> +} + +// CHECK-LABEL: func @merge_with_dynamic_shape +func.func @merge_with_dynamic_shape(%arg0: tensor<2xf32>, %arg1: tensor<3xf32>) -> tensor { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor<2xf32>, tensor<3xf32>) -> (tensor, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<2xf32>, tensor<3xf32>) -> (tensor, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor + } + func.return %result : tensor +} + +// CHECK-LABEL: func @merge_with_unranked_shape +func.func @merge_with_unranked_shape(%arg0: tensor<2xf32>, %arg1: tensor<3xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { + +// CHECK: tf_executor.Merge{{.*}}(tensor<2xf32>, tensor<3xf32>) -> (tensor<*xf32>, tensor, !tf_executor.control) + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<2xf32>, tensor<3xf32>) -> (tensor<*xf32>, tensor, !tf_executor.control) + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/fra\22me" : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0 frame "some/fra\"me" : tensor<*xf32> + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_broadcast(%{{.*}}: tensor<8xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter_broadcast(%arg0: tensor<8xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/fra\22me" : (tensor<8xf32>) -> (tensor<*xf32>, !tf_executor.control) + %res:2 = tf_executor.Enter %arg0 frame "some/fra\"me" : (tensor<8xf32>) -> (tensor<*xf32>, !tf_executor.control) + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_parallel_iterations(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter_parallel_iterations(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/frame" parallel_iterations 42 : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0 frame "some/frame" parallel_iterations 42 : tensor<*xf32> + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_constant(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter_constant(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/frame" constant : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0 frame "some/frame" constant : tensor<*xf32> + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_parallel_iterations_constant(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter_parallel_iterations_constant(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/frame" parallel_iterations 42 constant : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0 frame "some/frame" parallel_iterations 42 constant : tensor<*xf32> + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_with_attributes(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @enter_with_attributes(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { +// CHECK: tf_executor.Enter %{{.*}} frame "some/frame" : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %res:2 = tf_executor.Enter %arg0 frame "some/frame" : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_control(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @enter_control(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> +// CHECK: tf_executor.Enter %{{.*}}, %{{.*}}, %{{.*}} frame "some/frame" : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0, %1#2, %1#2 frame "some/frame" : tensor<*xf32> + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @enter_control_longform(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @enter_control_longform(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> +// CHECK: tf_executor.Enter %{{.*}}, %{{.*}}, %{{.*}} frame "some/frame" : tensor<*xf32> + %res:2 = tf_executor.Enter %arg0, %1#2, %1#2 frame "some/frame" : (tensor<*xf32>, !tf_executor.control, !tf_executor.control) -> (tensor<*xf32>, !tf_executor.control) + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @nextiteration(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @nextiteration(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> + tf_executor.NextIteration.Sink[%1#1] %1#0 : tensor<*xf32> +// CHECK: tf_executor.NextIteration.Source : tensor<*xf32> +// CHECK: tf_executor.NextIteration.Sink[%{{.*}}] %{{.*}} : tensor<*xf32> + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @nextiteration_with_attributes(%{{.*}}: tensor<*xf32>, %{{.*}}: i1) -> tensor<*xf32> { +func.func @nextiteration_with_attributes(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + tf_executor.NextIteration.Sink[%1#1] %1#0 : tensor<*xf32> {attr4 = 42 : i64, tf_executor.attr_push = "other_value"} +// CHECK: tf_executor.NextIteration.Source : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} +// CHECK: tf_executor.NextIteration.Sink[%{{.*}}] %{{.*}} : tensor<*xf32> {attr4 = 42 : i64, tf_executor.attr_push = "other_value"} + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @nextiteration_control(%{{.*}}: tensor<*xf32>, %{{.*}}: tensor) -> tensor<*xf32> { +func.func @nextiteration_control(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + %2:2 = tf_executor.Enter %arg0, %1#2, %1#2 frame "some/frame" : tensor<*xf32> + %3:3 = tf_executor.NextIteration.Source : tensor<*xf32> + tf_executor.NextIteration.Sink[%3#1] %3#0, %1#2 : tensor<*xf32> +// CHECK: tf_executor.NextIteration.Source : tensor<*xf32> +// CHECK: tf_executor.NextIteration.Sink[%{{.*}}] %{{.*}}, %{{.*}} : tensor<*xf32> + tf_executor.fetch %3#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @exit(%{{.*}}: tensor<*xf32>) -> tensor<*xf32> { +func.func @exit(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { +// CHECK: tf_executor.Exit %{{.*}} : tensor<*xf32> + %1:2 = tf_executor.Exit %arg0 : tensor<*xf32> + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @exit_with_attributes(%{{.*}}: tensor<*xf32>) -> tensor<*xf32> { +func.func @exit_with_attributes(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { +// CHECK: tf_executor.Exit %{{.*}} : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %1:2 = tf_executor.Exit %arg0 : tensor<*xf32> {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @exit_with_control(%{{.*}}: tensor<*xf32>, %{{.*}}: !tf_executor.control) -> tensor<*xf32> { +func.func @exit_with_control(%arg0: tensor<*xf32>, %arg1: !tf_executor.control) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.Exit %arg0, %arg1 : tensor<*xf32> + %2:2 = tf_executor.Exit %arg0, %1#1 : tensor<*xf32> + tf_executor.fetch %2#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @control_trigger(%{{.*}}: !tf_executor.control, %{{.*}}: !tf_executor.control) { +func.func @control_trigger(%arg0: !tf_executor.control, %arg1: !tf_executor.control) { + tf_executor.graph { +// CHECK: tf_executor.ControlTrigger %{{.*}}, %{{.*}} + %0 = tf_executor.ControlTrigger %arg0, %arg1 + } + func.return +} + +// CHECK-LABEL: func @control_trigger_with_attributes(%{{.*}}: !tf_executor.control, %{{.*}}: !tf_executor.control) { +func.func @control_trigger_with_attributes(%arg0: !tf_executor.control, %arg1: !tf_executor.control) { + tf_executor.graph { +// CHECK: tf_executor.ControlTrigger %{{.*}}, %{{.*}} {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %0 = tf_executor.ControlTrigger %arg0, %arg1 {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + } + func.return +} + +// CHECK-LABEL: func @loop_cond(%{{.*}}: tensor, %{{.*}}: !tf_executor.control) -> tensor { +func.func @loop_cond(%arg0: tensor, %arg1: !tf_executor.control) -> tensor { + %0 = tf_executor.graph { +// CHECK: tf_executor.LoopCond %{{.*}} : tensor + %1:2 = tf_executor.LoopCond %arg0 : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @loop_cond_with_attributes(%{{.*}}: tensor, %{{.*}}: !tf_executor.control) -> tensor { +func.func @loop_cond_with_attributes(%arg0: tensor, %arg1: !tf_executor.control) -> tensor { + %0 = tf_executor.graph { +// CHECK: tf_executor.LoopCond %{{.*}} : tensor {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + %1:2 = tf_executor.LoopCond %arg0 : tensor {attr3 = 32 : i64, tf_executor.attr_fetch = "some_value"} + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @loop_cond_with_control(%{{.*}}: tensor, %{{.*}}: !tf_executor.control) -> tensor { +func.func @loop_cond_with_control(%arg0: tensor, %arg1: !tf_executor.control) -> tensor { + %0 = tf_executor.graph { +// CHECK: tf_executor.LoopCond %{{.*}}, %{{.*}} : tensor + %1:2 = tf_executor.LoopCond %arg0, %arg1 : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @loop_cond_with_control_broadcast(%{{.*}}: tensor, %{{.*}}: !tf_executor.control) -> tensor<*xi1> { +func.func @loop_cond_with_control_broadcast(%arg0: tensor, %arg1: !tf_executor.control) -> tensor<*xi1> { + %0 = tf_executor.graph { +// CHECK: tf_executor.LoopCond %{{.*}}, %{{.*}} : (tensor, !tf_executor.control) -> (tensor<*xi1>, !tf_executor.control) + %1:2 = tf_executor.LoopCond %arg0, %arg1 : (tensor, !tf_executor.control) -> (tensor<*xi1>, !tf_executor.control) + tf_executor.fetch %1#0 : tensor<*xi1> + } + func.return %0 : tensor<*xi1> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_invalid.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_invalid.mlir new file mode 100644 index 0000000000000000000000000000000000000000..07f68824e1c95bf9c9ee0b318fb4d2f3ce10e966 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_invalid.mlir @@ -0,0 +1,784 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics + +func.func @invalid_type() -> !tf_executor.foobar +// expected-error@-1 {{unknown tf_executor type: foobar}} + +// ----- + +// Check that tf_executor.graph does not accept any operand. +func.func @graph_with_invalid_op(%arg0: tensor<*xf32>) { + "tf_executor.graph" (%arg0) ({}) : (tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.graph' op requires zero operands}} + func.return +} + +// ----- + +// Check that an empty graph is invalid (it needs a region). +func.func @empty_graph() { + "tf_executor.graph" () ({ +// expected-error@-1 {{'tf_executor.graph' op region #0 ('body') failed to verify constraint: region with 1 blocks}} + }) : () -> () + func.return +} + +// ----- + +// Check that an empty graph is invalid (it needs a region). +func.func @empty_graph() { + "tf_executor.graph" () ({ +// expected-error@-1 {{'tf_executor.graph' op expects a non-empty block}} + ^entry: + }) : () -> () + func.return +} + +// ----- + +// Check that only tf_executor operations can be present in a tf_executor.graph. +func.func @graph_with_invalid_op(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = "tf_executor.graph" () ({ + %val = arith.addf %arg0, %arg0 : tensor<*xf32> +// expected-error@-1 {{'arith.addf' op unallowed inside a tf_executor.graph region}} + tf_executor.fetch %val : tensor<*xf32> + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that tf_executor.graph can't be nested directly in a tf_executor.graph. +func.func @nested_graph() { + tf_executor.graph { + tf_executor.graph {} +// expected-error@-1 {{'tf_executor.graph' op unallowed directly inside another tf_executor.graph}} + } + func.return +} + +// ----- + +// Check that a tf_executor.fetch is terminating a tf_executor.graph (custom parser) +func.func @graph_with_invalid_terminator(%arg0: tensor<*xf32>) -> tensor<*xf32> { + tf_executor.graph { +// expected-error@-1 {{custom op 'tf_executor.graph' expects a tf_executor.fetch terminator}} + func.return + } + func.return %arg0 : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.fetch parent is a graph. +func.func @parent_is_graph() { + "tf.some_op"() ({ + tf_executor.fetch +// expected-error@-1 {{'tf_executor.fetch' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that a tf_executor.fetch is terminating a tf_executor.graph (verifier) +func.func @graph_with_invalid_terminator(%arg0: tensor<*xf32>) -> tensor<*xf32> { +// expected-error@+2 {{'tf_executor.yield' op invalid tf_executor.graph terminator, fetch expected}} + "tf_executor.graph" () ({ + tf_executor.yield + }) : () -> () + func.return %arg0 : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.fetch is terminating a tf_executor.graph. +func.func @graph_with_invalid_terminator(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = "tf_executor.graph" () ({ + "tf_executor.fetch"() : () -> () +// expected-error@-1 {{'tf_executor.fetch' op does not have enough operands to cover the graph returned values}} + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a graph with multiple regions issues an error +func.func @graph_with_multiple_region(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = tf_executor.graph { +// expected-error@-1 {{custom op 'tf_executor.graph' expects a single block region}} + cf.br ^bb + ^bb: + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a fetch with not enough operands triggers the verifier. +func.func @invalid_fetch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = "tf_executor.graph"() ({ + "tf_executor.fetch"() : () -> () +// expected-error@-1 {{'tf_executor.fetch' op does not have enough operands to cover the graph returned values}} + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a fetch with not enough data-operands but more control inputs triggers the verifier. +func.func @invalid_fetch(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) -> tensor<*xf32> { + %result = "tf_executor.graph"() ({ + "tf_executor.fetch"(%ctl) : (!tf_executor.control) -> () +// expected-error@-1 {{'tf_executor.fetch' op operand #0 is a control type, can't be bound to a graph result}} + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a fetch with not too many operands triggers the verifier. +func.func @invalid_fetch(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %result = "tf_executor.graph"() ({ + "tf_executor.fetch"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.fetch' op operand #1 does not have a graph results to bind}} + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a fetch with operands that mistmatch the graph result type triggers the verifier. +func.func @invalid_fetch(%arg0: tensor<*xf32>) -> i32 { + %result = "tf_executor.graph"() ({ + "tf_executor.fetch"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.fetch' op operand #0 type mismatch graph results}} + }) : () -> i32 + func.return %result : i32 +} + +// ----- + +// Check that a fetch with operands after a control input triggers the verifier. +func.func @invalid_fetch(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) -> tensor<*xf32> { + %result = "tf_executor.graph"() ({ + "tf_executor.fetch"(%arg0, %ctl, %arg0) : (tensor<*xf32>, !tf_executor.control, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.fetch' op found non-control operand #2 after control operand}} +// expected-error@-2 {{'tf_executor.fetch' op failed to verify that all control inputs must appear after any non-control input}} + }) : () -> tensor<*xf32> + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.island parent is a graph. +func.func @parent_is_graph() { + "tf.some_op"() ({ + %ctl = tf_executor.island {} +// expected-error@-1 {{'tf_executor.island' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that an island can't have other operands than controls. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"(%arg0) ({ +// expected-error@-1 {{'tf_executor.island' op operand #0 must be variadic of control}} + }) : (tensor<*xf32>) -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island must have at least a control result. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ +// expected-error@-1 {{'tf_executor.island' op expected 1 or more results}} + }) : () -> () + } + func.return +} + +// ----- + +// Check that an island region can't be empty. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ +// expected-error@-1 {{'tf_executor.island' op region #0 ('body') failed to verify constraint: region with 1 blocks}} + }) : () -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island body can't be empty. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ +// expected-error@-1 {{'tf_executor.island' op expects a non-empty block}} + ^entry: + }) : () -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island body doesn't have any block arguments. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + // expected-error@-1 {{expects body without any arguments}} + ^entry(%arg: tensor<2xi32>): + tf_executor.yield + }) : () -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island body can't be empty. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ +// expected-error@+1 {{'func.return' op invalid tf_executor.island terminator, yield expected}} + func.return + }) : () -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that a tf_executor.yield parent is a tf_executor.island. +func.func @parent_is_island() { + "tf.some_op"() ({ + tf_executor.yield +// expected-error@-1 {{'tf_executor.yield' op expects parent op 'tf_executor.island'}} + }) : () -> () + func.return +} + +// ----- + +// Check that an island yield matches the island results. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + "tf_executor.yield"(%arg0) : (tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.yield' op has 1 operand, but island returns 0}} + }) : () -> (!tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island yield matches the island results. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + "tf_executor.yield"(%arg0) : (tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.yield' op operand #0 type mismatch island results}} + }) : () -> (i32, !tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island yield matches the island results. +func.func @invalid_island(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + "tf_executor.yield"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.yield' op operand #1 type mismatch island results}} + }) : () -> (tensor<*xf32>, i32, !tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island yield controls are after all non-control inputs. +func.func @invalid_yield(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + "tf_executor.yield"(%arg0, %ctl, %arg0) : (tensor<*xf32>, !tf_executor.control, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.yield' op unexpected control type for operand #1}} + }) : () -> (tensor<*xf32>, !tf_executor.control, tensor<*xf32>, !tf_executor.control) + } + func.return +} + +// ----- + +// Check that an island yield controls are after all non-control inputs. +func.func @invalid_yield(%arg0: tensor<*xf32>, %ctl: !tf_executor.control) { + tf_executor.graph { + "tf_executor.island"() ({ + "tf_executor.yield"(%arg0, %ctl) : (tensor<*xf32>, !tf_executor.control) -> () +// expected-error@-1 {{'tf_executor.yield' op unexpected control type for operand #1}} + }) : () -> (tensor<*xf32>, !tf_executor.control, !tf_executor.control) + } + func.return +} + +// ----- + +// Check that a tf_executor.Switch parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>, %arg1: tensor) { + "tf.some_op"() ({ + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> +// expected-error@-1 {{'tf_executor.Switch' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that a switch always needs at least two arguments. +func.func @invalid_switch(%arg0: tensor<*xf32>) { + tf_executor.graph { + %true, %false, %ctlSwitch = "tf_executor.Switch"(%arg0) : (tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Switch' op expected 2 or more operands}} + } + func.return +} + +// ----- + +// Check that a switch always needs at least two arguments. +func.func @invalid_switch(%arg0: tensor<*xf32>) { + tf_executor.graph { + %true, %false, %ctlSwitch = tf_executor.Switch %arg0 : tensor<*xf32> +// expected-error@-1 {{custom op 'tf_executor.Switch' expects a single data type and a predicate}} + } + func.return +} + +// ----- + +// Check that a switch second argument must be a valid predicate (i1). +func.func @invalid_switch(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = "tf_executor.Switch"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Switch' op operand #1 must be tensor of 1-bit signless integer values}} + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a switch result type matches the input type. +func.func @invalid_switch(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = "tf_executor.Switch"(%arg1, %arg1) : (tensor, tensor) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Switch' op failed to verify that data operand must be broadcastable to true result}} + tf_executor.fetch %true : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a switch result type matches the input type. +func.func @invalid_switch(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = "tf_executor.Switch"(%arg1, %arg1) : (tensor, tensor) -> (tensor, tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Switch' op failed to verify that data operand must be broadcastable to false result}} + tf_executor.fetch %false : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor._SwitchN parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>, %arg1: tensor) { + "tf.some_op"() ({ + %1:6 = tf_executor._SwitchN %arg0, %arg1 of 5 : tensor<*xf32> +// expected-error@-1 {{'tf_executor._SwitchN' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that switchN result numbers matches the num_out attribute. +func.func @invalid_switchN(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %fetches = tf_executor.graph { + + %1:3 = "tf_executor._SwitchN"(%arg1, %arg0) {num_outs = 5} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor._SwitchN' op expect `num_outs` (5) results but got 2}} + + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %fetches : tensor<*xf32> +} + +// ----- + +// Check that data operands of SwitchN have tensor type +func.func @invalid_switchN(%arg0: i32, %arg1: tensor) -> tensor<*xi32> { + %result = tf_executor.graph { + %1:3 = "tf_executor._SwitchN"(%arg0, %arg1) {num_outs = 2} : (i32, tensor) -> (tensor<*xi32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor._SwitchN' op expects data operand to have tensor type but got 'i32'}} + tf_executor.fetch %1#0 : tensor<*xi32> + } + func.return %result : tensor<*xi32> +} + +// ----- + +// Check that result of SwitchN has tensor type +func.func @invalid_switchN(%arg0: tensor<*xi32>, %arg1: tensor) -> i32 { + %result = tf_executor.graph { + %1:3 = "tf_executor._SwitchN"(%arg0, %arg1) {num_outs = 2} : (tensor<*xi32>, tensor) -> (i32, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor._SwitchN' op expects outputs to have tensor type but got 'i32'}} + tf_executor.fetch %1#0 : i32 + } + func.return %result : i32 +} + +// ----- + +// Check that if any result is a ref type, then data operand needs to be ref too. +func.func @invalid_switchN(%arg0: tensor<4xf32>, %arg1: tensor) -> tensor<4x!tf_type.f32ref> { + %fetches = tf_executor.graph { + + %1:3 = "tf_executor._SwitchN"(%arg0, %arg1) {num_outs = 2} : (tensor<4xf32>, tensor) -> (tensor<4x!tf_type.f32ref>, tensor<4xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor._SwitchN' op expects same operand and output element type but got 'tensor<4xf32>' vs 'tensor<4x!tf_type.f32ref>'}} + tf_executor.fetch %1#0 : tensor<4x!tf_type.f32ref> + } + func.return %fetches : tensor<4x!tf_type.f32ref> +} + +// ----- + +// Check that switchN data operand is broadcastable with all output types +func.func @invalid_switchN(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %fetches = tf_executor.graph { + + %1:3 = "tf_executor._SwitchN"(%arg0, %arg1) {num_outs = 2} : (tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor._SwitchN' op expects data operand to be broadcastable with all output types but got 'tensor<*xf32>' vs 'tensor'}} + + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %fetches : tensor<*xf32> +} + +// ----- + +// Check that switchN custom type has a single entry. +func.func @invalid_switchN(%arg0: tensor, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %fetches = tf_executor.graph { + + %1:3 = tf_executor._SwitchN %arg1, %arg0 of 2 : tensor<*xf32>, i32 +// expected-error@-1 {{custom op 'tf_executor._SwitchN' expects only a single data type}} + + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %fetches : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.Merge parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>) { + "tf.some_op"() ({ + %value, %idx, %ctlMerge = tf_executor.Merge %arg0, %arg0 : tensor<*xf32> +// expected-error@-1 {{'tf_executor.Merge' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that merge has at least one operand. +func.func @invalid_merge(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + + %value, %idx, %ctlMerge = "tf_executor.Merge"() : () -> (tensor<*xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects at least one operand}} + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that merge has at least one non-control operand. +func.func @invalid_merge(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + + %value, %idx, %ctlMerge = "tf_executor.Merge"(%ctlSwitch) : (!tf_executor.control) -> (tensor<*xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects a non-control input}} + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that data operands of merge have tensor type +func.func @invalid_merge(%arg0: tensor<*xi32>, %arg1: i32) -> tensor<*xi32> { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<*xi32>, i32) -> (tensor<*xi32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects data operands to have tensor type but got 'i32'}} + tf_executor.fetch %value : tensor<*xi32> + } + func.return %result : tensor<*xi32> +} + +// ----- + +// Check that result of merge has tensor type +func.func @invalid_merge(%arg0: tensor<*xi32>, %arg1: tensor) -> i32 { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> (i32, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op result #0 must be tensor of any non-token type values, but got 'i32'}} + tf_executor.fetch %value : i32 + } + func.return %result : i32 +} + +// ----- + +// Check that merge data inputs are all the same type +func.func @invalid_merge(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + + %value, %idx, %ctlMerge = "tf_executor.Merge"(%true, %false, %arg1) : (tensor<*xf32>, tensor<*xf32>, tensor) -> (tensor<*xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects all operands to be broadcastable with output type but got 'tensor' vs 'tensor<*xf32>'}} + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that merge data inputs are broadcastable to the output +func.func @invalid_merge(%arg0: tensor<*xf32>, %arg1: tensor<4xf32>) -> tensor<8xf32> { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<*xf32>, tensor<4xf32>) -> (tensor<8xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects all operands to be broadcastable with output type but got 'tensor<4xf32>' vs 'tensor<8xf32>'}} + tf_executor.fetch %value : tensor<8xf32> + } + func.return %result : tensor<8xf32> +} + +// ----- + +// Check that merge data inputs of variant type are broadcastable to the output +func.func @invalid_merge(%arg0: tensor<*x!tf_type.variant>, %arg1: tensor<4x!tf_type.variant>) -> tensor<8x!tf_type.variant> { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<*x!tf_type.variant>, tensor<4x!tf_type.variant>) -> (tensor<8x!tf_type.variant>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects all operands to be broadcastable with output type but got 'tensor<4x!tf_type.variant>' vs 'tensor<8x!tf_type.variant>'}} + tf_executor.fetch %value : tensor<8x!tf_type.variant> + } + func.return %result : tensor<8x!tf_type.variant> +} + +// ----- + +// Check that merge data inputs of resource type are broadcastable to the output +func.func @invalid_merge(%arg0: tensor<*x!tf_type.resource>, %arg1: tensor<4x!tf_type.resource>) -> tensor<8x!tf_type.resource> { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<*x!tf_type.resource>, tensor<4x!tf_type.resource>) -> (tensor<8x!tf_type.resource>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op expects all operands to be broadcastable with output type but got 'tensor<4x!tf_type.resource>' vs 'tensor<8x!tf_type.resource>'}} + tf_executor.fetch %value : tensor<8x!tf_type.resource> + } + func.return %result : tensor<8x!tf_type.resource> +} + +// ----- + +// Check that if result is a ref type, all operands need to be ref too. +func.func @invalid_merge(%arg0: tensor<4x!tf_type.f32ref>, %arg1: tensor<4xf32>) -> tensor<4x!tf_type.f32ref> { + %result = tf_executor.graph { + %value, %idx, %ctlMerge = "tf_executor.Merge"(%arg0, %arg1) : (tensor<4x!tf_type.f32ref>, tensor<4xf32>) -> (tensor<4x!tf_type.f32ref>, tensor, !tf_executor.control) + // expected-error@-1 {{'tf_executor.Merge' op expects same operand and output element type but got 'tensor<4xf32>' vs 'tensor<4x!tf_type.f32ref>'}} + tf_executor.fetch %value : tensor<4x!tf_type.f32ref> + } + func.return %result : tensor<4x!tf_type.f32ref> +} + +// ----- + +// Check that merge data inputs can't appear after control input. +func.func @invalid_merge(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<*xf32> { + %result = tf_executor.graph { + %true, %false, %ctlSwitch = tf_executor.Switch %arg0, %arg1 : tensor<*xf32> + + %value, %idx, %ctlMerge = "tf_executor.Merge"(%true, %ctlSwitch, %false) : (tensor<*xf32>, !tf_executor.control, tensor<*xf32>) -> (tensor<*xf32>, tensor, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Merge' op failed to verify that all control inputs must appear after any non-control input}} +// expected-error@-2 {{'tf_executor.Merge' op found non-control operand #2 after control operand}} + tf_executor.fetch %value : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.Enter parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>) { + "tf.some_op"() ({ + %res:2 = tf_executor.Enter %arg0 frame "some/fra\"me" : tensor<*xf32> +// expected-error@-1 {{'tf_executor.Enter' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that Enter return value is the same type as the input. +func.func @invalid_enter(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %result = tf_executor.graph { + %res:2 = "tf_executor.Enter"(%arg1) { frame_name = "some/fra\"me"} : (i1) -> (tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Enter' op failed to verify that data operand must be broadcastable to result}} + tf_executor.fetch %res#0 : tensor<*xf32> + } + func.return %result : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.NextIteration.Sink parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>, %arg1: !tf_executor.token) { + "tf.some_op"() ({ + tf_executor.NextIteration.Sink[%arg1] %arg0 : tensor<*xf32> +// expected-error@-1 {{'tf_executor.NextIteration.Sink' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that a tf_executor.NextIteration.Source parent is a graph. +func.func @parent_is_graph() { + "tf.some_op"() ({ + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> +// expected-error@-1 {{'tf_executor.NextIteration.Source' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +func.func @invalid_nextiteration(%arg0: tensor<*xf32>, %arg1: !tf_executor.token) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> +// expected-error@-1 {{'tf_executor.NextIteration.Source' op expects a single user for produced token}} + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @invalid_nextiteration(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> +// expected-error@-1 {{'tf_executor.NextIteration.Source' op token should be consumed by a sink op}} + tf_executor.island { + "tf.consume_token"(%1#1) : (!tf_executor.token) -> () + } + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @invalid_nextiteration(%arg0: tensor<*xf32>, %arg1: !tf_executor.token) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + %2 = "tf.produce_token"() : () -> (!tf_executor.token) + tf_executor.yield %2 : !tf_executor.token + } + "tf_executor.NextIteration.Sink"(%1#0, %arg0) : (!tf_executor.token, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.NextIteration.Sink' op expects a token produced by a tf_executor.NextIteration.Source op}} + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @invalid_nextiteration(%arg0: tensor<*xf32>, %arg1: !tf_executor.token) -> tensor<*xf32> { + %0 = tf_executor.graph { + "tf_executor.NextIteration.Sink"(%arg1, %arg0) : (!tf_executor.token, tensor<*xf32>) -> () +// expected-error@-1 {{'tf_executor.NextIteration.Sink' op expects a token directly produced by a tf_executor.NextIteration.Source op}} + tf_executor.fetch %arg0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +func.func @invalid_nextiteration(%arg0: tensor<*xf32>, %arg1: i1) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:3 = tf_executor.NextIteration.Source : tensor<*xf32> + "tf_executor.NextIteration.Sink"(%1#1, %arg1) : (!tf_executor.token, i1) -> () +// expected-error@-1 {{'tf_executor.NextIteration.Sink' op input type 'i1' mismatch the tf_executor.NextIteration.Source output type: 'tensor<*xf32>'}} + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.Exit parent is a graph. +func.func @parent_is_graph(%arg0: tensor<*xf32>) { + "tf.some_op"() ({ + %1:2 = tf_executor.Exit %arg0 : tensor<*xf32> +// expected-error@-1 {{'tf_executor.Exit' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +func.func @exit(%arg0: tensor<*xi32>) -> tensor<*xf32> { + %0 = tf_executor.graph { + %1:2 = "tf_executor.Exit"(%arg0) : (tensor<*xi32>) -> (tensor<*xf32>, !tf_executor.control) +// expected-error@-1 {{'tf_executor.Exit' op failed to verify that data operand must be broadcastable to result}} + tf_executor.fetch %1#0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> +} + +// ----- + +// Check that a tf_executor.ControlTrigger parent is a graph. +func.func @parent_is_graph(%arg0: !tf_executor.control, %arg1: !tf_executor.control) { + "tf.some_op"() ({ + %0 = tf_executor.ControlTrigger %arg0, %arg1 +// expected-error@-1 {{'tf_executor.ControlTrigger' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} + +// ----- + +// Check that a tf_executor.LoopCond parent is a graph. +func.func @parent_is_graph(%arg0: tensor, %arg1: !tf_executor.control) { + "tf.some_op"() ({ + %1:2 = tf_executor.LoopCond %arg0, %arg1 : tensor +// expected-error@-1 {{'tf_executor.LoopCond' op expects parent op 'tf_executor.graph'}} + }) : () -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_location_roundtrip.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_location_roundtrip.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6791d1c90690be6e70aa3e552ea5d3e5db920e50 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_location_roundtrip.mlir @@ -0,0 +1,67 @@ +// RUN: tf-opt %s -mlir-print-debuginfo -mlir-print-local-scope | tf-opt -mlir-print-debuginfo -mlir-print-op-generic -mlir-print-local-scope | FileCheck %s + +// This file should be written in the generic form with debug locations. +// (that is, as if printed with `-mlir-print-debuginfo -mlir-print-op-generic`). +// The test parses the file, prints it in the pretty form with debug locations, +// then parses it back, then prints it in the generic form again. +// This should be an exact roundtrip. +// This is a rare example of a test where exact output checking makes +// sense. +// +// To debug this test, it is most useful to look at the output of the first +// tf-opt invocation. + + + +// This test case exercises the "tf_executor.island wraps" syntax. +// When parsing it back, we should recover all 3 locations (the +// tf_executor.island, tf.Identity, and tf_executor.yield). + +// CHECK-LABEL: "func.func" +// CHECK: sym_name = "island_one_op_all_locs_same" +// CHECK: "tf_executor.graph"() ({ +// CHECK-NEXT: "tf_executor.island"() ({ +// CHECK-NEXT: "tf.Identity"(%{{.*}}) : (tensor) -> tensor loc("identity@some_function") +// CHECK-NEXT: "tf_executor.yield"(%{{.*}}) : (tensor) -> () loc("identity@some_function") +// CHECK-NEXT: }) : () -> (tensor, !tf_executor.control) loc("identity@some_function") +// CHECK-NEXT: "tf_executor.fetch"(%{{.*}}) : (tensor) -> () loc(unknown) +// CHECK-NEXT: }) : () -> tensor loc(unknown) +// CHECK-NEXT: "func.return"(%{{.*}}) : (tensor) -> () loc(unknown) + +func.func @island_one_op_all_locs_same(%arg0: tensor) -> tensor { + %0 = "tf_executor.graph"() ({ + %1:2 = "tf_executor.island"() ({ + %2 = "tf.Identity"(%arg0) : (tensor) -> tensor loc("identity@some_function") + "tf_executor.yield"(%2) : (tensor) -> () loc("identity@some_function") + }) : () -> (tensor, !tf_executor.control) loc("identity@some_function") + "tf_executor.fetch"(%1#0) : (tensor) -> () loc(unknown) + }) : () -> tensor loc(unknown) + "func.return"(%0) : (tensor) -> () loc(unknown) +} loc(unknown) + +// This test cases exercises our handling of the "tf_executor.island wraps" +// syntax. In particular, that syntax only prints out a single location, so +// it is incorrect to use that syntax if the island, wrapped op, and yield +// don't have identical locations. + +// CHECK-LABEL: "func.func" +// CHECK: sym_name = "island_one_op_all_locs_NOT_same" +// CHECK: "tf_executor.graph"() ({ +// CHECK-NEXT: "tf_executor.island"() ({ +// CHECK-NEXT: "tf.Identity"(%{{.*}}) : (tensor) -> tensor loc("identity@some_function") +// CHECK-NEXT: "tf_executor.yield"(%{{.*}}) : (tensor) -> () loc("identity@some_function") +// CHECK-NEXT: }) : () -> (tensor, !tf_executor.control) loc("NOT_identity@some_function") +// CHECK-NEXT: "tf_executor.fetch"(%{{.*}}) : (tensor) -> () loc(unknown) +// CHECK-NEXT: }) : () -> tensor loc(unknown) +// CHECK-NEXT: "func.return"(%{{.*}}) : (tensor) -> () loc(unknown) + +func.func @island_one_op_all_locs_NOT_same(%arg0: tensor) -> tensor { + %0 = "tf_executor.graph"() ({ + %1:2 = "tf_executor.island"() ({ + %2 = "tf.Identity"(%arg0) : (tensor) -> tensor loc("identity@some_function") + "tf_executor.yield"(%2) : (tensor) -> () loc("identity@some_function") + }) : () -> (tensor, !tf_executor.control) loc("NOT_identity@some_function") + "tf_executor.fetch"(%1#0) : (tensor) -> () loc(unknown) + }) : () -> tensor loc(unknown) + "func.return"(%0) : (tensor) -> () loc(unknown) +} loc(unknown) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_printer.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_printer.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7aa1237f11ec0dda1315e5b859880e84c4e3b976 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_printer.mlir @@ -0,0 +1,73 @@ +// RUN: tf-opt %s | tf-opt | FileCheck %s + +// Tests printer for tf_executor.island "wraps" short form. + +// CHECK-LABEL: func @island_wrap_print +func.func @island_wrap_print(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island wraps "tf.IdentityN" + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) loc("identity@some_function") + tf_executor.yield %1#0, %1#1 : tensor, tensor loc("identity@some_function") + } loc("identity@some_function") + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @island_no_wrap_print_mismatched_results +func.func @island_no_wrap_print_mismatched_results(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + // CHECK-NOT: wraps + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) loc("identity@some_function") + tf_executor.yield %1#1, %1#0 : tensor, tensor loc("identity@some_function") + } loc("identity@some_function") + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @island_no_wrap_print_mismatched_op_location +func.func @island_no_wrap_print_mismatched_op_location(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + // CHECK-NOT: wraps + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) loc(unknown) + tf_executor.yield %1#0, %1#1 : tensor, tensor loc("identity@some_function") + } loc("identity@some_function") + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @island_no_wrap_print_mismatched_yield_location +func.func @island_no_wrap_print_mismatched_yield_location(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + // CHECK-NOT: wraps + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) loc("identity@some_function") + tf_executor.yield %1#0, %1#1 : tensor, tensor loc(unknown) + } loc("identity@some_function") + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @island_no_wrap_print_multiple_ops +func.func @island_no_wrap_print_multiple_ops(%arg0: tensor, %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + // CHECK-NOT: wraps + %0:3 = tf_executor.island { + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) loc("identity@some_function") + %2:2 = "tf.IdentityN"(%1#0, %1#1) : (tensor, tensor) -> (tensor, tensor) loc("identity@some_function") + tf_executor.yield %2#0, %2#1 : tensor, tensor loc("identity@some_function") + } loc("identity@some_function") + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_side_effect.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_side_effect.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f6c2bbce78b0b6b5b76da6bcd60c4d87635c918f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops_side_effect.mlir @@ -0,0 +1,19 @@ +// RUN: tf-opt %s -cse | FileCheck %s +// Checks that CSE runs without generating invalid IR and doesn't CSE ops like +// NextIteration.Source and NextIteration.Sink. +// CHECK-LABEL: func @main +// CHECK: tf_executor.NextIteration.Source +// CHECK: tf_executor.NextIteration.Source +// CHECK: tf_executor.NextIteration.Sink +// CHECK: tf_executor.NextIteration.Sink +func.func @main() -> (tensor<1xi32>, tensor<1xi32>) { + %0, %1 = tf_executor.graph { + %output_1, %token_1, %control_1 = tf_executor.NextIteration.Source : tensor<1xi32> {T = i32, device = ""} + %output_2, %token_2, %control_2 = tf_executor.NextIteration.Source : tensor<1xi32> {T = i32, device = ""} + tf_executor.NextIteration.Sink [%token_1] %output_1 : tensor<1xi32> {T = i32, device = ""} + tf_executor.NextIteration.Sink [%token_2] %output_2 : tensor<1xi32> {T = i32, device = ""} + tf_executor.fetch %output_1, %output_2 : tensor<1xi32>, tensor<1xi32> + } + func.return %0, %1 : tensor<1xi32>, tensor<1xi32> +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_optimize.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_optimize.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2583f396f86302d742dfcd3c8eab37cc899e7261 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_optimize.mlir @@ -0,0 +1,134 @@ +// RUN: tf-opt %s -tf-optimize | FileCheck %s + +// CHECK-LABEL: @fuseMulIntoConv2d +func.func @fuseMulIntoConv2d(%arg0: tensor<1x112x112x3xf32>) -> tensor<1x28x23x2xf32> { + %cst0 = arith.constant dense<[[[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], [[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]], [[13.0, 14.0], [15.0, 16.0], [17.0, 18.0]]]]> : tensor<1x3x3x2xf32> + %cst2 = arith.constant dense<[1.0, 2.0]> : tensor<2xf32> + %0 = "tf.Conv2D"(%arg0, %cst0) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", dilations = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 5, 1]} : (tensor<1x112x112x3xf32>, tensor<1x3x3x2xf32>) -> tensor<1x28x23x2xf32> + %1 = "tf.Mul"(%0, %cst2) : (tensor<1x28x23x2xf32>, tensor<2xf32>) -> tensor<1x28x23x2xf32> + + func.return %1 : tensor<1x28x23x2xf32> + // CHECK: %[[CST:.*]] = "tf.Const{{.*}} dense< + // CHECK-SAME: [1.000000e+00, 4.000000e+00], [3.000000e+00, 8.000000e+00], [5.000000e+00, 1.200000e+01] + // CHECK-SAME: [7.000000e+00, 1.600000e+01], [9.000000e+00, 2.000000e+01], [1.100000e+01, 2.400000e+01] + // CHECK-SAME: [1.300000e+01, 2.800000e+01], [1.500000e+01, 3.200000e+01], [1.700000e+01, 3.600000e+01] + // CHECK: %[[CONV:.*]] = "tf.Conv2D"(%arg0, %[[CST]]) <{data_format = "NHWC", dilations = [1, 2, 3, 1], explicit_paddings = [], padding = "SAME", strides = [1, 4, 5, 1], use_cudnn_on_gpu = true}> + // CHECK: return %[[CONV]] : tensor<1x28x23x2xf32> +} + +// CHECK-LABEL: @notfuseMulIntoConv2d +// filter and multiply are not broadcastable +func.func @notfuseMulIntoConv2d(%arg0: tensor<1x112x112x3xf32>) -> tensor<1x28x23x2xf32> { + %cst0 = arith.constant dense<[[[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], [[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]], [[13.0, 14.0], [15.0, 16.0], [17.0, 18.0]]]]> : tensor<1x3x3x2xf32> + %cst2 = arith.constant dense<3.0> : tensor<23x2xf32> + %0 = "tf.Conv2D"(%arg0, %cst0) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", dilations = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 5, 1]} : (tensor<1x112x112x3xf32>, tensor<1x3x3x2xf32>) -> tensor<1x28x23x2xf32> + %1 = "tf.Mul"(%0, %cst2) : (tensor<1x28x23x2xf32>, tensor<23x2xf32>) -> tensor<1x28x23x2xf32> + + func.return %1 : tensor<1x28x23x2xf32> + // CHECK: %cst_0 = arith.constant dense<3.000000e+00> : tensor<23x2xf32> + // CHECK: %0 = "tf.Conv2D"(%arg0, %cst) <{data_format = "NHWC", dilations = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 5, 1]}> {T = "tfdtype$DT_FLOAT"} + // CHECK: %1 = "tf.Mul"(%0, %cst_0) : (tensor<1x28x23x2xf32>, tensor<23x2xf32>) -> tensor<1x28x23x2xf32> + // CHECK: return %1 : tensor<1x28x23x2xf32> +} + + +// CHECK-LABEL: simplifyBroadcastReshape +func.func @simplifyBroadcastReshape(%arg0: tensor<1x8x1x1x1x1x1x18xbf16>) -> tensor<8x6x6x18xbf16> { + %cst_1 = arith.constant dense<[1, 8, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x8x6x1x6x1x1x18xbf16> + %cst_2 = arith.constant dense<[8, 6, 6, 18]> : tensor<4xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<1x8x6x1x6x1x1x18xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + func.return %98 : tensor<8x6x6x18xbf16> + + // CHECK-DAG: %[[CST:.*]] = "tf.Const"() <{value = dense<[8, 1, 1, 18]> : tensor<4xi64>}> : () -> tensor<4xi64> + // CHECK-DAG: %[[CST1:.*]] = "tf.Const"() <{value = dense<[8, 6, 6, 18]> : tensor<4xi64>}> : () -> tensor<4xi64> + // CHECK: %[[RESHAPE:.*]] = "tf.Reshape"(%arg0, %[[CST]]) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<4xi64>) -> tensor<8x1x1x18xbf16> + // CHECK: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[RESHAPE]], %[[CST1]]) : (tensor<8x1x1x18xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + // CHECK: return %[[BROADCAST]] : tensor<8x6x6x18xbf16> +} + +// CHECK-LABEL: simplifyBroadcastReshapeExtraDims +func.func @simplifyBroadcastReshapeExtraDims(%arg0: tensor<1x8x1x1x1x1x1x18xbf16>) -> tensor<7x8x6x6x18xbf16> { + %cst_1 = arith.constant dense<[7, 1, 8, 6, 1, 6, 1, 1, 18]> : tensor<9xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<9xi64>) -> tensor<7x1x8x6x1x6x1x1x18xbf16> + %cst_2 = arith.constant dense<[7, 8, 6, 6, 18]> : tensor<5xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<7x1x8x6x1x6x1x1x18xbf16>, tensor<5xi64>) -> tensor<7x8x6x6x18xbf16> + func.return %98 : tensor<7x8x6x6x18xbf16> + + // CHECK-DAG: %[[CST:.*]] = "tf.Const"() <{value = dense<[1, 8, 1, 1, 18]> : tensor<5xi64>}> : () -> tensor<5xi64> + // CHECK-DAG: %[[CST1:.*]] = "tf.Const"() <{value = dense<[7, 8, 6, 6, 18]> : tensor<5xi64>}> : () -> tensor<5xi64> + // CHECK: %[[RESHAPE:.*]] = "tf.Reshape"(%arg0, %[[CST]]) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<5xi64>) -> tensor<1x8x1x1x18xbf16> + // CHECK: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[RESHAPE]], %[[CST1]]) : (tensor<1x8x1x1x18xbf16>, tensor<5xi64>) -> tensor<7x8x6x6x18xbf16> + // CHECK: return %[[BROADCAST]] : tensor<7x8x6x6x18xbf16> +} + +// CHECK-LABEL: simplifyBroadcastReshapeOnes +func.func @simplifyBroadcastReshapeOnes(%arg0: tensor<1x1x1x1x1x1x1x18xbf16>) -> tensor<1x6x1x6x18xbf16> { + %cst_1 = arith.constant dense<[1, 1, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x1x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x1x6x1x6x1x1x18xbf16> + %cst_2 = arith.constant dense<[1, 6, 1, 6, 18]> : tensor<5xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<1x1x6x1x6x1x1x18xbf16>, tensor<5xi64>) -> tensor<1x6x1x6x18xbf16> + func.return %98 : tensor<1x6x1x6x18xbf16> + + // CHECK-DAG: %[[CST:.*]] = "tf.Const"() <{value = dense<[1, 1, 1, 1, 18]> : tensor<5xi64>}> : () -> tensor<5xi64> + // CHECK-DAG: %[[CST1:.*]] = "tf.Const"() <{value = dense<[1, 6, 1, 6, 18]> : tensor<5xi64>}> : () -> tensor<5xi64> + // CHECK: %[[RESHAPE:.*]] = "tf.Reshape"(%arg0, %[[CST]]) : (tensor<1x1x1x1x1x1x1x18xbf16>, tensor<5xi64>) -> tensor<1x1x1x1x18xbf16> + // CHECK: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%[[RESHAPE]], %[[CST1]]) : (tensor<1x1x1x1x18xbf16>, tensor<5xi64>) -> tensor<1x6x1x6x18xbf16> + // CHECK: return %[[BROADCAST]] : tensor<1x6x1x6x18xbf16> +} + +// CHECK-LABEL: avoidSimplifyBroadcastReshape +func.func @avoidSimplifyBroadcastReshape(%arg0: tensor<1x8x1x1x1x1x1x18xbf16>) -> (tensor<1x8x6x1x6x1x1x18xbf16>, tensor<8x6x6x18xbf16>) { + %cst_1 = arith.constant dense<[1, 8, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x8x6x1x6x1x1x18xbf16> + %cst_2 = arith.constant dense<[8, 6, 6, 18]> : tensor<4xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<1x8x6x1x6x1x1x18xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + func.return %97, %98 : tensor<1x8x6x1x6x1x1x18xbf16>, tensor<8x6x6x18xbf16> + + // CHECK-DAG: %[[CST:.*]] = arith.constant dense<[1, 8, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + // CHECK-DAG: %[[CST1:.*]] = arith.constant dense<[8, 6, 6, 18]> : tensor<4xi64> + // CHECK: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%arg0, %[[CST]]) : (tensor<1x8x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x8x6x1x6x1x1x18xbf16> + // CHECK: %[[RESHAPE:.*]] = "tf.Reshape"(%[[BROADCAST]], %[[CST1]]) : (tensor<1x8x6x1x6x1x1x18xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + // CHECK: return %[[BROADCAST]], %[[RESHAPE]] : tensor<1x8x6x1x6x1x1x18xbf16>, tensor<8x6x6x18xbf16> +} + +// CHECK-LABEL: avoidSimplifyBroadcastReshapeUnmatchedDims +// The reshape splits broadcasted dimensions, instead of eliminating size-1 dimensions. +// This results in a mismatch between the non-unit dimensions in the input and output. +func.func @avoidSimplifyBroadcastReshapeUnmatchedDims(%arg0: tensor<1x1x1x1x1x1x1x18xbf16>) -> tensor<1x3x2x1x3x2x18xbf16> { + %cst_1 = arith.constant dense<[1, 1, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x1x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x1x6x1x6x1x1x18xbf16> + %cst_2 = arith.constant dense<[1, 3, 2, 1, 3, 2, 18]> : tensor<7xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<1x1x6x1x6x1x1x18xbf16>, tensor<7xi64>) -> tensor<1x3x2x1x3x2x18xbf16> + func.return %98 : tensor<1x3x2x1x3x2x18xbf16> + + // CHECK-DAG: %[[CST:.*]] = arith.constant dense<[1, 1, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + // CHECK-DAG: %[[CST1:.*]] = arith.constant dense<[1, 3, 2, 1, 3, 2, 18]> : tensor<7xi64> + // CHECK: %[[BROADCAST:.*]] = "tf.BroadcastTo"(%arg0, %[[CST]]) : (tensor<1x1x1x1x1x1x1x18xbf16>, tensor<8xi64>) -> tensor<1x1x6x1x6x1x1x18xbf16> + // CHECK: %[[RESHAPE:.*]] = "tf.Reshape"(%[[BROADCAST]], %[[CST1]]) : (tensor<1x1x6x1x6x1x1x18xbf16>, tensor<7xi64>) -> tensor<1x3x2x1x3x2x18xbf16> + // CHECK: return %[[RESHAPE]] : tensor<1x3x2x1x3x2x18xbf16> +} + +// CHECK-LABEL: avoidSimplifyBroadcastReshapeUnknownDims +func.func @avoidSimplifyBroadcastReshapeUnknownDims(%arg0: tensor<1x?x1x1x1x1x1x?xbf16>) -> tensor<8x6x6x18xbf16> { + %cst_1 = arith.constant dense<[1, -1, 6, 1, 6, 1, 1, -1]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<1x?x1x1x1x1x1x?xbf16>, tensor<8xi64>) -> tensor<1x?x6x1x6x1x1x?xbf16> + %cst_2 = arith.constant dense<[8, 6, 6, 18]> : tensor<4xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<1x?x6x1x6x1x1x?xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + func.return %98 : tensor<8x6x6x18xbf16> + + // CHECK: "tf.BroadcastTo" + // CHECK: "tf.Reshape" +} + +// CHECK-LABEL: avoidSimplifyBroadcastReshapeUnknownRanks +func.func @avoidSimplifyBroadcastReshapeUnknownRanks(%arg0: tensor<*xbf16>) -> tensor<8x6x6x18xbf16> { + %cst_1 = arith.constant dense<[1, 8, 6, 1, 6, 1, 1, 18]> : tensor<8xi64> + %97 = "tf.BroadcastTo"(%arg0, %cst_1) : (tensor<*xbf16>, tensor<8xi64>) -> tensor<*xbf16> + %cst_2 = arith.constant dense<[8, 6, 6, 18]> : tensor<4xi64> + %98 = "tf.Reshape"(%97, %cst_2) : (tensor<*xbf16>, tensor<4xi64>) -> tensor<8x6x6x18xbf16> + func.return %98 : tensor<8x6x6x18xbf16> + + // CHECK: "tf.BroadcastTo" + // CHECK: "tf.Reshape" +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..6c8ef18852d7d79b44a0f43d4229ae924a565041 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/BUILD @@ -0,0 +1,339 @@ +load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary") +load("@xla//third_party/rules_python/python:py_library.bzl", "py_library") +load("//tensorflow:tensorflow.default.bzl", "filegroup") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +py_library( + name = "common", + srcs = ["common.py"], + strict_deps = True, + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/python:pywrap_mlir", + "//tensorflow/python/lib/io:file_io", + "@absl_py//absl:app", + "@absl_py//absl/flags", + "@absl_py//absl/logging", + ], +) + +py_library( + name = "common_v1", + srcs = ["common_v1.py"], + strict_deps = True, + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/python:pywrap_mlir", + "@absl_py//absl:app", + "@absl_py//absl/flags", + "@absl_py//absl/logging", + ], +) + +py_binary( + name = "basic", + testonly = 1, + srcs = ["basic.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "cyclic_object_graph", + srcs = ["cyclic_object_graph.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "multi_variables_v1", + srcs = ["multi_variables_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "import_restore_v1", + srcs = ["import_restore_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "shapes_for_arguments", + srcs = ["shapes_for_arguments.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "control_flow_upgrade_legacy_v1", + srcs = ["control_flow_upgrade_legacy_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + "//tensorflow/python/ops:control_flow_ops", + ], +) + +py_binary( + name = "exported_python_args", + srcs = ["exported_python_args.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "basic_v1_no_variable_lifting", + srcs = ["basic_v1_no_variable_lifting.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "multi_arguments_results_v1", + srcs = ["multi_arguments_results_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + "//tensorflow/python/ops:array_ops", + ], +) + +py_binary( + name = "no_input_shape_v1", + srcs = ["no_input_shape_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + "//tensorflow/core:protos_all_py", + ], +) + +py_binary( + name = "structured_input", + srcs = ["structured_input.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "defun_export", + srcs = ["defun_export.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + "//tensorflow/python/framework:function", + ], +) + +py_binary( + name = "basic_v1", + srcs = ["basic_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "duplicate_method_names_v1", + srcs = ["duplicate_method_names_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "debug_info", + srcs = ["debug_info.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "keras", + srcs = ["keras.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "hash_table_v1", + srcs = ["hash_table_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "include_variables_in_init_v1", + srcs = ["include_variables_in_init_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "hash_table_asset_v1", + srcs = ["hash_table_asset_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "control_flow_duplicate_v1", + srcs = ["control_flow_duplicate_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "shared_variable_v1", + srcs = ["shared_variable_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "dag_object_graph", + srcs = ["dag_object_graph.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "call_to_exported", + srcs = ["call_to_exported.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "remove_init_variable_v1", + srcs = ["remove_init_variable_v1.py"], + strict_deps = True, + deps = [ + ":common_v1", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "partially_shaped_variables", + srcs = ["partially_shaped_variables.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "structured_output", + srcs = ["structured_output.py"], + strict_deps = True, + deps = [ + ":common", + "//tensorflow:tensorflow_py", + ], +) + +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "@llvm-project//llvm:FileCheck", + ], +) + +test_files = glob( + ["*.py"], + exclude = [ + "common.py", + "common_v1.py", + ], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + default_size = "medium", + default_tags = [ + "no_mac", # TODO(b/191167848) + "no_oss", # TODO(b/190855110) + "no_rocm", + ], + driver = "@llvm-project//mlir:run_lit.sh", + exclude = [ + "common.py", + "common_v1.py", + ], + per_test_extra_data = { + file: [file[:-3]] + for file in test_files + }, + test_file_exts = ["py"], + use_lit_test_suite = False, # Each test gets a large binary, and is too big when consolidated. +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/README.md b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/README.md new file mode 100644 index 0000000000000000000000000000000000000000..18612760a511c8bb0dc8ebf473ccd9540be505ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/README.md @@ -0,0 +1,48 @@ +# SavedModel importer FileCheck tests. + +## Debugging tests + +While debugging tests, the following commands are handy. + +Run FileCheck test: + +``` +bazel run :foo.py.test +``` + +Run just the Python file and look at the output: + +``` +bazel run :foo +``` + +Generate saved model to inspect proto: + +``` +bazel run :foo -- --save_model_path=/tmp/my.saved_model +# Inspect /tmp/my.saved_model/saved_model.pb +``` + +## Rationale for Python-based tests + +For a SavedModel importer, the natural place to start is to feed in the +SavedModel format directly and test the output MLIR. We don't do that though. + +The SavedModel format is a directory structure which contains a SavedModel proto +and some other stuff (mostly binary files of some sort) in it. That makes it not +suitable for use as a test input, since it is not human-readable. Even just the +text proto for the SavedModel proto is difficult to use as a test input, since a +small piece of Python code (e.g. just a tf.Add) generates thousands of lines of +text proto. + +That points to a solution though: write our tests starting from the Python API's +that generate the SavedModel. That leads to very compact test inputs. + +As the SavedModel work progresses, it's likely to be of interest to find a +shortcut between the Python `tf.Module` and the SavedModel MLIR representation +that doesn't involve serializing a SavedModel to disk and reading it back. + +## Potential improvements + +The test iteration cycle for these tests is very long (usually over a minute). +We need to find a way to improve this in the future. diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic.py new file mode 100644 index 0000000000000000000000000000000000000000..2829a77685e3a0f55324e56eb7dd2cd5277b6512 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic.py @@ -0,0 +1,57 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/basic | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + + +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + self.v42 = tf.Variable(42.0) + self.c43 = tf.constant(43.0) + + # During serialization, the constants are given internal (non-user-accessible, non-semantically-load-bearing) exported names. + # CHECK: "tf_saved_model.global_tensor"() <{sym_name = "[[CONST:[a-zA-Z_0-9.]+]]", type = tensor, value = dense<4.300000e+01> : tensor}> {tf_saved_model.exported_names = [{{.*}}]} : () -> () + + # CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR:[a-zA-Z_0-9]+]]", type = tensor, value = dense<4.200000e+01> : tensor}> {tf_saved_model.exported_names = ["v42"]} : () -> () + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor {tf._user_specified_name = "x", tf_saved_model.index_path = [0]}, + # CHECK-SAME: %arg1: tensor>> + # CHECK-SAME: %arg2: tensor>> + # CHECK-SAME: tensor {tf_saved_model.index_path = []}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["some_function"] + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def some_function(self, x): + return x + self.v42 + self.c43 + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..8082a692bcb2d357b7d807957247037876bf24a0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1.py @@ -0,0 +1,68 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/basic_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + +# CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR:[a-zA-Z_0-9]+]]", type = tensor<1x3xf32>, value = {{.*}} : tensor<1x3xf32>}> : () -> () + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG0:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]}, +# CHECK-SAME: [[ARG1:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR]]}) +# CHECK-SAME: -> (tensor<3x3xf32> {tf_saved_model.index_path = ["r"]}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] + +# CHECK-NEXT: [[R0:%.*]] = "tf.ReadVariableOp"([[ARG1]]) {{{.*}}} : (tensor>>) -> tensor<1x3xf32> +# CHECK-NEXT: [[R1:%.*]] = "tf.MatMul"([[ARG0]], [[R0]]) <{{{.*}}}> {device = ""} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> +# CHECK-NEXT: return [[R1]] : tensor<3x3xf32> + + +def Test(): + + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.compat.v1.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + r = tf.matmul(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1_no_variable_lifting.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1_no_variable_lifting.py new file mode 100644 index 0000000000000000000000000000000000000000..659fcf0fd876ff955d4f11cf024ff8d9c945881a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/basic_v1_no_variable_lifting.py @@ -0,0 +1,53 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/basic_v1_no_variable_lifting | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-NOT: tf_saved_model.global_tensor + + +def Test(): + + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.compat.v1.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + r = tf.matmul(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test, lift_variables=False) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/call_to_exported.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/call_to_exported.py new file mode 100644 index 0000000000000000000000000000000000000000..0ea79f65a96f9543ee197cb76c82a41147cd2279 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/call_to_exported.py @@ -0,0 +1,74 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/call_to_exported | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + self.v = tf.Variable(42.0) + + # We guarantee that there are no calls to exported functions from inside the + # module. + # + # If there is a call to an exported function, we create a wrapper function + # that forwards to the other function and put the tf_saved_model attributes on + # the wrapper. + # + # The reason for doing this is so that later interprocedural passes don't have + # to worry about what to do with these attributes. + # An example of where this would happen is when converting to XLA, which + # requires eliminating mutable variables (and is thus sort of like an + # interprocedural SSA formation, which in particular will + # modify signatures interprocedurally). + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor {tf._user_specified_name = "x", tf_saved_model.index_path = [0]}, + # CHECK-SAME: %arg1: tensor> + # CHECK-SAME: ) -> ( + # CHECK-SAME: tensor {tf_saved_model.index_path = [0]}, + # CHECK-SAME: tensor {tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes{{.*}}tf_saved_model.exported_names = ["callee"] + # CHECK: "tf.StatefulPartitionedCall"{{.*}}f = @[[CALLEE_INTERNAL:[a-zA-Z_0-9]+]] + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor {tf._user_specified_name = "x", tf_saved_model.index_path = [0]}, + # CHECK-SAME: %arg1: tensor> + # CHECK-SAME: ) -> ( + # CHECK-SAME: tensor {tf_saved_model.index_path = [0]}, + # CHECK-SAME: tensor<*xf32> {tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes{{.*}}tf_saved_model.exported_names = ["caller"] + # CHECK: "tf.StatefulPartitionedCall"{{.*}}f = @[[CALLEE_INTERNAL]] + # + # CHECK: func private @[[CALLEE_INTERNAL]] + # CHECK-NOT: tf_saved_model.exported_names + + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def callee(self, x): + return x, self.v + + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def caller(self, x): + return self.callee(x) + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common.py new file mode 100644 index 0000000000000000000000000000000000000000..9d5dae8af6518da6328e2daf29a00f4ecb86eb75 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common.py @@ -0,0 +1,98 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Serves as a common "main" function for all the SavedModel tests. + +There is a fair amount of setup needed to initialize tensorflow and get it +into a proper TF2 execution mode. This hides that boilerplate. +""" + +import tempfile + +from absl import app +from absl import flags +from absl import logging +import tensorflow.compat.v2 as tf +from tensorflow.python import pywrap_mlir # pylint: disable=g-direct-tensorflow-import +from tensorflow.python.lib.io import file_io + +# Use /tmp to make debugging the tests easier (see README.md) +flags.DEFINE_string('save_model_path', '', 'Path to save the model to.') +FLAGS = flags.FLAGS + + +# This function needs to take a "create_module_fn", as opposed to just the +# module itself, because the creation of the module has to be delayed until +# after absl and tensorflow have run various initialization steps. +def do_test(create_module_fn, exported_names=None, show_debug_info=False): + """Runs test. + + 1. Performs absl and tf "main"-like initialization that must run before almost + anything else. + 2. Converts `tf.Module` to SavedModel + 3. Converts SavedModel to MLIR + 4. Prints the textual MLIR to stdout (it is expected that the caller will have + FileCheck checks in its file to check this output). + + This is only for use by the MLIR SavedModel importer tests. + + Args: + create_module_fn: A callable taking no arguments, which returns the + `tf.Module` to be converted and printed. + exported_names: A set of exported names for the MLIR converter (default is + "export all"). + show_debug_info: If true, shows debug locations in the resulting MLIR. + """ + if exported_names is None: + exported_names = [] + + # Make LOG(ERROR) in C++ code show up on the console. + # All `Status` passed around in the C++ API seem to eventually go into + # `LOG(ERROR)`, so this makes them print out by default. + logging.set_stderrthreshold('error') + + # In true TF2 releases, v2 behavior is enabled as part of module __init__. In + # TF1 builds, it must be enabled manually. If you get an error here, + # it means that TF was used in V1 mode prior to calling this. + tf.enable_v2_behavior() + + def app_main(argv): + """Function passed to absl.app.run.""" + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + if FLAGS.save_model_path: + save_model_path = FLAGS.save_model_path + else: + save_model_path = tempfile.mkdtemp(suffix='.saved_model') + save_options = tf.saved_model.SaveOptions(save_debug_info=show_debug_info) + tf.saved_model.save( + create_module_fn(), save_model_path, options=save_options + ) + logging.info('Saved model to: %s', save_model_path) + mlir = pywrap_mlir.experimental_convert_saved_model_to_mlir( + save_model_path, ','.join(exported_names), show_debug_info + ) + # We don't strictly need this, but it serves as a handy sanity check + # for that API, which is otherwise a bit annoying to test. + # The canonicalization shouldn't affect these tests in any way. + mlir = pywrap_mlir.experimental_run_pass_pipeline( + mlir, 'canonicalize', show_debug_info + ) + print(mlir) + filename = '%s/result.mlirbc' % save_model_path + pywrap_mlir.experimental_write_bytecode(filename, mlir) + if not file_io.file_exists(filename): + raise app.UsageError('Failed to create bytecode output.') + + app.run(app_main) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..6a46e23b89b2eb8f99bc2ef5f4cbc38c6916b3c8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/common_v1.py @@ -0,0 +1,142 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Serves as a common "main" function for all the SavedModel tests. + +There is a fair amount of setup needed to initialize tensorflow and get it +into a proper TF2 execution mode. This hides that boilerplate. +""" + +import tempfile +from absl import app +from absl import flags +from absl import logging +import tensorflow.compat.v1 as tf + +from tensorflow.python import pywrap_mlir # pylint: disable=g-direct-tensorflow-import + +# Use /tmp to make debugging the tests easier (see README.md) +flags.DEFINE_string('save_model_path', '', 'Path to save the model to.') +FLAGS = flags.FLAGS + + +def set_tf_options(): + # Default TF1.x uses reference variables that are not supported by SavedModel + # v1 Importer. To use SavedModel V1 Importer, resource variables should be + # enabled. + tf.enable_resource_variables() + tf.compat.v1.disable_eager_execution() + + +# This function needs to take a "create_module_fn", as opposed to just the +# module itself, because the creation of the module has to be delayed until +# after absl and tensorflow have run various initialization steps. +def do_test( + create_signature, + canonicalize=False, + show_debug_info=False, + use_lite=False, + lift_variables=True, + include_variables_in_initializers=False, +): + """Runs test. + + 1. Performs absl and tf "main"-like initialization that must run before almost + anything else. + 2. Converts signature_def_map to SavedModel V1 + 3. Converts SavedModel V1 to MLIR + 4. Prints the textual MLIR to stdout (it is expected that the caller will have + FileCheck checks in its file to check this output). + + This is only for use by the MLIR SavedModel importer tests. + + Args: + create_signature: A functor that return signature_def_map, init_op and + assets_collection. signature_def_map is a map from string key to + signature_def. The key will be used as function name in the resulting + MLIR. + canonicalize: If true, canonicalizer will be run on the resulting MLIR. + show_debug_info: If true, shows debug locations in the resulting MLIR. + use_lite: If true, importer will not do any graph transformation such as + lift variables. + lift_variables: If false, no variable lifting will be done on the graph. + include_variables_in_initializers: If false, removes variables in + initializer functions before lifting variables or adding new variable + initialization patterns in the initializer function. + """ + + # Make LOG(ERROR) in C++ code show up on the console. + # All `Status` passed around in the C++ API seem to eventually go into + # `LOG(ERROR)`, so this makes them print out by default. + logging.set_stderrthreshold('error') + + def app_main(argv): + """Function passed to absl.app.run.""" + if len(argv) > 1: + raise app.UsageError('Too many command-line arguments.') + if FLAGS.save_model_path: + save_model_path = FLAGS.save_model_path + else: + save_model_path = tempfile.mktemp(suffix='.saved_model') + + signature_def_map, init_op, assets_collection = create_signature() + + sess = tf.Session() + sess.run(tf.initializers.global_variables()) + builder = tf.saved_model.builder.SavedModelBuilder(save_model_path) + builder.add_meta_graph_and_variables( + sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map, + main_op=init_op, + assets_collection=assets_collection, + strip_default_attrs=True, + ) + builder.save() + + logging.info('Saved model to: %s', save_model_path) + exported_names = '' + upgrade_legacy = True + if use_lite: + mlir = pywrap_mlir.experimental_convert_saved_model_v1_to_mlir_lite( + save_model_path, + exported_names, + ','.join([tf.saved_model.tag_constants.SERVING]), + upgrade_legacy, + show_debug_info, + ) + # We don't strictly need this, but it serves as a handy sanity check + # for that API, which is otherwise a bit annoying to test. + # The canonicalization shouldn't affect these tests in any way. + mlir = pywrap_mlir.experimental_run_pass_pipeline( + mlir, 'tf-standard-pipeline', show_debug_info + ) + else: + mlir = pywrap_mlir.experimental_convert_saved_model_v1_to_mlir( + save_model_path, + exported_names, + ','.join([tf.saved_model.tag_constants.SERVING]), + lift_variables, + include_variables_in_initializers, + upgrade_legacy, + show_debug_info, + ) + + if canonicalize: + mlir = pywrap_mlir.experimental_run_pass_pipeline( + mlir, 'canonicalize', show_debug_info + ) + print(mlir) + + app.run(app_main) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_duplicate_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_duplicate_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..47a93686bfb47fd63e2122b6b72a8a63b2abf91a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_duplicate_v1.py @@ -0,0 +1,57 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/control_flow_duplicate_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Tests handling dupliate functions after V1 control flow is functionalized. + +# CHECK: func {{.*}} tf_saved_model.exported_names = ["key_1"] +# CHECK: "tf.If" +# CHECK-SAME: else_branch = @[[else:[a-zA-Z_0-9]+]] +# CHECK-SAME: then_branch = @[[then:[a-zA-Z_0-9]+]] + +# CHECK: func {{.*}} tf_saved_model.exported_names = ["key_2"] +# CHECK: "tf.If" +# CHECK-SAME: else_branch = @[[else]] +# CHECK-SAME: then_branch = @[[then]] + +# CHECK: func private @[[else]]( +# CHECK: func private @[[then]]( + + +def Test(): + + zero = tf.constant(0) + one = tf.constant(1) + x = tf.placeholder(tf.int32, shape=(), name='input') + result = tf.cond(x > zero, lambda: tf.square(x), lambda: tf.add(x, one)) + + tensor_info_result = tf.compat.v1.saved_model.utils.build_tensor_info(result) + + signature_def = tf.saved_model.signature_def_utils.build_signature_def( + inputs=None, + outputs={'result': tensor_info_result}, + method_name='some_function') + + return {'key_1': signature_def, 'key_2': signature_def}, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_upgrade_legacy_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_upgrade_legacy_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..ff5d8fd7f266bc9d0881ac10b432d199b250a82d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/control_flow_upgrade_legacy_v1.py @@ -0,0 +1,59 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/control_flow_upgrade_legacy_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 +from tensorflow.python.ops import control_flow_ops + +# Tests V1 control flow is functionalized. + +# CHECK-NOT: tf_executor.Switch +# CHECK-NOT: tf_executor.Merge +# CHECK: "tf.If" +# CHECK-SAME: else_branch = @"key/[[else:[a-zA-Z_0-9]+]]" +# CHECK-SAME: then_branch = @"key/[[then:[a-zA-Z_0-9]+]]" + +# CHECK: func private @"key/[[else]]"( +# CHECK-SAME: tf._original_func_name +# CHECK: func private @"key/[[then]]"( +# CHECK-SAME: tf._original_func_name + + +def Test(): + data = tf.constant([1, 2, 3, 4, 5, 6]) + # Create placeholders to prevent constant folding. + x_op = tf.placeholder(dtype=tf.int32) + y_op = tf.placeholder(dtype=tf.int32) + less_op = tf.less(x_op, y_op) + switch_op = control_flow_ops.switch(data, less_op) + merge_op = control_flow_ops.merge(switch_op)[0] + result = tf.transpose(merge_op) + + tensor_info_result = tf.compat.v1.saved_model.utils.build_tensor_info(result) + + signature_def = tf.saved_model.signature_def_utils.build_signature_def( + inputs=None, + outputs={'result': tensor_info_result}, + method_name='some_function') + + return {'key': signature_def}, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/cyclic_object_graph.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/cyclic_object_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..58aac11de1e68224d4beeb455f700a7adc92fdae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/cyclic_object_graph.py @@ -0,0 +1,42 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/cyclic_object_graph | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class ReferencesParent(tf.Module): + + def __init__(self, parent): + super(ReferencesParent, self).__init__() + self.parent = parent + # CHECK: tf_saved_model.global_tensor + # CHECK-SAME: tf_saved_model.exported_names = ["child.my_variable"] + self.my_variable = tf.Variable(3.) + + +# Creates a cyclic object graph. +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + self.child = ReferencesParent(self) + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/dag_object_graph.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/dag_object_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..68241e27aaebfadb3fa1419d0f7e1d2faf6c3302 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/dag_object_graph.py @@ -0,0 +1,45 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/dag_object_graph | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class Child(tf.Module): + + def __init__(self): + super(Child, self).__init__() + self.my_variable = tf.Variable(3.) + + +# Creates a dag object graph. +# There is only one instance of `Child`, but it is reachable via two names. +# Thus, self.my_variable is reachable via two paths. +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + self.child1 = Child() + self.child2 = self.child1 + + # CHECK: tf_saved_model.global_tensor + # CHECK-SAME: tf_saved_model.exported_names = ["child1.my_variable", "child2.my_variable"] + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/debug_info.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/debug_info.py new file mode 100644 index 0000000000000000000000000000000000000000..f36d50fd8b9bc83798e10c91f8fbbc6a814e79e5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/debug_info.py @@ -0,0 +1,40 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/debug_info | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + + @tf.function(input_signature=[ + tf.TensorSpec([], tf.float32), + tf.TensorSpec([], tf.float32) + ]) + def some_function(self, x, y): + return x + y + # Basic check that the debug info file is being correctly saved and loaded. + # + # CHECK: "tf.AddV2"{{.*}}loc(#loc{{[0-9]+}}) + # CHECK: "tf.Identity"{{.*}}loc(#loc{{[0-9]+}}) + # CHECK: #loc{{[0-9]+}} = loc("{{.*}}debug_info.py":{{[0-9]+}}:{{[0-9]+}}) + # CHECK: #loc{{[0-9]+}} = loc(callsite(#loc{{[0-9]+}} at #loc{{[0-9]+}})) + + +if __name__ == '__main__': + common.do_test(TestModule, show_debug_info=True) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/defun_export.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/defun_export.py new file mode 100644 index 0000000000000000000000000000000000000000..5657b4fb1cf1a74b1fe99e8c31c202d8970d248a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/defun_export.py @@ -0,0 +1,59 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/defun_export | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 +from tensorflow.python.framework import function + + +@function.Defun(tf.float32, tf.float32) +def plus(a, b): + return a + b + + +def test_defun(): + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.constant([[2.0], [2.0], [2.0]]) + + # Verify that the function defined using function.Defun + # has a corresponding tf.LegacyCall op. + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: [[ARG0:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["y"]}, + # CHECK-SAME: [[ARG1:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]} + # + # CHECK-NEXT: [[R0:%.*]] = "tf.LegacyCall"([[ARG1]], [[ARG0]]) + z = plus(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_y = tf.compat.v1.saved_model.utils.build_tensor_info(y) + tensor_info_z = tf.compat.v1.saved_model.utils.build_tensor_info(z) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'x': tensor_info_x, + 'y': tensor_info_y + }, + outputs={'z': tensor_info_z}, + method_name='test_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(test_defun) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/duplicate_method_names_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/duplicate_method_names_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..0632c8c68316636de0c53020b1f0c0e21d458145 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/duplicate_method_names_v1.py @@ -0,0 +1,55 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/duplicate_method_names_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Tests different SignatureDef's with identical method_name string + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: {{.*}}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: {{.*}}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key2"] + + +def Test(): + + x = tf.constant(1.0, shape=(3, 3)) + y = tf.constant(1.0, shape=(3, 3)) + + s = tf.transpose(x) + t = tf.transpose(y) + + tensor_info_s = tf.compat.v1.saved_model.utils.build_tensor_info(s) + tensor_info_t = tf.compat.v1.saved_model.utils.build_tensor_info(t) + + signature_def = tf.saved_model.signature_def_utils.build_signature_def( + inputs=None, outputs={'s': tensor_info_s}, method_name='some_function') + signature_def2 = tf.saved_model.signature_def_utils.build_signature_def( + inputs=None, outputs={'t': tensor_info_t}, method_name='some_function') + + # Create two signatures that share the same variable. + return {'key': signature_def, 'key2': signature_def2}, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/exported_python_args.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/exported_python_args.py new file mode 100644 index 0000000000000000000000000000000000000000..1adcfb98410ef1b5c4b84c725dba540411dfb6df --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/exported_python_args.py @@ -0,0 +1,37 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: (! %p/exported_python_args 2>&1) | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long,dangerous-default-value +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def some_function(self, x): + return self.callee(x) + + # CHECK: While importing SavedModel function 'callee': in input signature: + # CHECK-SAME: Unhandled structured value kind {{.*}} at index path: .1.foo + @tf.function + def callee(self, x, n={'foo': 42}): + return x + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..f7176d57364e0f662885446f5a1d175266b90990 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py @@ -0,0 +1,77 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/hash_table_asset_v1| FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import os +import tempfile + +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# CHECK: "tf_saved_model.session_initializer"() <{initializers = [@[[init:.*]]]}> : () -> () +# CHECK: "tf_saved_model.asset"() <{filename = {{.*}}, sym_name = "[[asset1:__tf_saved_model_asset1_.*]]"}> +# CHECK: "tf_saved_model.asset"() <{filename = {{.*}}, sym_name = "[[asset0:__tf_saved_model_asset0_.*]]"}> + +# CHECK: func @[[init]] +# CHECK-SAME: [[ARG0:%.*]]: tensor {tf_saved_model.bound_input = @[[asset0]]} +# CHECK-SAME: [[ARG1:%.*]]: tensor {tf_saved_model.bound_input = @[[asset1]]} +# CHECK-SAME: tf_saved_model.initializer_type = "init_op" +# CHECK-NEXT: [[R0:%.*]] = "tf.HashTableV2"() +# CHECK-SAME: shared_name = "[[hash_table:.*]]" +# CHECK-NEXT: "tf.InitializeTableFromTextFileV2"([[R0]], [[ARG0]]) + + +def write_vocabulary_file(vocabulary): + """Write temporary vocab file for module construction.""" + tmpdir = tempfile.mkdtemp() + vocabulary_file = os.path.join(tmpdir, 'tokens.txt') + with tf.io.gfile.GFile(vocabulary_file, 'w') as f: + for entry in vocabulary: + f.write(entry + '\n') + return vocabulary_file + + +def test(): + + vocabulary_file = write_vocabulary_file(['cat', 'is', 'on', 'the', 'mat']) + table_initializer = tf.lookup.TextFileInitializer( + vocabulary_file, tf.string, tf.lookup.TextFileIndex.WHOLE_LINE, tf.int64, + tf.lookup.TextFileIndex.LINE_NUMBER) + # Incur another bound_input on the asset, but with a different sym_name, i.e., + # __tf_saved_model_asset1_tokens.txt vs. __tf_saved_model_asset0_tokens.txt. + table = tf.lookup.StaticVocabularyTable(table_initializer, num_oov_buckets=10) + vocab_file_tensor = tf.convert_to_tensor( + vocabulary_file, tf.string, name='asset_filepath') + tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file_tensor) + + x = tf.placeholder(tf.string, shape=(), name='input') + r = table.lookup(x) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, tf.tables_initializer(), tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS) + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(test, use_lite=True) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..27fb843aea38f32c94e157650e38309a34c49f85 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_v1.py @@ -0,0 +1,89 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/hash_table_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + +# CHECK: "tf_saved_model.global_tensor"() +# CHECK: "tf_saved_model.session_initializer"() <{initializers = [@[[init:.*]]]}> : () -> () + +# CHECK: func @[[init]] +# CHECK-SAME: tf_saved_model.initializer_type = "init_op" +# CHECK-NEXT: [[R6:%.*]] = "tf.Const"() +# CHECK-NEXT: [[R5:%.*]] = "tf.Const"() <{value = dense<[1, 2, +# CHECK-NEXT: [[R7:%.*]] = "tf.HashTableV2"() +# CHECK-SAME: shared_name = "[[hash_table:.*]]" +# CHECK-NEXT: "tf.LookupTableImportV2"([[R7]], [[R5]], [[R6]]) + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG0:%.*]]: tensor +# CHECK-SAME: [[ARG1:%.*]]: tensor : () -> () + +# CHECK: func @[[restore]]( +# CHECK-SAME: [[variable_path:%.*]]: tensor {tf_saved_model.bound_input = @[[sym_name]]} +# CHECK-SAME: tf_saved_model.exported_names = ["{{__tf_saved_model_session_initializer.*}}"] +# CHECK-SAME: tf_saved_model.initializer_type = "restore_op" +# CHECK: [[handle:%.*]] = "tf.VarHandleOp" +# CHECK-SAME: shared_name = [[shared_name:".*"]] +# CHECK: [[v0:%.*]] = "tf.RestoreV2"([[variable_path]] +# CHECK: [[v1:%.*]] = "tf.Identity"([[v0]]) +# CHECK: "tf.AssignVariableOp"([[handle]], [[v1]]) + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: tf_saved_model.exported_names = ["key"] +# CHECK: tf.VarHandleOp +# CHECK-SAME: shared_name = [[shared_name]] + + +def Test(): + + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.compat.v1.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + r = tf.matmul(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test, use_lite=True) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/include_variables_in_init_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/include_variables_in_init_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..f94ebdcd4f4e4a12bcb1049fdda5d7927d799053 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/include_variables_in_init_v1.py @@ -0,0 +1,87 @@ +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/include_variables_in_init_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + +# CHECK: "tf_saved_model.global_tensor"() +# CHECK: "tf_saved_model.session_initializer"() <{initializers = [@[[INIT_FUNC:[a-zA-Z_0-9]+]]]}> : () -> () + +# Initializer function. This should contain the initialization sequence for the +# variable. +# CHECK: func @[[INIT_FUNC]](%[[ARG_0:.*]]: tensor>> {tf_saved_model.bound_input = @y}) attributes { +# CHECK-SAME: tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_init"] +# CHECK-SAME: tf_saved_model.initializer_type = "init_op" +# CHECK-SAME: } +# CHECK-DAG: %[[CST_0:.*]] = "tf.Const"() <{{{.*dense<.*> : tensor<2xi32>.*}}}> {{{.*}}} : () -> tensor<2xi32> +# CHECK: %[[RAND_STD_NORMAL:.*]] = "tf.RandomStandardNormal"(%[[CST_0]]) +# CHECK: "tf.AssignVariableOp"(%[[ARG_0]], %[[RAND_STD_NORMAL]]){{.*}}: (tensor>>, tensor<1x3xf32>) -> () +# CHECK: return + +# The function for the signature "key". +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: %[[ARG_1:.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]} +# CHECK-SAME: %[[ARG_2:.*]]: tensor>> {tf_saved_model.bound_input = @y} +# CHECK-SAME: -> (tensor<3x3xf32> {tf_saved_model.index_path = ["r"]}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] +# CHECK-NEXT: %[[READ_VAR_0:.*]] = "tf.ReadVariableOp"(%[[ARG_2]]) {{{.*}}} : (tensor>>) -> tensor<1x3xf32> +# CHECK-NEXT: %[[MATMUL_0:.*]] = "tf.MatMul"(%[[ARG_1]], %[[READ_VAR_0]]) <{{{.*}}}> {{{.*}}} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> +# CHECK-NEXT: return %[[MATMUL_0]] : tensor<3x3xf32> + + +def Test(): + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.compat.v1.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True, + ) + r = tf.matmul(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return ( + { + 'key': ( + tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function', + ) + ) + }, + tf.initializers.global_variables(), + None, + ) + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test, include_variables_in_initializers=True) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/keras.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/keras.py new file mode 100644 index 0000000000000000000000000000000000000000..47280a4bb03206ac2eb195ae767575e6ffb4cf87 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/keras.py @@ -0,0 +1,48 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/keras | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +def mnist_model(): + """Creates a MNIST model.""" + model = tf.keras.models.Sequential() + model.add(tf.keras.layers.Flatten()) + model.add(tf.keras.layers.Dense(128, activation='relu')) + model.add(tf.keras.layers.Dense(10, activation='softmax')) + return model + + +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + self.model = mnist_model() + + # CHECK: func {{@[a-zA-Z_0-9]+}}(%arg0: tensor<1x28x28x1xf32> {tf._user_specified_name = "x", tf_saved_model.index_path = [0]} + # CHECK: attributes {{.*}} tf_saved_model.exported_names = ["my_predict"] + @tf.function(input_signature=[ + tf.TensorSpec([1, 28, 28, 1], tf.float32), + ]) + def my_predict(self, x): + return self.model(x) + + +if __name__ == '__main__': + common.do_test(TestModule, exported_names=['my_predict']) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_arguments_results_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_arguments_results_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..e34013e41aa25dfcbd37724bc707fbd888176f61 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_arguments_results_v1.py @@ -0,0 +1,88 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/multi_arguments_results_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 +from tensorflow.python.ops import array_ops + +# Tests multiple inputs and outputs with index paths. + +# CHECK-LABEL: func @key( +# CHECK-SAME: %[[ARG0:.*]]: tensor<3x5xf32> {tf_saved_model.index_path = ["y"]} +# CHECK-SAME: %[[ARG1:.*]]: tensor<5x3xf32> {tf_saved_model.index_path = ["x"]} +# CHECK-SAME: tensor<3x3xf32> {tf_saved_model.index_path = ["t"]} +# CHECK-SAME: tensor<5x5xf32> {tf_saved_model.index_path = ["s"]} +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] +# CHECK-DAG: %[[MUL0:.*]] = "tf.MatMul"(%[[ARG1]], %[[ARG0]]) +# CHECK-DAG: %[[MUL1:.*]] = "tf.MatMul"(%[[ARG0]], %[[ARG1]]) +# CHECK: %[[IDENTITY:.*]]:2 = "tf.IdentityN"(%[[MUL1]], %[[MUL0]]) +# CHECK: return %[[IDENTITY]]#0, %[[IDENTITY]]#1 + +# CHECK-LABEL: func @key2( +# CHECK-SAME: %[[ARG1:.*]]: tensor<5x3xf32> {tf_saved_model.index_path = ["b"]} +# CHECK-SAME: %[[ARG0:.*]]: tensor<3x5xf32> {tf_saved_model.index_path = ["a"]} +# CHECK-SAME: tensor<5x5xf32> {tf_saved_model.index_path = ["d"]} +# CHECK-SAME: tensor<3x3xf32> {tf_saved_model.index_path = ["c"]} +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key2"] +# CHECK-DAG: %[[MUL1:.*]] = "tf.MatMul"(%[[ARG0]], %[[ARG1]]) +# CHECK-DAG: %[[MUL2:.*]] = "tf.MatMul"(%[[ARG1]], %[[ARG0]]) +# CHECK: %[[IDENTITY:.*]]:2 = "tf.IdentityN"(%[[MUL1]], %[[MUL2]]) +# CHECK: return %[[IDENTITY]]#1, %[[IDENTITY]]#0 + + +def Test(): + + x = tf.constant(1.0, shape=(5, 3)) + y = tf.constant(1.0, shape=(3, 5)) + + s = tf.matmul(x, y) + t = tf.matmul(y, x) + [t, s] = array_ops.identity_n([t, s]) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_y = tf.compat.v1.saved_model.utils.build_tensor_info(y) + tensor_info_s = tf.compat.v1.saved_model.utils.build_tensor_info(s) + tensor_info_t = tf.compat.v1.saved_model.utils.build_tensor_info(t) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'x': tensor_info_x, + 'y': tensor_info_y + }, + outputs={ + 's': tensor_info_s, + 't': tensor_info_t + }, + method_name='some_function')), + 'key2': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={ + 'a': tensor_info_y, + 'b': tensor_info_x, + }, + outputs={ + 'c': tensor_info_t, + 'd': tensor_info_s, + }, + method_name='reverse_arguments')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_variables_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_variables_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..8ece90c0997fcb0504e89864d8b5b45556feabbd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/multi_variables_v1.py @@ -0,0 +1,60 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/multi_variables_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR0:[a-zA-Z_0-9]+]]", type = tensor<5x3xf32>, value = {{.*}} : tensor<5x3xf32>}> : () -> () +# CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR1:[a-zA-Z_0-9]+]]", type = tensor<3x5xf32>, value = {{.*}} : tensor<3x5xf32>}> : () -> () +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG0:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR0]]}, +# CHECK-SAME: [[ARG1:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR1]]}) +# CHECK-SAME: -> (tensor<5x5xf32> {{{.*}}}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] + +# CHECK-NEXT: [[R0:%.*]] = "tf.ReadVariableOp"([[ARG0]]) {{{.*}}} : (tensor>>) -> tensor<5x3xf32> +# CHECK-NEXT: [[R1:%.*]] = "tf.ReadVariableOp"([[ARG1]]) {{{.*}}} : (tensor>>) -> tensor<3x5xf32> +# CHECK-NEXT: [[R2:%.*]] = "tf.MatMul"([[R0]], [[R1]]) <{{{.*}}}> {{{.*}}} : (tensor<5x3xf32>, tensor<3x5xf32>) -> tensor<5x5xf32> + + +def Test(): + + x = tf.compat.v1.get_variable( + name='x', + shape=(5, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + y = tf.compat.v1.get_variable( + name='y', + shape=(3, 5), + initializer=tf.random_normal_initializer(), + trainable=True) + z = tf.matmul(x, y) + tensor_info_z = tf.compat.v1.saved_model.utils.build_tensor_info(z) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs=None, + outputs={'z': tensor_info_z}, + method_name='some_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/no_input_shape_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/no_input_shape_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..6e17089c00c6ac9073c8b23b8d162ca38e47ca5c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/no_input_shape_v1.py @@ -0,0 +1,62 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/no_input_shape_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 +from tensorflow.core.protobuf import meta_graph_pb2 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG:%.*]]: tensor<*xf32> {tf_saved_model.index_path = ["x"]} + +# CHECK: [[shape:%.*]] = "tf.Shape"([[ARG]]) +# CHECK-NEXT: [[batch_size:%.*]] = "tf.StridedSlice"([[shape]], +# CHECK-NEXT: [[result:%.*]] = "tf.Pack"([[batch_size]], +# CHECK-NEXT: return [[result]] : tensor<2xi32> + + +def Test(): + + x = tf.placeholder(dtype=tf.float32, shape=[None]) + batch_size = tf.shape(x)[0] + r = tf.convert_to_tensor([batch_size, 1]) + + tensor_info_x = meta_graph_pb2.TensorInfo( + name=x.name, dtype=tf.as_dtype(x.dtype).as_datatype_enum) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/partially_shaped_variables.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/partially_shaped_variables.py new file mode 100644 index 0000000000000000000000000000000000000000..13f933ae3120ad9cd18d42075ea18e5ca4071e29 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/partially_shaped_variables.py @@ -0,0 +1,34 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/partially_shaped_variables | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + + def __init__(self): + super(TestModule, self).__init__() + # CHECK: "tf_saved_model.global_tensor"() <{is_mutable, {{.*}} type = tensor<*xf32>, value = dense<0.000000e+00> : tensor<1xf32>}> {tf_saved_model.exported_names = ["v0"]} : () -> () + # CHECK: "tf_saved_model.global_tensor"() <{is_mutable, {{.*}} type = tensor, value = dense<[0.000000e+00, 1.000000e+00]> : tensor<2xf32>}> {tf_saved_model.exported_names = ["v1"]} : () -> () + self.v0 = tf.Variable([0.], shape=tf.TensorShape(None)) + self.v1 = tf.Variable([0., 1.], shape=[None]) + + +if __name__ == '__main__': + common.do_test(TestModule, exported_names=[]) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/remove_init_variable_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/remove_init_variable_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..fbd21a7e250625339300bb7c2874f6b5f132ac43 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/remove_init_variable_v1.py @@ -0,0 +1,69 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/remove_init_variable_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# Verify that the tf.versions attribute exists. It is difficult to enforce +# contents, since the version numbers change over time. The conversion logic +# itself is verified in the common graphdef converter, so here just assert +# it is being invoked. +# CHECK: module +# CHECK-SAME: tf.versions +# CHECK-SAME: bad_consumers +# CHECK-SAME: min_consumer +# CHECK-SAME: producer + +# CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR:[a-zA-Z_0-9]+]]", type = tensor<1x3xf32>, value = {{.*}} : tensor<1x3xf32>}> : () -> () +# CHECK-NOT: session_initializer + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG0:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]}, +# CHECK-SAME: [[ARG1:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR]]}) +# CHECK-SAME: -> (tensor<3x3xf32> {tf_saved_model.index_path = ["r"]}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] + +# CHECK-NEXT: [[R0:%.*]] = "tf.ReadVariableOp"([[ARG1]]) {{{.*}}} : (tensor>>) -> tensor<1x3xf32> +# CHECK-NEXT: [[R1:%.*]] = "tf.MatMul"([[ARG0]], [[R0]]) <{{{.*}}}> {{{.*}}} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> +# CHECK-NEXT: return [[R1]] : tensor<3x3xf32> + + +def Test(): + + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.compat.v1.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + r = tf.matmul(x, y) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.compat.v1.saved_model.utils.build_tensor_info(r) + + return { + 'key': (tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function')) + }, tf.initializers.global_variables(), None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test, canonicalize=True) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shapes_for_arguments.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shapes_for_arguments.py new file mode 100644 index 0000000000000000000000000000000000000000..d12661e208f41209d5b887b33ba891e57db70c8b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shapes_for_arguments.py @@ -0,0 +1,42 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/shapes_for_arguments | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + + # Check that we get shapes annotated on function arguments. + # + # Besides checking the shape on the function input argument, this test also + # checks that the shape on the input argument is propagated to the return + # value. + # We eventually want to move the shape inference to a pass separate from + # the initial import, in which case that aspect of this test doesn't make much + # sense and will be superceded by MLIR->MLIR shape inference tests. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}(%arg0: tensor {{.*}}) -> (tensor {{.*}}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["some_function"] + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def some_function(self, x): + return x + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shared_variable_v1.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shared_variable_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..4cfd9c785b441d9561fa330257ed39b19d46b6c0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/shared_variable_v1.py @@ -0,0 +1,65 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/shared_variable_v1 | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v1 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 + +# CHECK: "tf_saved_model.global_tensor"() <{is_mutable, sym_name = "[[VAR:[a-zA-Z_0-9]+]]", type = tensor<1x3xf32>, value = {{.*}} : tensor<1x3xf32>}> : () -> () + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG0:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]}, +# CHECK-SAME: [[ARG1:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR]]}) +# CHECK-SAME: -> (tensor<3x3xf32> {tf_saved_model.index_path = ["r"]}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key"] + +# CHECK: func {{@[a-zA-Z_0-9]+}}( +# CHECK-SAME: [[ARG2:%.*]]: tensor<3x1xf32> {tf_saved_model.index_path = ["x"]}, +# CHECK-SAME: [[ARG3:%.*]]: tensor>> {tf_saved_model.bound_input = @[[VAR]]}) +# CHECK-SAME: -> (tensor<3x3xf32> {tf_saved_model.index_path = ["r"]}) +# CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["key2"] + + +def Test(): + + x = tf.constant([[1.0], [1.0], [1.0]]) + y = tf.get_variable( + name='y', + shape=(1, 3), + initializer=tf.random_normal_initializer(), + trainable=True) + r = tf.matmul(x, y) + + tensor_info_x = tf.saved_model.utils.build_tensor_info(x) + tensor_info_r = tf.saved_model.utils.build_tensor_info(r) + + signature_def = tf.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_function') + signature_def2 = tf.saved_model.signature_def_utils.build_signature_def( + inputs={'x': tensor_info_x}, + outputs={'r': tensor_info_r}, + method_name='some_other_function') + + # Create two signatures that share the same variable. + return {'key': signature_def, 'key2': signature_def2}, None, None + + +if __name__ == '__main__': + common_v1.set_tf_options() + common_v1.do_test(Test) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_input.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_input.py new file mode 100644 index 0000000000000000000000000000000000000000..9a1841f76fa534242e7a5ab4f3260082324a1285 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_input.py @@ -0,0 +1,125 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/structured_input | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + # The fNNNN name prefixes in this file are such that the sorted order of the + # functions in the resulting MLIR output match the order in the source file, + # allowing us to conveniently co-locate the CHECK's with the code they are + # checking. + # + # Note: CHECK-DAG doesn't work with CHECK-SAME/CHECK-NEXT. + + # Check index paths for arguments. + # The outer layer of the index path indexes into the arguments. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor<1xf32> {tf._user_specified_name = "x", tf_saved_model.index_path = [0]}, + # CHECK-SAME: %arg1: tensor<2xf32> {tf._user_specified_name = "y", tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0000_function_arity"] + @tf.function(input_signature=[ + tf.TensorSpec([1], tf.float32), + tf.TensorSpec([2], tf.float32) + ]) + def f0000_function_arity(self, x, y): + return + + # Check index paths for lists. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor {tf._user_specified_name = "l", tf_saved_model.index_path = [0, 0]}, + # CHECK-SAME: %arg1: tensor {tf._user_specified_name = "l", tf_saved_model.index_path = [0, 1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0001_list_2_elements"] + @tf.function(input_signature=[[ + tf.TensorSpec([], tf.float32), + tf.TensorSpec([], tf.float32), + ]]) + def f0001_list_2_elements(self, l): + return + + # Check index paths for dicts. + # Keys are linearized in sorted order, matching `tf.nest.flatten`. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor<1xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "x"]}, + # CHECK-SAME: %arg1: tensor<2xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "y"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0002_dict_2_keys"] + @tf.function(input_signature=[{ + 'x': tf.TensorSpec([1], tf.float32), + 'y': tf.TensorSpec([2], tf.float32), + }]) + def f0002_dict_2_keys(self, d): + return + + # Check index paths for dicts, where the keys are not in sorted order. + # The index path should be insensitive to the key order. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor<1xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "x"]}, + # CHECK-SAME: %arg1: tensor<2xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "y"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0003_dict_2_keys_out_of_order"] + @tf.function(input_signature=[{ + 'y': tf.TensorSpec([2], tf.float32), + 'x': tf.TensorSpec([1], tf.float32), + }]) + def f0003_dict_2_keys_out_of_order(self, d): + return + + # Slightly stronger stress test of multiple dict keys. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor<1xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "a"]}, + # CHECK-SAME: %arg1: tensor<2xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "b"]}, + # CHECK-SAME: %arg2: tensor<3xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "c"]}, + # CHECK-SAME: %arg3: tensor<4xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "x"]}, + # CHECK-SAME: %arg4: tensor<5xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "y"]}, + # CHECK-SAME: %arg5: tensor<6xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "z"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0004_dict_many_keys"] + @tf.function(input_signature=[{ + 'x': tf.TensorSpec([4], tf.float32), + 'y': tf.TensorSpec([5], tf.float32), + 'z': tf.TensorSpec([6], tf.float32), + 'a': tf.TensorSpec([1], tf.float32), + 'b': tf.TensorSpec([2], tf.float32), + 'c': tf.TensorSpec([3], tf.float32), + }]) + def f0004_dict_many_keys(self, d): + return + + # Check a slightly more complex recursive structure. + # Note that list elements can have heterogenous types. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor<1xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "x", 0]}, + # CHECK-SAME: %arg1: tensor<2xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "x", 1]}, + # CHECK-SAME: %arg2: tensor<3xf32> {tf._user_specified_name = "d", tf_saved_model.index_path = [0, "y"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0005_more_complex_recursive_structure"] + @tf.function(input_signature=[{ + 'x': [tf.TensorSpec([1], tf.float32), + tf.TensorSpec([2], tf.float32)], + 'y': tf.TensorSpec([3], tf.float32), + }]) + def f0005_more_complex_recursive_structure(self, d): + return + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_output.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_output.py new file mode 100644 index 0000000000000000000000000000000000000000..74c9a38946af76f3ac315571ae9e3c698b6ddf0d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/structured_output.py @@ -0,0 +1,121 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# RUN: %p/structured_output | FileCheck %s + +# pylint: disable=missing-docstring,line-too-long +import tensorflow.compat.v2 as tf +from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common + + +class TestModule(tf.Module): + # The fNNNN name prefixes in this file are such that the sorted order of the + # functions in the resulting MLIR output match the order in the source file, + # allowing us to conveniently co-locate the CHECK's with the code they are + # checking. + # + # Note: CHECK-DAG doesn't work with CHECK-SAME/CHECK-NEXT. + + # Check index paths for results. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = []}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0000_single_return"] + @tf.function(input_signature=[]) + def f0000_single_return(self): + return tf.constant(1.0, shape=[1]) + + # Check index paths for results with multiple return values. + # Note that semantically in Python, multiple return values are equivalent + # to returning a tuple/list. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = [0]}, + # CHECK-SAME: tensor<2xf32> {tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0001_multiple_results_no_punctuation"] + @tf.function(input_signature=[]) + def f0001_multiple_results_no_punctuation(self): + return tf.constant(1.0, shape=[1]), tf.constant(1.0, shape=[2]) + + # Check index paths for results written explicitly with parentheses. + # This is semantically equivalent to the earlier test without parentheses, + # but this test serves as documentation of this behavior for the purposes + # of tf_saved_model users. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = [0]}, + # CHECK-SAME: tensor<2xf32> {tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0002_multiple_results_parentheses"] + @tf.function(input_signature=[]) + def f0002_multiple_results_parentheses(self): + return (tf.constant(1.0, shape=[1]), tf.constant(1.0, shape=[2])) + + # Check index paths for results written explicitly with brackets. + # This is semantically equivalent to the earlier test without parentheses, + # but this test serves as documentation of this behavior for the purposes + # of tf_saved_model users. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = [0]}, + # CHECK-SAME: tensor<2xf32> {tf_saved_model.index_path = [1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0003_multiple_results_brackets"] + @tf.function(input_signature=[]) + def f0003_multiple_results_brackets(self): + return [tf.constant(1.0, shape=[1]), tf.constant(1.0, shape=[2])] + + # Check index paths for lists. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = [0, 0]}, + # CHECK-SAME: tensor<2xf32> {tf_saved_model.index_path = [0, 1]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0004_list_2_elements"] + @tf.function(input_signature=[]) + def f0004_list_2_elements(self): + return [[tf.constant(1.0, shape=[1]), tf.constant(1.0, shape=[2])]] + + # Check index paths for dicts. + # Keys are linearized in sorted order, matching `tf.nest.flatten`. + # More thorough testing of this is in structured_input.py. The underlying code + # path for linearization is shared, so no need to replicate that testing here. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}() -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = ["x"]}, + # CHECK-SAME: tensor<2xf32> {tf_saved_model.index_path = ["y"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0005_dict_2_keys"] + @tf.function(input_signature=[]) + def f0005_dict_2_keys(self): + return { + 'x': tf.constant(1.0, shape=[1]), + 'y': tf.constant(1.0, shape=[2]), + } + + # Check index paths for outputs are correctly handled in the presence of + # multiple return statements. + # + # CHECK: func {{@[a-zA-Z_0-9]+}}( + # CHECK-SAME: %arg0: tensor {tf._user_specified_name = "x", tf_saved_model.index_path = [0]} + # CHECK-SAME: ) -> ( + # CHECK-SAME: tensor<1xf32> {tf_saved_model.index_path = ["x"]}) + # CHECK-SAME: attributes {{.*}} tf_saved_model.exported_names = ["f0006_multiple_return_statements"] + @tf.function(input_signature=[tf.TensorSpec([], tf.float32)]) + def f0006_multiple_return_statements(self, x): + if x > 3.: + return {'x': tf.constant(1.0, shape=[1])} + else: + return {'x': tf.constant(1.0, shape=[1])} + + +if __name__ == '__main__': + common.do_test(TestModule) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_asset_sinking.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_asset_sinking.mlir new file mode 100644 index 0000000000000000000000000000000000000000..70af9152803617027665a284f265d0292cc7ddc8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_asset_sinking.mlir @@ -0,0 +1,30 @@ +// RUN: tf-opt %s -split-input-file -tf-saved-model-asset-sinking='saved-model-dir=foo/bar' | FileCheck %s + +// CHECK-LABEL: module @asset +module @asset attributes {tf_saved_model.semantics} { + "tf_saved_model.session_initializer"() {initializers = [@init]} : () -> () + + // CHECK-NOT: "tf_saved_model.asset" + "tf_saved_model.asset"() {filename = "assets/test0.txt", sym_name = "asset0"} : () -> () + "tf_saved_model.asset"() {filename = "assets/test1.txt", sym_name = "asset1"} : () -> () + + // CHECK: func @init() + func.func @init(%arg0: tensor {tf_saved_model.bound_input = @asset0}, %arg1: tensor {tf_saved_model.bound_input = @asset1}) attributes {tf_saved_model.exported_names = ["init"]} { + // CHECK-DAG: %[[ASSET0:.*]] = "tf.Const"() <{value = dense<"foo/bar/assets/test0.txt"> : tensor}> + // CHECK-DAG: %[[ASSET1:.*]] = "tf.Const"() <{value = dense<"foo/bar/assets/test1.txt"> : tensor}> + + // CHECK: %[[VAR0:.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {container = "", shared_name = "var0"} : () -> tensor>> + + // CHECK: "tf.AssignVariableOp"(%[[VAR0]], %[[ASSET0]]) + "tf.AssignVariableOp"(%0, %arg0) : (tensor>>, tensor) -> () + + // CHECK: %[[VAR1:.*]] = "tf.VarHandleOp"() + %1 = "tf.VarHandleOp"() {container = "", shared_name = "var1"} : () -> tensor>> + + // CHECK: "tf.AssignVariableOp"(%[[VAR1]], %[[ASSET1]]) + "tf.AssignVariableOp"(%1, %arg1) : (tensor>>, tensor) -> () + + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2cb5c40f01f13b11130232cffb657588875889ea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir @@ -0,0 +1,33 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-saved-model-dedup-bound-input-binding-pass | FileCheck %s + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // Test case: Remove duplicate bound_input symbols. + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "w", type = tensor, value = dense<43.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "x", type = tensor, value = dense<44.0> : tensor } : () -> () + // CHECK: func @f + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @v} + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @w} + // CHECK: %arg2: tensor>> {tf_saved_model.bound_input = @x} + // CHECK-NOT: %arg3 + // CHECK-NOT: %arg4 + func.func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @w}, + %arg2: tensor>> {tf_saved_model.bound_input = @v}, + %arg3: tensor>> {tf_saved_model.bound_input = @x}, + %arg4: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %val0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %val1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %val2 = "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor + %val3 = "tf.ReadVariableOp"(%arg3) : (tensor>>) -> tensor + %val4 = "tf.ReadVariableOp"(%arg4) : (tensor>>) -> tensor + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_assets.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_assets.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2f8b3af4d824259a6484b1d4a698b44d750fc57e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_assets.mlir @@ -0,0 +1,86 @@ +// RUN: tf-opt -verify-diagnostics -tf-saved-model-freeze-assets -split-input-file %s | FileCheck %s + +module attributes {tf_saved_model.semantics} { + + // Test case: Basic freezing. + + "tf_saved_model.asset"() {filename = "assets/table.txt", sym_name = "v"} : () -> () + + // CHECK: func @f() + func.func @f(%arg0: tensor {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + "tf.InitializeTableFromTextFileV2"(%0, %arg0) {delimiter = "\09", device = "", key_index = -2 : i64, offset = 0 : i64, value_index = -1 : i64, vocab_size = 437 : i64} : (tensor, tensor) -> () + // CHECK: [[CST:%.+]] = "tf.Const"() <{value = dense<"assets/table.txt"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + // CHECK: [[HASHTABLE:%.+]] = "tf.HashTableV2"() + // CHECK: "tf.InitializeTableFromTextFileV2"([[HASHTABLE]], [[CST]]) + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Sanity check handling of non-bound inputs. + // The pass shouldn't do anything in this case. + + // CHECK: func @f(%arg0 + func.func @f(%arg0: tensor {tf_saved_model.index_path = [0]}) + attributes {tf_saved_model.exported_names = ["f"]} { + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + "tf.InitializeTableFromTextFileV2"(%0, %arg0) {delimiter = "\09", device = "", key_index = -2 : i64, offset = 0 : i64, value_index = -1 : i64, vocab_size = 437 : i64} : (tensor, tensor) -> () + // CHECK: "tf.InitializeTableFromTextFileV2"(%0, %arg0) + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Sanity check handling of non tf.InitializeTableFromTextFileV2 op usages. + + "tf_saved_model.asset"() {filename = "assets/table.txt", sym_name = "v"} : () -> () + + // CHECK: func @f(%arg0 + func.func @f(%arg0: tensor {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor) -> () + func.return + } + + func.func private @f_callee(%arg0: tensor) { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.asset"() {filename = "assets/table.txt", sym_name = "v"} : () -> () + "tf_saved_model.asset"() {filename = "assets/table2.txt", sym_name = "w"} : () -> () + + // CHECK: func @f() + func.func @f(%arg0: tensor {tf_saved_model.bound_input = @v}, %arg1: tensor {tf_saved_model.bound_input = @w}) + attributes {tf_saved_model.exported_names = ["f"]} { + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + "tf.InitializeTableFromTextFileV2"(%0, %arg0) {delimiter = "\09", device = "", key_index = -2 : i64, offset = 0 : i64, value_index = -1 : i64, vocab_size = 437 : i64} : (tensor, tensor) -> () + %1 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + "tf.InitializeTableFromTextFileV2"(%1, %arg1) {delimiter = "\09", device = "", key_index = -2 : i64, offset = 0 : i64, value_index = -1 : i64, vocab_size = 437 : i64} : (tensor, tensor) -> () + // CHECK-DAG: [[CST_1:%.+]] = "tf.Const"() <{value = dense<"assets/table2.txt"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + // CHECK-DAG: [[CST:%.+]] = "tf.Const"() <{value = dense<"assets/table.txt"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + // CHECK: [[HASHTABLE:%.+]] = "tf.HashTableV2"() + // CHECK: "tf.InitializeTableFromTextFileV2"([[HASHTABLE]], [[CST]]) + // CHECK: [[HASHTABLE_1:%.+]] = "tf.HashTableV2"() + // CHECK: "tf.InitializeTableFromTextFileV2"([[HASHTABLE_1]], [[CST_1]]) + func.return + } +} + +// ----- + +// Test running the pass on a module that does not have +// tf_saved_model.semantics. +module {} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors.mlir new file mode 100644 index 0000000000000000000000000000000000000000..79e0c65ad98b69f6079206390e212c2d76f7d7a4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors.mlir @@ -0,0 +1,387 @@ +// RUN: tf-opt -verify-diagnostics -tf-saved-model-freeze-global-tensors -split-input-file %s | FileCheck %s + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + + // Test case: Basic freezing. + + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + + // CHECK: func @f() + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.Const"() <{value = dense<1.000000e+00> : tensor}> + func.return + } +} + +// ----- + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + + // Test case: Sanity check handling of non-bound inputs. + // The pass shouldn't do anything in this case. + + // CHECK: func @f(%arg0: tensor>> {tf_saved_model.index_path = [0]}) + func.func @f(%arg0: tensor>> {tf_saved_model.index_path = [0]}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Fail if mutable global tensors are found. + + // expected-error @+1 {{is not immutable}} + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } + +} + +// ----- + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + + // Test case: success if bound input user's only none ReadVariableOp instance + // is call. + + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<21.0> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> () + func.return + } + + func.func private @f_callee(%arg0: tensor>>) { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @g_callee} : (tensor>>) -> () + func.return + } + + func.func private @g_callee(%arg0: tensor>>) { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.Const"() <{value = dense<2.100000e+01> : tensor}> + func.return + } +} + +// ----- + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + + // Test case: success if bound input user's only none ReadVariableOp instance + // is call with read inside function + + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<32.0> : tensor } : () -> () + + func.func @g(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["g"]} { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @g_callee} : (tensor>>) -> () + func.return + } + + func.func private @g_callee(%arg0: tensor>>) { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.Const"() <{value = dense<3.200000e+01> : tensor}> + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: fail if bound input user's only none ReadVariableOp instance + // is call with write inside function + + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> () + func.return + } + + func.func private @f_callee(%arg0: tensor>>) { + %0 = "tf.Const"() {value = dense<5.000000e+00> : tensor} : () -> tensor + // expected-error @+1 {{immutable bound input}} + "tf.AssignAddVariableOp"(%arg0, %0) : (tensor>>, tensor) -> () + func.return + } +} + +// ----- + +// expected-error @+1 {{could not freeze all global tensors in the module}} +module attributes {tf_saved_model.semantics} { + + // Test case: Fail if some global tensor ops remain + + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() {sym_name = "v2", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return + } +} + +// ----- + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() {sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() {sym_name = "v2", type = tensor, value = dense<2.0> : tensor } : () -> () + + func.func @f(%arg1: tensor>> {tf_saved_model.bound_input = @"v"}, %arg2: tensor>> {tf_saved_model.bound_input = @"v2"}) + attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK-DAG: "tf.Const"() <{value = dense<1.000000e+00> : tensor}> + %0 = "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor>>) -> tensor + + // CHECK-DAG: "tf.Const"() <{value = dense<2.000000e+00> : tensor}> + %1 = "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor>>) -> tensor + func.return + } +} + +// ----- + +// Test running the pass on a module that does not have +// tf_saved_model.semantics. +// CHECK-LABEL: module +module {} + +// ----- + +!tf_res = tensor>> +!tf_str = tensor<3x!tf_type.string> + +// CHECK-LABEL: module attributes +module attributes {tf_saved_model.semantics} { + +"tf_saved_model.global_tensor"() {sym_name = "v1", type = tensor, value = dense<3.0> : tensor } : () -> () +"tf_saved_model.global_tensor"() {sym_name = "v2", type = tensor, value = dense<2.0> : tensor } : () -> () + +// CHECK-LABEL: @body +func.func private @body(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res) { + %graph:2 = tf_executor.graph { + %value, %value_control = tf_executor.island wraps "tf.GetKey"() : () -> tensor + %ret0, %ret0_control = tf_executor.island wraps "tf.SomeOp"() : () -> !tf_res + %ret1, %ret1_control = tf_executor.island wraps "tf.SomeOp"() : () -> !tf_res + %control_unknown = tf_executor.island wraps "tf.UnknownOp"() : () -> () + %key, %key_control = tf_executor.island wraps "tf.GetKey"() : () -> !tf_str + // CHECK: "tf.ReadVariableOp"(%arg0) + %read1, %read1_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg1) + %read2, %read2_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) : (!tf_res) -> tensor + tf_executor.island wraps "tf.TPUExecuteAndUpdateVariables"(%ret0, %ret1, %key) { + device_var_reads_indices = [0, 1], + device_var_updates_indices = [0, 1]} : (!tf_res, !tf_res, !tf_str) -> () + tf_executor.fetch %ret0, %ret1: !tf_res, !tf_res + } + func.return %graph#0, %graph#1 : !tf_res, !tf_res +} + +// CHECK-LABEL: @cond +func.func private @cond(%arg0: !tf_res, %arg1: !tf_res) -> (tensor) { + %graph = tf_executor.graph { + %island, %ctrl = tf_executor.island { + %pred = "tf.SomeOp"() : () -> tensor + tf_executor.yield %pred : tensor + } + tf_executor.fetch %island : tensor + } + func.return %graph : tensor +} + +// CHECK-LABEL: @test_while_loop +func.func @test_while_loop(%arg0: !tf_res {tf._composite_device = "/job:tpu_host_worker/replica:0/task:0/device:COMPOSITE:0", tf_saved_model.bound_input = @v1}, + %arg1: !tf_res {tf_saved_model.bound_input = @v2}) + attributes {tf_saved_model.exported_names = ["test_while_loop"]} { + // CHECK-DAG: Const{{.*}}2.0 + // CHECK-DAG: Const{{.*}}3.0 + %read1 = "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor + %read2 = "tf.ReadVariableOp"(%arg1) : (!tf_res) -> tensor + // CHECK: tf_executor.graph + tf_executor.graph { + %handle0, %handle0_control = tf_executor.island wraps "tf.SomeOp"() : () -> !tf_res + %handle1, %handle1_control = tf_executor.island wraps "tf.SomeOp"() : () -> !tf_res + %control_A = tf_executor.island wraps "tf.OpA"() : () -> () + %while_out:2, %while_control = tf_executor.island(%control_A) wraps "tf.While"( + %handle0, %handle1) { + body = @body, cond = @cond, is_stateless = false + } : (tensor>>, tensor>>) -> (tensor>>, tensor>>) + %control_B = tf_executor.island(%while_control) wraps "tf.OpB"() : () -> () + tf_executor.fetch + } + func.return +} +} + +// ----- + +// Test variable is frozen when it is used inside `TF::BatchFunctionOp` without +// assignment. + +module attributes {tf_saved_model.semantics} { + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() {sym_name = "var1", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func private @f_batch_callee(%arg0: tensor, %arg1: tensor>>) -> (tensor, tensor) { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + func.return %arg0, %0 : tensor, tensor + } + // CHECK: func.func private @f_batch_callee(%[[ARG_0:.*]]: tensor) -> (tensor, tensor) + // CHECK-DAG: %[[CST_0:.*]] = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor + // CHECK: return %[[ARG_0]], %[[CST_0]] : tensor, tensor + + func.func @f(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> (tensor<*xf32> {tf_saved_model.index_path = []}, tensor<*xf32> {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %arg = "tf.Const"() {value = dense<1.000000e+00> : tensor<1xf32>} : () -> tensor<1xf32> + %0, %1 = "tf.BatchFunction"(%arg, %handle) {f = @f_batch_callee, operandSegmentSizes = array, batch_timeout_micros = 1000, max_batch_size = 8, num_batch_threads = 2} : (tensor<1xf32>, tensor>>) -> (tensor<*xf32>, tensor<*xf32>) + func.return %0, %1 : tensor<*xf32>, tensor<*xf32> + } + // CHECK: func.func @f() -> (tensor<*xf32> {tf_saved_model.index_path = []}, tensor<*xf32> {tf_saved_model.index_path = []}) attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: %cst = "tf.Const"() <{value = dense<1.000000e+00> : tensor<1xf32>}> : () -> tensor<1xf32> + // CHECK: %0:2 = "tf.BatchFunction"(%cst) <{batch_timeout_micros = 1000 : i64, f = @f_batch_callee, max_batch_size = 8 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array}> : (tensor<1xf32>) -> (tensor<*xf32>, tensor<*xf32>) + // CHECK: return %0#0, %0#1 : tensor<*xf32>, tensor<*xf32> + // CHECK: } +} + +// ----- + +// Tests that "tf._input_shapes" attribute is updated correctly + +module attributes {tf_saved_model.semantics} { + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() {sym_name = "var1", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %cst = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %val = "tf.PartitionedCall"(%cst, %handle) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor, tensor>>) -> (tensor) + func.return %val : tensor + } + // CHECK: func.func @f() -> (tensor {tf_saved_model.index_path = []}) attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: %cst = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor + // CHECK: %0 = "tf.PartitionedCall"(%cst) <{config = "", config_proto = "", executor_type = "", f = @f_callee}> : (tensor) -> tensor + // CHECK: return %0 : tensor + // CHECK: } + func.func private @f_callee(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>) -> tensor attributes {tf._input_shapes = [#tf_type.shape<0>, #tf_type.shape<>]} { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>) -> tensor + %1 = "tf.AddV2"(%arg0, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor + } + // CHECK: func.func private @f_callee(%arg0: tensor) -> tensor attributes {tf._input_shapes = [#tf_type.shape<0>]} { + // CHECK: %cst = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor + // CHECK: %0 = "tf.AddV2"(%arg0, %cst) : (tensor, tensor) -> tensor + // CHECK: return %0 : tensor + // CHECK: } +} + +// ----- + +// Test While region immutable case. + +module attributes {tf_saved_model.semantics} { + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() {sym_name = "var1", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %res:2 = func.call @f_1(%handle) : (tensor>>) -> (tensor, tensor) + func.return %res#0 : tensor + } + // CHECK: func.func @f() -> (tensor {tf_saved_model.index_path = []}) attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: %0:2 = call @f_1() : () -> (tensor, tensor) + // CHECK: return %0#0 : tensor + // CHECK: } + + // CHECK: func private @f_1() -> (tensor, tensor) + func.func private @f_1(%arg0: tensor>>)-> (tensor, tensor) { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %cst = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %1:3 = "tf.WhileRegion"(%arg0, %0, %cst) ({ + ^bb0(%carg0: tensor<*x!tf_type.resource>, %carg1: tensor, %carg2 : tensor): + %limit = arith.constant dense<5> : tensor + %cond = "tf.Less"(%carg1, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor<*x!tf_type.resource>, %barg1: tensor, %barg2: tensor): + %val = "tf.PartitionedCall"(%barg0) {config = "", config_proto = "", executor_type = "", f = @f_callee_callee} : (tensor<*x!tf_type.resource>) -> (tensor) + "tf.Yield"(%barg0, %barg1, %val) : (tensor<*x!tf_type.resource>,tensor, tensor) -> () + }) {is_stateless = true} : (tensor>>, tensor, tensor) -> (tensor<*x!tf_type.resource>, tensor, tensor) + func.return %1#2, %1#2 : tensor, tensor + } + + // CHECK: func.func private @f_callee_callee() -> tensor { + func.func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %0 : tensor + } + // CHECK: %cst = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor + // CHECK: return %cst : tensor + // CHECK: } +} + +// ----- + +// Make sure global tensors marked as immutable are not written to. +module attributes {tf_saved_model.semantics} { + "tf_saved_model.global_tensor"() {sym_name = "var1", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> () + attributes {tf_saved_model.exported_names = ["f"]} { + %cst = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + // expected-error @+1 {{immutable bound input}} + "tf.AssignVariableOp"(%handle, %cst) : (tensor>>, tensor) -> () + func.return + } + + func.func @f2(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f2"]} { + %0 = "tf.ReadVariableOp"(%handle) : (tensor>>) -> (tensor) + func.return %0 : tensor + } +} + +// ----- + +// Test If Region immutable case. + +module attributes {tf_saved_model.semantics} { + "tf_saved_model.global_tensor"() {sym_name = "var1", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%handle: tensor>> {tf_saved_model.bound_input = @var1}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %arg0 = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %0 = "tf.IfRegion"(%arg0) ({ + // CHECK-NOT: "tf.ReadVariableOp" + %1 = "tf.ReadVariableOp"(%handle) : (tensor>>) -> (tensor) + "tf.Yield"(%1) : (tensor) -> () + }, { + %2 = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + "tf.Yield"(%2) : (tensor) -> () + }) {is_stateless = true} : (tensor) -> (tensor) + func.return %0 : tensor + } +} + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors_mutable_tensors.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors_mutable_tensors.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ca87f2dafe328fbeb76189381c5f2bd4030855ec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_freeze_global_tensors_mutable_tensors.mlir @@ -0,0 +1,16 @@ +// RUN: tf-opt "-tf-saved-model-freeze-global-tensors=allow-mutable-tensors=true" -split-input-file %s | FileCheck %s + +module attributes {tf_saved_model.semantics} { + + // Test case: Do not fail if the tensor is mutable but allow_mutable_tensors is true + + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<1.0> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: "tf.ReadVariableOp" + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor>>) -> tensor + func.return + } + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f25b94518e4e9d729621de95ba538f26ee63e6ad --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init.mlir @@ -0,0 +1,99 @@ +// RUN: tf-opt -tf-saved-model-initialize-variables-in-session-init -split-input-file -verify-diagnostics %s | FileCheck %s + + +// Test with no session init function. +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var1"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } + + // CHECK-LABEL: func @SessionInitializerFunction() + // CHECK-SAME: tf_saved_model.exported_names = ["SessionInitializerFunction"] + // CHECK-SAME: tf_saved_model.initializer_type = "restore_op" + // CHECK: %[[VAR:.*]] = "tf.VarHandleOp" + // CHECK-SAME: "var1" + // CHECK: %[[CST:.*]] = arith.constant dense<> : tensor<0xf32> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[CST]]) +} + +// ----- + +// Test with an existing session init function. +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + "tf_saved_model.session_initializer"() {initializers = [@Init]} : () -> () + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var1"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } + + func.func @Init() attributes {tf_saved_model.exported_names = ["Init"], tf_saved_model.initializer_type = "restore_op"} { + func.return + } + + // CHECK-LABEL: func @Init + // CHECK-SAME: tf_saved_model.exported_names = ["Init"] + // CHECK-SAME: tf_saved_model.initializer_type = "restore_op" + // CHECK: %[[VAR:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: "var1" + // CHECK: %[[CST:.*]] = arith.constant dense<> : tensor<0xf32> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[CST]]) +} + + +// ----- + +// Test invalid tensor +// expected-error@below{{Can't find variable var5 in session}} +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var5"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } +} + +// ----- + +// Test with empty initializers. +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + "tf_saved_model.session_initializer"() {initializers = []} : () -> () + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var1"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } + + // CHECK-LABEL: func @SessionInitializerFunction + // CHECK-SAME: tf_saved_model.exported_names = ["SessionInitializerFunction"] + // CHECK-SAME: tf_saved_model.initializer_type = "restore_op" + // CHECK: %[[VAR:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: "var1" + // CHECK: %[[CST:.*]] = arith.constant dense<> : tensor<0xf32> + // CHECK: "tf.AssignVariableOp"(%[[VAR]], %[[CST]]) +} + +// ----- + +// Variable not in session. No initialization should happen. +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // CHECK-NOT: "tf.AssignVariableOp"() + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + // CHECK: "tf.VarHandleOp" + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "invalid", shared_name = "invalid_var"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init_fail.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init_fail.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e274c9bc3277b15e6ade8e143908dd6a13e4b67d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_initialize_variables_in_session_init_fail.mlir @@ -0,0 +1,12 @@ +// RUN: tf-opt -tf-saved-model-initialize-variables-in-session-init -verify-diagnostics -fail-to-fetch-local-device-manager %s + +// expected-error@below{{No Local Device Manager}} +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var1"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + func.return %1 : tensor<100x50xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4306e880cd047626980e780b33132fb942a31371 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables.mlir @@ -0,0 +1,124 @@ +// RUN: tf-opt -verify-diagnostics -tf-saved-model-lift-variables-test=import-variables-as-dense-resources=true -split-input-file %s | FileCheck --check-prefix=CheckWithDense %s --dump-input=fail +// RUN: tf-opt -verify-diagnostics -tf-saved-model-lift-variables-test -split-input-file %s | FileCheck %s --dump-input=fail + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // Test case: Freezing VarHandleOp ops. + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "", shared_name = "dense/kernel"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "", shared_name = "dense/bias"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + func.return %4 : tensor<100x50xf32> + } + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/kernel" + // CHECK: value = dense<0.000000e+00> + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/bias" + // CHECK: value = dense<0.000000e+00> + // CHECK: func @serving_default( + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) + + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/kernel" + // CheckWithDense: value = dense_resource + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/bias" + // CheckWithDense: value = dense_resource + // CheckWithDense: func @serving_default( + // CheckWithDense: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CheckWithDense: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // Test case: Freezing shared VarHandleOp ops. + + func.func @f(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["f"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "", shared_name = "dense/kernel"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "", shared_name = "dense/bias"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + func.return %4 : tensor<100x50xf32> + } + + func.func @f2(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["f2"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "", shared_name = "dense/kernel"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "", shared_name = "dense/bias"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + func.return %4 : tensor<100x50xf32> + } + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/kernel" + // CHECK: value = dense<0.000000e+00> + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/bias" + // CHECK: value = dense<0.000000e+00> + // CHECK: func @f( + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) + + // CHECK: func @f2( + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) + + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/kernel" + // CheckWithDense: value = dense_resource + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/bias" + // CheckWithDense: value = dense_resource + // CheckWithDense: func @f( + // CheckWithDense: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CheckWithDense: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) + + // CheckWithDense: func @f2( + // CheckWithDense: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CheckWithDense: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // Test case: Fix bound_inputs' types. + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<*xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor>>) -> tensor<*xf32> + %1 = "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor>>) -> tensor<*xf32> + %2 = "tf.Add"(%0, %1) {device = ""} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> + } + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/kernel" + // CHECK: value = dense<0.000000e+00> + // CHECK: "tf_saved_model.global_tensor"() + // CHECK: sym_name = "dense/bias" + // CHECK: value = dense<0.000000e+00> + // CHECK: func @serving_default( + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) + + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/kernel" + // CheckWithDense: value = dense_resource + // CheckWithDense: "tf_saved_model.global_tensor"() + // CheckWithDense: sym_name = "dense/bias" + // CheckWithDense: value = dense_resource + // CheckWithDense: func @serving_default( + // CheckWithDense: %arg0: tensor>> {tf_saved_model.bound_input = @"dense/kernel"}, + // CheckWithDense: %arg1: tensor>> {tf_saved_model.bound_input = @"dense/bias"}) +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables_invalid_session.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables_invalid_session.mlir new file mode 100644 index 0000000000000000000000000000000000000000..72cbfe6ec722338f989398fbf8a8aafc498431f6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_lift_variables_invalid_session.mlir @@ -0,0 +1,33 @@ +// RUN: tf-opt -verify-diagnostics -tf-saved-model-lift-variables-invalid-session-test -split-input-file %s | FileCheck %s --dump-input=fail + +// Test case: Invalid session. +// expected-error @+1 {{'builtin.module' op no session provided}} +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "", shared_name = "dense/kernel"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "", shared_name = "dense/bias"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + func.return %4 : tensor<100x50xf32> + } +} + +// ----- + +// Test case: No errors on no resource arguments. +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // CHECK-LABEL: @serving_default + func.func @serving_default() -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "", shared_name = "dense/kernel"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "", shared_name = "dense/bias"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + func.return %4 : tensor<100x50xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_mark_initialized_variables.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_mark_initialized_variables.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c23fa762ed5052260abf073e19a97e092409bdc1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_mark_initialized_variables.mlir @@ -0,0 +1,28 @@ +// RUN: tf-opt -tf-saved-model-mark-initialized-variables-test %s | FileCheck %s +// RUN: tf-opt -tf-saved-model-mark-initialized-variables-invalid-session-test %s | FileCheck %s --check-prefix=INVALID + + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + func.func @serving_default(%arg0: tensor>> {tf.resource_name = "dense/kernel"}, %arg1: tensor>> {tf.resource_name = "dense/bias"}) -> (tensor<100x50xf32> {tf_saved_model.index_path = ["dense_2"]}) + attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "dense_2/Add:0"}, tf_saved_model.exported_names = ["serving_default"]} { + %0 = "tf.VarHandleOp"() {_class = ["loc:@dense/kernel"], allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var1"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = ""} : (tensor>>) -> tensor<100x50xf32> + %2 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var2"} : () -> tensor>> + %3 = "tf.ReadVariableOp"(%2) {device = ""} : (tensor>>) -> tensor<50xf32> + %4 = "tf.Add"(%1, %3) {device = ""} : (tensor<100x50xf32>, tensor<50xf32>) -> tensor<100x50xf32> + %5 = "tf.VarHandleOp"() {_class = ["loc:@dense/bias"], allowed_devices = [], container = "", device = "/job:worker/replica:0/task:1/device:CPU:0", shared_name = "var3"} : () -> tensor>> + func.return %4 : tensor<100x50xf32> + } + // CHECK: "tf.VarHandleOp" + // CHECK-SAME: shared_name = "var1" + // CHECK-SAME: _is_initialized = true + // CHECK: "tf.VarHandleOp" + // CHECK-SAME: shared_name = "var2" + // CHECK-SAME: _is_initialized = true + // CHECK: "tf.VarHandleOp" + // CHECK-SAME: shared_name = "var3" + // CHECK-SAME: _is_initialized = false + + // INVALID-NOT: _is_initialized +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8e1939f9284d335d28d3239fd97c73505e5a934d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir @@ -0,0 +1,91 @@ +// RUN: tf-opt %s | tf-opt | FileCheck %s + +module attributes {tf_saved_model.semantics} { + + // CHECK: tf_saved_model.session_initializer + "tf_saved_model.session_initializer"() { + initializers = [@init] + } : () -> () + + // CHECK: tf_saved_model.asset + "tf_saved_model.asset"() { + filename = "asset_filename", + sym_name = "asset_sym_name" + } : () -> () + + // Representation for constants: (immutable) global tensor. + // CHECK: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() { + tf_saved_model.exported_names = ["some_const"], + sym_name = "some_constant", + type = tensor<1x64xf32>, + value = dense<42.0> : tensor<1x64xf32> + } : () -> () + + // Representation for variables: mutable global tensor. + // CHECK: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() { + is_mutable, + tf_saved_model.exported_names = ["some_var", "some.other.name"], + sym_name = "some_variable", + type = tensor, + value = dense<42.0> : tensor<1x64xf32> + } : () -> () + + // Representation for functions: func's with attributes. + // CHECK: func @__concrete_function_run_computation + func.func @__concrete_function_run_computation( + %arg0: tensor {tf_saved_model.index_path = [0, "foo"]}, + %arg1: tensor>> {tf_saved_model.bound_input = @some_constant}, + %arg2: tensor>> {tf_saved_model.bound_input = @some_variable} + ) -> ( + tensor {tf_saved_model.index_path = [0, "bar"]} + ) attributes { tf_saved_model.exported_names = ["some_func"] } + { + "tf.some_call"() {callee = @f} : () -> () + func.return %arg0 : tensor + } + + func.func private @f() attributes { + func.return + } + + // Representation for init functions + // CHECK: func @init + // CHECK-SAME: exported_names = ["__tf_saved_model_session_initializer"] + func.func @init( + %arg0: tensor {tf_saved_model.bound_input = @asset_sym_name}, + %arg1: tensor>> {tf_saved_model.bound_input = @some_constant} + ) attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"]} + { + "tf.some_call"(%arg1) : (tensor>>) -> () + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // CHECK: func @f + func.func @f( + %arg0: tensor {tf.resource_name = "resource"} + ) attributes { tf_saved_model.exported_names = ["foo.some_func"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + // CHECK: func @f + func.func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b035526855a79052a5d2a4be5589842fc2c55cb4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir @@ -0,0 +1,470 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -allow-unregistered-dialect + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{unknown tf_saved_model dialect arg attribute 'tf_saved_model.not_a_real_arg_attr'}} + func.func private @f(%arg0: tensor {tf_saved_model.not_a_real_arg_attr = 1 : i32}) { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.bound_input' attribute should be a FlatSymbolRefAttr}} + func.func @f( + %arg0: tensor {tf_saved_model.bound_input = 1 : i32} + ) attributes { tf_saved_model.exported_names = ["foo.some_func"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.bound_input' attribute must reference a valid symbol, got invalid symbol 'doesnt_exist'}} + func.func @f( + %arg0: tensor {tf_saved_model.bound_input = @doesnt_exist} + ) attributes { tf_saved_model.exported_names = ["foo.some_func"] } { + func.return + } + +} + +// ----- + +// expected-error@+1 {{'tf_saved_model.exported_names' must be on an op whose immediate parent has attribute 'tf_saved_model.semantics'}} +func.func @f() attributes { tf_saved_model.exported_names = ["foo.some_func"] } { + func.return +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.exported_names' must be on a 'func' or 'tf_saved_model.global_tensor' op}} + "some_dialect.some_op"() { + tf_saved_model.exported_names = ["foo"] + } : () -> () + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.exported_names' must be an array of strings}} + func.func @f() attributes { tf_saved_model.exported_names = 1 : i32} { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-note@+1 {{previously seen here}} + func.func @f() attributes { tf_saved_model.exported_names = ["foo"]} { + func.return + } + + // expected-error@+1 {{duplicate exported name 'foo'}} + func.func @g() attributes { tf_saved_model.exported_names = ["foo"]} { + func.return + } + +} + +// ----- + +// expected-error@+1 {{'tf_saved_model.semantics' must be on a module op}} +"some_dialect.some_op"() {tf_saved_model.semantics} : () -> () + +// ----- + +// expected-error@+1 {{unknown tf_saved_model dialect attribute 'tf_saved_model.not_a_real_op_attr'}} +"some_dialect.some_op"() {tf_saved_model.not_a_real_op_attr} : () -> () + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.index_path' attribute should be an ArrayAttr}} + func.func @f( + %arg0: tensor {tf_saved_model.index_path = 1} + ) attributes { tf_saved_model.exported_names = ["f"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf_saved_model.index_path' elements should be strings or 64-bit integers}} + func.func @f( + %arg0: tensor {tf_saved_model.index_path = [1.0] } + ) attributes { tf_saved_model.exported_names = ["f"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{all arguments should have 'tf_saved_model.index_path', 'tf_saved_model.bound_input' or 'tf.resource_name' attributes}} + func.func @f( + %arg0: tensor + ) attributes { tf_saved_model.exported_names = ["f"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'tf.resource_name' attribute is not allowed unless it is being under construction}} + func.func @f( + %arg0: tensor {tf.resource_name = "resource"} + ) attributes { tf_saved_model.exported_names = ["foo.some_func"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.global_tensor"() { sym_name = "some_constant", type = tensor, value = dense<42.0> : tensor } : () -> () + + // expected-error@+1 {{all 'tf_saved_model.index_path' arg attributes should precede all 'tf_saved_model.bound_input' arg attributes}} + func.func @f( + %arg0: tensor {tf_saved_model.bound_input = @some_constant}, + %arg1: tensor {tf_saved_model.index_path = [0]} + ) attributes { tf_saved_model.exported_names = ["f"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{all results should have 'tf_saved_model.index_path' attributes}} + func.func @f() -> tensor + attributes { tf_saved_model.exported_names = ["f"] } { + %ret = "some_dialect.some_op"() : () -> tensor + func.return %ret : tensor + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Sanity-check that we are verifying tf_saved_model.index_path attributes + // on results as well. The underlying verification logic is shared, + // so no need to test all error cases. + + // expected-error@+1 {{'tf_saved_model.index_path' elements should be strings or 64-bit integers}} + func.func @f() -> (tensor {tf_saved_model.index_path = [1.0]}) + attributes { tf_saved_model.exported_names = ["f"] } { + %ret = "some_dialect.some_op"() : () -> tensor + func.return %ret : tensor + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + func.func @f() attributes { tf_saved_model.exported_names = ["f"] } { + // expected-error@+1 {{exported function cannot be internally referenced}} + "some_dialect.some_call"() { callee = @g } : () -> () + func.return + } + + // expected-note@+1 {{references this exported function}} + func.func @g() attributes { tf_saved_model.exported_names = ["g"] } { + func.return + } + +} + +// ----- + +// expected-error@+1 {{modules with 'tf_saved_model.semantics' must have analyzable symbol uses}} +module attributes {tf_saved_model.semantics} { + + func.func @root() attributes {tf_saved_model.exported_names = ["root"]} { + "some_unregistered_dialect.maybe_a_symbol_table"() ({ + func.return + }) : () -> () + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // expected-error@+1 {{'type' and 'value' attributes should have compatible tensor types}} + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v0", type = tensor<3xf32>, value = dense<42.0> : tensor<9xf32> } : () -> () +} + +// ----- + +module attributes {tf_saved_model.semantics} { + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + // expected-error@+1 {{duplicate 'tf_saved_model.bound_input' binding}} + func.func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<1.> : tensor<1xf32> } : () -> () + // expected-error@+1 {{can only apply 'tf_saved_model' argument attributes to exported functions}} + func.func private @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + -> (tensor {tf_saved_model.index_path = []}) { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return %0 : tensor + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<1.> : tensor<1xf32> } : () -> () + // expected-error@+1 {{bound input with type 'tensor' expected to have type 'tensor>>'}} + func.func @f(%arg0: tensor {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{'type' attribute for immutable 'tf_saved_model.global_tensor' should have a static shape}} + "tf_saved_model.global_tensor"() { sym_name = "v", type = tensor, value = dense<1.> : tensor<1xf32> } : () -> () +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function does not exist}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function should have no output}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func private @init() -> tensor<1xf32> { + %0 = "tf.Const"() {value = dense<[1.0]> : tensor<1xf32> } : () -> tensor<1xf32> + func.return %0 : tensor<1xf32> + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() { initializer = @init } : () -> () + // expected-error@+1 {{there must be no more than one session_initializer op}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func private @init() -> tensor<1xf32> { + %0 = "tf.Const"() {value = dense<[1.0]> : tensor<1xf32> } : () -> tensor<1xf32> + func.return %0 : tensor<1xf32> + } +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + + // expected-error@+1 {{exported function @f should be public}} + func.func private @f( + %arg0: tensor {tf.resource_name = "resource"} + ) attributes {tf_saved_model.exported_names = ["foo.some_func"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{non-exported function @f should be private}} + func.func @f( + %arg0: tensor {tf.resource_name = "resource"} + ) { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function does not exist}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function should have no output}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() -> (tensor<1xf32> {tf_saved_model.index_path = ["output"]}) + attributes { tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "init_op" } { + %0 = "tf.Const"() {value = dense<[1.0]> : tensor<1xf32> } : () -> tensor<1xf32> + func.return %0 : tensor<1xf32> + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + // expected-error@+1 {{there must be no more than one session_initializer op}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() -> (tensor<1xf32> {tf_saved_model.index_path = ["output"]}) + attributes { tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "init_op" } { + %0 = "tf.Const"() {value = dense<[1.0]> : tensor<1xf32> } : () -> tensor<1xf32> + func.return %0 : tensor<1xf32> + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function should be exported}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func private @init() { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{the initializer function should have only one exported name}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() attributes { tf_saved_model.exported_names = ["a", "b"], tf_saved_model.initializer_type = "restore_op" } { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + + // expected-error@+1 {{tf_saved_model.initializer_type should be one of 'restore_op' or 'init_op'. Got: unknown_init_op}} + func.func @init() attributes { + tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], + tf_saved_model.initializer_type = "unknown_init_op" } { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() { initializers = [@init_0, @init_1] } : () -> () + + func.func @init_0() attributes { + tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_init_0"], + tf_saved_model.initializer_type = "restore_op" } { + func.return + } + + // expected-error@+1 {{Attribute tf_saved_model.initializer_type should not have duplicate values. Found duplicate: "restore_op"}} + func.func @init_1() attributes { + tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_init_1"], + tf_saved_model.initializer_type = "restore_op" } { + func.return + } +} + +// ----- + +// expected-error@+1 {{Attribute tf_saved_model.initializer_type should be on a func::FuncOp}} +module attributes {tf_saved_model.semantics, tf_saved_model.initializer_type = "restore_op"} { +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + + // expected-error@+1 {{tf_saved_model.initializer_type should be a StringAttr}} + func.func @init() attributes { + tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], + tf_saved_model.initializer_type = 123 } { + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // expected-error@+1 {{unknown symbol operation}} + "some_dialect.some_op"() {sym_name = "v"} : () -> () + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes { tf_saved_model.exported_names = ["a"] } { + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + // expected-error@+1 {{duplicate 'tf_saved_model.bound_input' binding}} + func.func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors.mlir new file mode 100644 index 0000000000000000000000000000000000000000..371f38e17829a355a5bb0af71ef5ef851d09fe76 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors.mlir @@ -0,0 +1,246 @@ +// RUN: tf-opt -tf-saved-model-optimize-global-tensors -split-input-file %s | FileCheck %s + +//===----------------------------------------------------------------------===// +// Immutability. +//===----------------------------------------------------------------------===// + +module attributes {tf_saved_model.semantics} { + + // Test case: Basic test of marking immutable. + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return %val : tensor + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Don't mark immutable if the variable is mutated. + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + %c0 = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + "tf.AssignVariableOp"(%arg0, %c0) : (tensor>>, tensor) -> () + func.return + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Don't mark immutable if the variable is exported. + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK: is_mutable + // CHECK-SAME: }> {{{.*}}} : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", tf_saved_model.exported_names = ["v"], type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return %val : tensor + } + +} + + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Check that a non-bound input is left unchanged. + + // CHECK: func @g + func.func @g(%arg0: tensor {tf_saved_model.index_path = [0]}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["g"]} { + // CHECK: return %arg0 + func.return %arg0 : tensor + } + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + + // Test case: Check that no change is made for a global tensor that is already + // immutable. + + "tf_saved_model.global_tensor"() { sym_name = "c", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: func @h(%arg0: tensor>> {tf_saved_model.bound_input = @c}) + func.func @h(%arg0: tensor>> {tf_saved_model.bound_input = @c}) + attributes {tf_saved_model.exported_names = ["h"]} { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return + } + +} + +// ----- + +//===----------------------------------------------------------------------===// +// Erasing unused global tensors. +//===----------------------------------------------------------------------===// + +module attributes {tf_saved_model.semantics} { + + // Test case: Check that an exported global tensor that isn't bound to an + // argument is not erased. + + "tf_saved_model.global_tensor"() { sym_name = "exported_unbound", tf_saved_model.exported_names = ["exported_unbound"], type = tensor, value = dense<42.> : tensor } : () -> () + // CHECK: sym_name = "exported_unbound" + + // Test case: Check that a global tensor that isn't even bound to an argument + // is erased. + + "tf_saved_model.global_tensor"() { sym_name = "unexported_unbound", type = tensor, value = dense<42.> : tensor } : () -> () + // CHECK-NOT: sym_name = "unexported_unbound" + +} + +// ----- + +//===----------------------------------------------------------------------===// +// Erasing unused bound inputs. +//===----------------------------------------------------------------------===// + +module attributes {tf_saved_model.semantics} { + + // We erase the argument that this global tensor is bound to, so we delete + // the global tensor too. + // CHECK-NOT: tf_saved_model.global_tensor + "tf_saved_model.global_tensor"() { sym_name = "c", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: func @f() + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @c}) + attributes {tf_saved_model.exported_names = ["f"]} { + func.return + } + +} + +// ----- + +// Test running the pass on a module that does not have +// tf_saved_model.semantics. +module {} + +// ----- + +// Test use as an input in unhandled op +module attributes {tf_saved_model.semantics} { + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.unhandled_op"(%arg0) : (tensor>>) -> () + func.return + } +} + + +// ----- + +// Test use as a region capture in an unhandled op +module attributes {tf_saved_model.semantics} { + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.unhandled"() ({ + %val = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + "tf.unhandled_terminator"() : () -> () + }) : () -> () + func.return + } +} + +// ----- + +// Test use as region capture as well as input in an unhandled op +// to the unhandled op. +module attributes {tf_saved_model.semantics} { + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "u", type = tensor, value = dense<22.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}, %arg1: tensor>> {tf_saved_model.bound_input = @u}) + attributes {tf_saved_model.exported_names = ["f"]} { + %0 = "tf.unhandled"(%arg0) ({ + %val = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + "tf.unhandled_terminator"() : () -> () + }) : (tensor>>) -> (tensor>>) + func.return + } +} + +// ----- + +// Test multiple global tensors uses as operands for an unhandled op. +module attributes {tf_saved_model.semantics} { + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "u", type = tensor, value = dense<22.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}, %arg1: tensor>> {tf_saved_model.bound_input = @u}) + attributes {tf_saved_model.exported_names = ["f"]} { + "tf.unhandled"(%arg0, %arg1) : (tensor>>, tensor>>) -> () + func.return + } +} + +// ----- + +// Test presence of tf_saved_model.asset's as bound inputs. +// It should not crash. +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.session_initializer"() {initializers = [@legacy_init_op]} : () -> () + "tf_saved_model.asset"() {filename = "assets/foo.txt", sym_name = "asset"} : () -> () + // CHECK: @legacy_init_op + func.func @legacy_init_op(%arg0: tensor {tf_saved_model.bound_input = @asset}) attributes {tf_saved_model.exported_names = ["f"]} { + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "hash_table_shared_name", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + "tf.InitializeTableFromTextFileV2"(%0, %arg0) {delimiter = "\09", device = "", key_index = -2 : i64, value_index = -1 : i64, vocab_size = 205 : i64} : (tensor, tensor) -> () + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors_interprocedural.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors_interprocedural.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8244a69d7f8520707a829f379510fc2241420899 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_optimize_global_tensors_interprocedural.mlir @@ -0,0 +1,220 @@ +// RUN: tf-opt -tf-saved-model-optimize-global-tensors -split-input-file %s | FileCheck %s +//===----------------------------------------------------------------------===// +// Immutability. +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: This test exercises marking a global tensor as immutable after it propagates + // via set of chained calls -> f -> f_callee -> f_callee_callee + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + func.func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee_callee} : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %val : tensor + } + + func.func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> tensor + func.return %val : tensor + } +} + +// ----- + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: + // This test exercises trying to mark immutable when same func is called by multiple callers + // with different global tensors. + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v2", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_common} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + func.func @f2(%arg0: tensor>> {tf_saved_model.bound_input = @v2}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f2"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_common} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + func.func private @f_common(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> tensor + func.return %val : tensor + } +} + +// ----- + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: This test exercises immutability without explicit use + // via ReadVariableOp + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> (tensor) + %val_2 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> (tensor) + func.return %val_2 : tensor + } + + func.func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %cst_1 = arith.constant dense<2.0> : tensor + func.return %cst_1 : tensor + } +} + +// ----- + +//===----------------------------------------------------------------------===// +// Test case: Test mutation detection propagates across function calls +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + // CHECK: func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee_callee} : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %val : tensor + } + + // CHECK: func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %c0 = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + "tf.AssignVariableOp"(%arg0, %c0) : (tensor<*x!tf_type.resource>, tensor) -> () + func.return %c0 : tensor + } +} + +// ----- + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: The inter-procedural analysis with different types of + // TF call ops + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + // CHECK: func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) + func.func @f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["f"]} { + %val = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + // CHECK: func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f_callee_callee} : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %val : tensor + } + + // CHECK: func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f_callee_callee(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %c0 = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + "tf.AssignVariableOp"(%arg0, %c0) : (tensor<*x!tf_type.resource>, tensor) -> () + func.return %c0 : tensor + } +} + +// ----- + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: The inter-procedural analysis does not recurse infinitely + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-NOT: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @exported_f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["exported_f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + + // CHECK: func private @f(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @g} : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %val : tensor + } + + // CHECK: func private @g(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @g(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f} : (tensor<*x!tf_type.resource>) -> (tensor) + func.return %val : tensor + } +} + +// ----- + +// CHECK-LABEL: module attributes {tf_saved_model.semantics} +module attributes {tf_saved_model.semantics} { + + // Test case: Inter-procedural analysis with resource usage in an + // unknown op, we assume mutating behavior and propagate that. + + // CHECK: "tf_saved_model.global_tensor"() <{ + // CHECK-SAME: is_mutable + // CHECK-SAME: }> : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.> : tensor } : () -> () + + func.func @exported_f(%arg0: tensor>> {tf_saved_model.bound_input = @v}) -> (tensor {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["exported_f"]} { + %val = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @f} : (tensor>>) -> (tensor) + func.return %val : tensor + } + + + // CHECK: func private @f(%arg0: tensor<*x!tf_type.resource>) -> tensor + func.func private @f(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %c0 = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + "tf.AssignAddVariableOp"(%arg0, %c0) : (tensor<*x!tf_type.resource>, tensor) -> () + func.return %c0 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_remove_vars_in_session_initializer.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_remove_vars_in_session_initializer.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e63c34da89a30126a12c5eb75270ea3ed6b5c5e2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_remove_vars_in_session_initializer.mlir @@ -0,0 +1,108 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-saved-model-remove-vars-in-session-initializer | FileCheck %s + +module attributes {tf_saved_model.semantics} { + // Test case: No session initializer op +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // Test case: No matching function for the given session initializer. + // expected-error@+1 {{'tf_saved_model.session_initializer' op the initializer function does not exist}} + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // Test case: Invalid multiple blocks in the initializer funcion. + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + // expected-error@+1 {{expects exactly one block in the MLIR function}} + func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "restore_op"} { + cf.br ^bb1 + ^bb1: + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics} { + // Test case: No variables + // CHECK: func @init() + // CHECK: tf.Const + // CHECK: return + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "restore_op"} { + "tf.Const"() {value = dense<[1.0]> : tensor<1xf32> } : () -> tensor<1xf32> + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // Test case: Variable removal. + // CHECK: func @init() + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.Const + // CHECK-NOT: tf.AssignAddVariableOp + // CHECK: return + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "restore_op"} { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "w"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %3 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + "tf.AssignAddVariableOp"(%0, %2) {dtype = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignAddVariableOp"(%1, %3) {dtype = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // Test case: Removal of shared variables. + // CHECK: func @init() + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.Const + // CHECK-NOT: tf.AssignAddVariableOp + // CHECK: return + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "restore_op"} { + %0 = "tf.VarHandleOp"() {container = "c", shared_name = "v"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.VarHandleOp"() {container = "c", shared_name = "w"} : () -> tensor<*x!tf_type.resource>> + %2 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %3 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %4 = "tf.Add"(%2, %3) : (tensor, tensor) -> tensor + "tf.AssignAddVariableOp"(%0, %4) {dtype = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignAddVariableOp"(%1, %4) {dtype = "tfdtype$DT_INT32"} : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return + } +} + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // Test case: Variable removed for the initializer function whose + // initializer_type is "init_op". + "tf_saved_model.session_initializer"() { initializers = [@init] } : () -> () + + func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer"], tf_saved_model.initializer_type = "init_op"} { + %0 = "tf.VarHandleOp"() {shared_name = "vocab_file"} : () -> tensor<*x!tf_type.resource>> + %1 = "tf.Const"() {value = dense<"assets/vocab_file.txt"> : tensor} : () -> tensor + "tf.AssignVariableOp"(%0, %1) : (tensor<*x!tf_type.resource>>, tensor) -> () + %2 = "tf.HashTableV2"() {key_dtype = !tf_type.string, shared_name = "table", value_dtype = i64} : () -> tensor + %3 = "tf.ReadVariableOp"(%0) : (tensor<*x!tf_type.resource>>) -> tensor + "tf.InitializeTableFromTextFileV2"(%2, %3) {key_index = -2 : i64, value_index = -1 : i64} : (tensor, tensor) -> () + func.return + } + // CHECK: func @init() + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.Const + // CHECK-NOT: tf.AssignVariableOp + // CHECK-NOT: tf.HashTableV2 + // CHECK-NOT: tf.ReadVariableOp + // CHECK: return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_side_effect.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_side_effect.mlir new file mode 100644 index 0000000000000000000000000000000000000000..db35737fba528b47211400cede20ee3b3f29e4b4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_side_effect.mlir @@ -0,0 +1,11 @@ +// RUN: tf-opt %s -tf-standard-pipeline | FileCheck %s + +// CHECK-LABEL: removeDeadReadVariableOp +func.func @removeDeadReadVariableOp(%arg0: tensor>>) -> tensor { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %1 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + func.return %0: tensor + + // CHECK: %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: return %0 +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..1c0c0ea1fd9f8b7aeb36dd01ea4136894024b7e7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/BUILD @@ -0,0 +1,27 @@ +load("//tensorflow:tensorflow.default.bzl", "filegroup") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +licenses(["notice"]) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "@llvm-project//mlir:run_lit.sh", + test_file_exts = [ + "mlir", + "pbtxt", + ], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/sccp-post-shape-inference.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/sccp-post-shape-inference.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4f0fab2f95778fb42c25145f0174bb9191670485 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_to_hlo_pipeline/sccp-post-shape-inference.mlir @@ -0,0 +1,26 @@ +// RUN: tf-opt -tf-to-hlo-pipeline %s | FileCheck %s + +// Verifies that constants generated post shape inference are propagated. +// get_shape result in this test. +module attributes {tf.versions = {producer = 179 : i32}} { + + // CHECK-LABEL: func @main + func.func @main(%arg0: tensor<10x19xf32>, %arg1: tensor<19x10xf32> {mhlo.is_same_data_across_replicas = true}) -> tensor { + %0 = "tf.Shape"(%arg0) : (tensor<10x19xf32>) -> tensor<2xi64> + %1 = "tf.Reshape"(%arg1, %0) : (tensor<19x10xf32>, tensor<2xi64>) -> tensor + + // CHECK: %[[RESULT:.*]] = mhlo.constant dense<[10, 19]> + %2 = "tf.PartitionedCall"(%1) {config = "", config_proto = "", executor_type = "", f = @get_shape} : (tensor) -> (tensor) + + // CHECK: return %[[RESULT]] + func.return %2 : tensor + } + + // CHECK-LABEL: func @get_shape + func.func @get_shape(%arg0 : tensor<*xi64>) -> tensor { + %0 = "tf.Shape"(%arg0) : (tensor<*xi64>) -> tensor + func.return %0 : tensor + } + +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_trait_folds.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_trait_folds.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3a58b702a39eb96449cfe7d429ea052842fc35ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tf_trait_folds.mlir @@ -0,0 +1,146 @@ +// RUN: tf-opt %s -tf-standard-pipeline | FileCheck %s + +// CHECK-LABEL: func @testSingleConj +// CHECK-SAME: ([[ARG0:%.+]]: tensor>) +func.func @testSingleConj(%arg0: tensor>) -> tensor> { + // CHECK: [[CONJ:%.+]] = "tf.Conj"([[ARG0]]) + %0 = "tf.Conj"(%arg0) : (tensor>) -> tensor> + // CHECK: return [[CONJ]] + func.return %0: tensor> +} + +// CHECK-LABEL: func @testDoubleConj +// CHECK-SAME: ([[ARG0:%.+]]: tensor>) +func.func @testDoubleConj(%arg0: tensor>) -> tensor> { + %0 = "tf.Conj"(%arg0) : (tensor>) -> tensor> + %1 = "tf.Conj"(%0) : (tensor>) -> tensor> + // CHECK: return [[ARG0]] + func.return %1: tensor> +} + +// CHECK-LABEL: func @testTripleConj +// CHECK-SAME: ([[ARG0:%.+]]: tensor>) +func.func @testTripleConj(%arg0: tensor>) -> tensor> { + // CHECK: [[CONJ:%.+]] = "tf.Conj"([[ARG0]]) + %0 = "tf.Conj"(%arg0) : (tensor>) -> tensor> + %1 = "tf.Conj"(%0) : (tensor>) -> tensor> + %2 = "tf.Conj"(%1) : (tensor>) -> tensor> + // CHECK: return [[CONJ]] + func.return %2: tensor> +} + +// CHECK-LABEL: func @testSingleReciprocal +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testSingleReciprocal(%arg0: tensor) -> tensor { + // CHECK: [[RECIPROCAL:%.+]] = "tf.Reciprocal"([[ARG0]]) + %0 = "tf.Reciprocal"(%arg0) : (tensor) -> tensor + // CHECK: return [[RECIPROCAL]] + func.return %0: tensor +} + +// CHECK-LABEL: func @testDoubleReciprocal +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testDoubleReciprocal(%arg0: tensor) -> tensor { + %0 = "tf.Reciprocal"(%arg0) : (tensor) -> tensor + %1 = "tf.Reciprocal"(%0) : (tensor) -> tensor + // CHECK: return [[ARG0]] + func.return %1: tensor +} + +// CHECK-LABEL: func @testTripleReciprocal +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testTripleReciprocal(%arg0: tensor) -> tensor { + // CHECK: [[RECIPROCAL:%.+]] = "tf.Reciprocal"([[ARG0]]) + %0 = "tf.Reciprocal"(%arg0) : (tensor) -> tensor + %1 = "tf.Reciprocal"(%0) : (tensor) -> tensor + %2 = "tf.Reciprocal"(%1) : (tensor) -> tensor + // CHECK: return [[RECIPROCAL]] + func.return %2: tensor +} + +// CHECK-LABEL: func @testSingleInvert +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testSingleInvert(%arg0: tensor) -> tensor { + // CHECK: [[INVERT:%.+]] = "tf.Invert"([[ARG0]]) + %0 = "tf.Invert"(%arg0) : (tensor) -> tensor + // CHECK: return [[INVERT]] + func.return %0: tensor +} + +// CHECK-LABEL: func @testDoubleInvert +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testDoubleInvert(%arg0: tensor) -> tensor { + %0 = "tf.Invert"(%arg0) : (tensor) -> tensor + %1 = "tf.Invert"(%0) : (tensor) -> tensor + // CHECK: return [[ARG0]] + func.return %1: tensor +} + +// CHECK-LABEL: func @testTripleInvert +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testTripleInvert(%arg0: tensor) -> tensor { + // CHECK: [[INVERT:%.+]] = "tf.Invert"([[ARG0]]) + %0 = "tf.Invert"(%arg0) : (tensor) -> tensor + %1 = "tf.Invert"(%0) : (tensor) -> tensor + %2 = "tf.Invert"(%1) : (tensor) -> tensor + // CHECK: return [[INVERT]] + func.return %2: tensor +} + +// CHECK-LABEL: func @testSingleNeg +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testSingleNeg(%arg0: tensor) -> tensor { + // CHECK: [[NEG:%.+]] = "tf.Neg"([[ARG0]]) + %0 = "tf.Neg"(%arg0) : (tensor) -> tensor + // CHECK: return [[NEG]] + func.return %0: tensor +} + +// CHECK-LABEL: func @testDoubleNeg +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testDoubleNeg(%arg0: tensor) -> tensor { + %0 = "tf.Neg"(%arg0) : (tensor) -> tensor + %1 = "tf.Neg"(%0) : (tensor) -> tensor + // CHECK: return [[ARG0]] + func.return %1: tensor +} + +// CHECK-LABEL: func @testTripleNeg +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testTripleNeg(%arg0: tensor) -> tensor { + // CHECK: [[NEG:%.+]] = "tf.Neg"([[ARG0]]) + %0 = "tf.Neg"(%arg0) : (tensor) -> tensor + %1 = "tf.Neg"(%0) : (tensor) -> tensor + %2 = "tf.Neg"(%1) : (tensor) -> tensor + // CHECK: return [[NEG]] + func.return %2: tensor +} + +// CHECK-LABEL: func @testSingleLogicalNot +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testSingleLogicalNot(%arg0: tensor) -> tensor { + // CHECK: [[LNOT:%.+]] = "tf.LogicalNot"([[ARG0]]) + %0 = "tf.LogicalNot"(%arg0) : (tensor) -> tensor + // CHECK: return [[LNOT]] + func.return %0: tensor +} + +// CHECK-LABEL: func @testDoubleLogicalNot +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testDoubleLogicalNot(%arg0: tensor) -> tensor { + %0 = "tf.LogicalNot"(%arg0) : (tensor) -> tensor + %1 = "tf.LogicalNot"(%0) : (tensor) -> tensor + // CHECK: return [[ARG0]] + func.return %1: tensor +} + +// CHECK-LABEL: func @testTripleLogicalNot +// CHECK-SAME: ([[ARG0:%.+]]: tensor) +func.func @testTripleLogicalNot(%arg0: tensor) -> tensor { + // CHECK: [[LNOT:%.+]] = "tf.LogicalNot"([[ARG0]]) + %0 = "tf.LogicalNot"(%arg0) : (tensor) -> tensor + %1 = "tf.LogicalNot"(%0) : (tensor) -> tensor + %2 = "tf.LogicalNot"(%1) : (tensor) -> tensor + // CHECK: return [[LNOT]] + func.return %2: tensor +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tfrt_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tfrt_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3a694ab29cf737aabe6323f38a61626f550b44f7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tfrt_ops.mlir @@ -0,0 +1,27 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics | FileCheck %s + +// Tests for TensorFlow TFRT ops with custom verifiers. + +//===--------------------------------------------------------------------===// +// Test TF operations (tf.*) +//===--------------------------------------------------------------------===// + +// CHECK-LABEL: func @testPwStreamResults +func.func @testPwStreamResults(%arg0: tensor, %arg1: tensor) { + "tf.PwStreamResults"(%arg0, %arg1) {names = ["foo", "bar"]} : (tensor, tensor) -> () + return +} + +// ----- +// CHECK-LABEL: func @test_ifrt_call +func.func @test_ifrt_call(%arg0: tensor, %arg1: tensor) { + %result = "tf.IfrtCall"(%arg0, %arg1) <{operandSegmentSizes = array, program_id = 1234 : i64, variable_arg_indices = [0 : i32, 1 : i32], variable_names = ["a", "b"]}> : (tensor, tensor) -> (tensor<1x1xf32>) + func.return +} + +// ----- +func.func @test_ifrt_call_fail_unsorted_variable_arg_indices(%arg0: tensor, %arg1: tensor) { + // expected-error@below {{variable_arg_indices must be sorted in ascending order}} + %result = "tf.IfrtCall"(%arg0, %arg1) <{operandSegmentSizes = array, program_id = 1234 : i64, variable_arg_indices = [1 : i32, 0 : i32], variable_names = ["a", "b"]}> : (tensor, tensor) -> (tensor<1x1xf32>) + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-annotate-dynamic-shape-inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-annotate-dynamic-shape-inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2299d9c4339a114941eb965c8bda963bcbb7b209 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-annotate-dynamic-shape-inputs.mlir @@ -0,0 +1,31 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -tf-tpu-annotate-dynamic-shape-inputs %s | FileCheck %s + +// Test that annotate the inputs of the cluster func to be dynamic shaped. + +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @main( + %arg0: tensor<2048xi64> {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, + %arg1: tensor<2048xi64> {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) -> tensor<2048xi32> { + %cst = "tf.Const"() {value = dense<1024> : tensor} : () -> tensor + %0:2 = "tf_device.launch"() ({ + %1 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<2048xi64>) -> tensor<2048xi32> + %2 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2048xi64>) -> tensor<2048xi32> + %3:2 = "tf.TPUCopyWithDynamicShape"(%1, %2, %cst, %cst) {operandSegmentSizes = array} : (tensor<2048xi32>, tensor<2048xi32>, tensor, tensor) -> (tensor<2048xi32>, tensor<2048xi32>) + // CHECK-NOT: tf.TPUAnnotateTensorsWithDynamicShape + %4:2 = "tf.TPUAnnotateTensorsWithDynamicShape"(%3#0, %3#1) : (tensor<2048xi32>, tensor<2048xi32>) -> (tensor<2048xi32>, tensor<2048xi32>) + tf_device.return %4#0, %4#1 : tensor<2048xi32>, tensor<2048xi32> + }) {device = "TPU_REPLICATED_HOST_0"} : () -> (tensor<2048xi32>, tensor<2048xi32>) + %1 = "tf_device.cluster_func"(%0#0, %0#1) {_replication_info = "cluster_test_fn", func = @tpu_func} : (tensor<2048xi32>, tensor<2048xi32>) -> tensor<2048xi32> + return %1: tensor<2048xi32> + } + // CHECK-LABEL: func @tpu_func + // CHECK: mhlo.type_extensions + func.func @tpu_func ( + %arg0: tensor<2048xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<2048xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor<2048xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + // TODO(b/292540052): Below tf.addV2 instruction is replaced with just + // returning arg0 due to the workaround mentioned in the above bug. Revert + // this after the bug is fixed. + // %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<2048xi32>, tensor<2048xi32>) -> tensor<2048xi32> + return %arg0 : tensor<2048xi32> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-cluster-cleanup-attributes.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-cluster-cleanup-attributes.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7076921df92f0cde71e5c840adef53c35fb8f4e9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-cluster-cleanup-attributes.mlir @@ -0,0 +1,58 @@ +// RUN: tf-opt %s -tf-tpu-cleanup-cluster-attributes | FileCheck %s + +// CHECK-LABEL: func @control_flow_cleanup +func.func @control_flow_cleanup(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: "tf_device.cluster" + // CHECK-NOT: _replication_info = + // CHECK-NOT: _xla_compile_device_type + // CHECK-NOT: device = + %1 = "tf_device.cluster"() ({ + %2 = "tf.Add"(%arg1, %arg1) : (tensor, tensor) -> tensor + %3 = "tf.IfRegion"(%arg0) ({ + %4 = "tf.Mul" (%arg1, %2) {device = "y"}: (tensor, tensor) -> tensor + "tf.Yield"(%4) : (tensor) -> () + }, { + // CHECK: device = "/device:TPU_REPLICATED_CORE:0" + %5 = "tf.Div" (%arg1, %2) {device = "/device:TPU_REPLICATED_CORE:0"} : (tensor, tensor) -> tensor + "tf.Yield"(%5) : (tensor) -> () + }) {is_stateless = true, _xla_compile_device_type = "TPU", _replication_info = "x" } : (tensor) -> (tensor) + tf_device.return %3 : tensor + // CHECK: {_replication_info = "x", _xla_compile_device_type = "TPU", cluster_attr = "cluster_attr", device = "y"} + }) {cluster_attr = "cluster_attr", _xla_compile_device_type = "TPU", _replication_info = "x", device = "y"} : () -> tensor + // CHECK: "tf.Add" + // CHECK-SAME: {_replication_info = "x", _xla_compile_device_type = "TPU", device = "y"} + %2 = "tf.Add"(%arg2, %1) {_xla_compile_device_type = "TPU", _replication_info = "x", device = "y"} : (tensor, tensor) -> tensor + // CHECK: return + func.return %2 : tensor +} + +// CHECK-LABEL: func @skip_launch_device +func.func @skip_launch_device(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: "tf_device.cluster" + // CHECK: "tf_device.launch" + // CHECK-NOT: _replication_info = + // CHECK-NOT: _xla_compile_device_type + // CHECK: device = "y" + %1 = "tf_device.cluster"() ({ + %2 = "tf_device.launch"() ({ + %3 = "tf.Add"(%arg1, %arg2) : (tensor, tensor) -> tensor + tf_device.return %3 : tensor + }) {_xla_compile_device_type = "TPU", _replication_info = "x", device = "y"} : () -> tensor + tf_device.return %2 : tensor + }) {cluster_attr = "cluster_attr", _xla_compile_device_type = "TPU", _replication_info = "x", device = "y"} : () -> tensor + + func.return %1 : tensor +} + +// CHECK-LABEL: func @remove_class_attribute +func.func @remove_class_attribute(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: "tf_device.cluster" + // CHECK: "tf.Add" + // CHECK-NOT: _class + %1 = "tf_device.cluster"() ({ + %2 = "tf.Add"(%arg1, %arg2) {_class = ["loc:@while/BiasAdd_21/handle"]} : (tensor, tensor) -> tensor + tf_device.return %2 : tensor + }) {cluster_attr = "cluster_attr", _xla_compile_device_type = "TPU", _replication_info = "x", device = "y"} : () -> tensor + + func.return %1 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-dynamic-layout-pass.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-dynamic-layout-pass.mlir new file mode 100644 index 0000000000000000000000000000000000000000..74dc8b2f5cd3434d7f1869c92d24bf1f3c081284 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-dynamic-layout-pass.mlir @@ -0,0 +1,528 @@ +// RUN: tf-opt %s -split-input-file -tf-tpu-dynamic-layout-pass | FileCheck %s + +// Tests that the pass can transform non-replicated execution. + +// CHECK: func @non_replicated(%[[ARG0:.*]]: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor +func.func @non_replicated(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 1 : i64, is_output = false}> + // CHECK: %[[ITER:.*]]:2 = "tf.IteratorGetNext" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert" + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK-DAG: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER]]#0, %[[LAYOUT0]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER]]#1, %[[LAYOUT1]]) {device = "/device:TPU:0"} + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[COPY0]], %[[COPY1]], %[[COMPILE]]#1) + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass does not transform two execute ops sharing the same +// compile op. + +// CHECK-LABEL: func @multiple_compile_uses +func.func @multiple_compile_uses(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute0 = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + %4:2 = "tf._UnKnownOp_"() : () -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + %execute1 = "tf_device.launch"() ({ + %5 = "tf.TPUExecute"(%4#0, %4#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %5 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute1 : tensor +} + +// ----- + +// Tests that the pass does not transform when tf.IteratorGetNext is on TPU. + +// CHECK-LABEL: func @on_tpu_iter +func.func @on_tpu_iter(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:TPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:TPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass does not transform when tf.IteratorGetNext is on CPU +// but generator is on TPU. + +// CHECK-LABEL: func @arg_on_tpu_iter_on_cpu +func.func @arg_on_tpu_iter_on_cpu(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:TPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass does not transform when tf.IteratorGetNext is on CPU but +// generator is on TPU. All intermediate nodes like tf.Identity between +// generator and IteratorGetNext are on CPU too. + +// CHECK-LABEL: func @arg_on_tpu_intermediate_ops_on_cpu +func.func @arg_on_tpu_intermediate_ops_on_cpu(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:TPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + %id1 = "tf.Identity"(%arg0) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<*x!tf_type.resource>) + %id2 = "tf.Identity"(%id1) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<*x!tf_type.resource>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2:2 = "tf.IteratorGetNext"(%id2) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass does not transform when tf.IteratorGetNext is on CPU but +// generator is on TPU. + +// CHECK-LABEL: func @var_handle_on_tpu_iter_on_cpu +func.func @var_handle_on_tpu_iter_on_cpu() -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + %var = "tf.VarHandleOp"() {container = "c", shared_name = "v", device = "/device:TPU:0"} : () -> tensor>> + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2:2 = "tf.IteratorGetNext"(%var) {device = "/device:CPU:0"} + : (tensor>>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass does not change unsupported input ops. + +// CHECK-LABEL: func @unsupported_ops +func.func @unsupported_ops(%arg0: tensor<3x3x1x32xf32> {tf.device = "/device:CPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + %2 = "tf._Unknown_"() : () -> tensor<3x3x1x32xf32> + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %execute = "tf_device.launch"() ({ + %3 = "tf.TPUExecute"(%arg0, %2, %compile#1) + : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + func.return %execute : tensor +} + +// ----- + +// Tests that the pass can transform replicated execution. + +// CHECK: func @replicated(%[[ARG0:.*]]: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor +func.func @replicated(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + // CHECK: %[[ITER0:.*]]:2 = "tf.IteratorGetNext" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 1 : i64, is_output = false}> + // CHECK: %[[ITER1:.*]]:2 = "tf.IteratorGetNext" + %3:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK-DAG: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#0, %[[LAYOUT0]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#1, %[[LAYOUT1]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY2:.*]] = "tf.TPUCopyWithLayout"(%[[ITER1]]#0, %[[LAYOUT0]]) {device = "/device:TPU:1"} + // CHECK-DAG: %[[COPY3:.*]] = "tf.TPUCopyWithLayout"(%[[ITER1]]#1, %[[LAYOUT1]]) {device = "/device:TPU:1"} + // CHECK: tf_device.replicate([%[[COPY0]], %[[COPY2]]] as %[[R0:.*]]: tensor<3x3x1x32xf32>, [%[[COPY1]], %[[COPY3]]] as %[[R1:.*]]: tensor<3x3x1x32xf32>) + %5:2 = tf_device.replicate([%2#0, %3#0] as %r0: tensor<3x3x1x32xf32>, [%2#1, %3#1] as %r1: tensor<3x3x1x32xf32>) + {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}} { + // CHECK: "tf.TPUExecute"(%[[R0]], %[[R1]], %[[COMPILE]]#1) + %execute = "tf_device.launch"() ({ + %4 = "tf.TPUExecute"(%r0, %r1, %compile#1) : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %4 : tensor + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor + tf_device.return %execute : tensor + } + func.return %5#0 : tensor +} + +// ----- + +// Tests that the pass can transform replicated execution with packed inputs. + +// CHECK: func @replicated_packed(%[[ARG0:.*]]: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor +func.func @replicated_packed(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + // CHECK: %[[ITER0:.*]]:2 = "tf.IteratorGetNext" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 1 : i64, is_output = false}> + + // CHECK-DAG: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#0, %[[LAYOUT0]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#1, %[[LAYOUT1]]) {device = "/device:TPU:0"} + // CHECK: tf_device.replicate(%[[COPY0]] as %[[R0:.*]]: tensor<3x3x1x32xf32>, %[[COPY1]] as %[[R1:.*]]: tensor<3x3x1x32xf32>) + %5:2 = tf_device.replicate(%2#0 as %r0: tensor<3x3x1x32xf32>, %2#1 as %r1: tensor<3x3x1x32xf32>) + {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}} { + // CHECK: "tf.TPUExecute"(%[[R0]], %[[R1]], %[[COMPILE]]#1) + %execute = "tf_device.launch"() ({ + %4 = "tf.TPUExecute"(%r0, %r1, %compile#1) : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %4 : tensor + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor + tf_device.return %execute : tensor + } + func.return %5#0 : tensor +} + +// ----- + +// Tests that the pass can transform replicated execution with both replicated +// and packed operands. + +// CHECK: func @replicated(%[[ARG0:.*]]: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor +func.func @replicated(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}, %arg1: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + // CHECK: %[[ITER0:.*]]:2 = "tf.IteratorGetNext" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 1 : i64, is_output = false}> + // CHECK: %[[ITER1:.*]] = "tf.IteratorGetNext" + %3 = "tf.IteratorGetNext"(%arg1) {device = "/device:CPU:0"} + : (tensor<*x!tf_type.resource>) -> tensor<3x3x1x32xf32> + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK-DAG: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#0, %[[LAYOUT0]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#1, %[[LAYOUT1]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY2:.*]] = "tf.TPUCopyWithLayout"(%[[ITER1]], %[[LAYOUT0]]) {device = "/device:TPU:1"} + // CHECK: tf_device.replicate([%[[COPY0]], %[[COPY2]]] as %[[R0:.*]]: tensor<3x3x1x32xf32>, %[[COPY1]] as %[[R1:.*]]: tensor<3x3x1x32xf32>) + %5:2 = tf_device.replicate([%2#0, %3] as %r0: tensor<3x3x1x32xf32>, %2#1 as %r1: tensor<3x3x1x32xf32>) + {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}} { + // CHECK: "tf.TPUExecute"(%[[R0]], %[[R1]], %[[COMPILE]]#1) + %execute = "tf_device.launch"() ({ + %4 = "tf.TPUExecute"(%r0, %r1, %compile#1) : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %4 : tensor + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor + tf_device.return %execute : tensor + } + func.return %5#0 : tensor +} + +// ----- + +// Tests that the pass does not change inputs inside replicate. + +// CHECK-LABEL: func @inside_replicated +func.func @inside_replicated(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}, %arg1: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) -> tensor { + %compile:2 = "tf_device.launch"() ({ + %1:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and two return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK-NOT: "tf.TPUGetLayoutOp" + // CHECK-NOT: "tf.TPUCopyWithLayout" + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %5:2 = tf_device.replicate([%arg0, %arg1] as %r0: tensor<*x!tf_type.resource>) + {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}} { + %2:2 = "tf.IteratorGetNext"(%r0) + : (tensor<*x!tf_type.resource>) -> (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>) + %execute = "tf_device.launch"() ({ + %4 = "tf.TPUExecute"(%2#0, %2#1, %compile#1) : (tensor<3x3x1x32xf32>, tensor<3x3x1x32xf32>, tensor<2x!tf_type.string>) -> tensor + tf_device.return %4 : tensor + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor + tf_device.return %execute : tensor + } + func.return %5#0 : tensor +} + +// ----- + +// Tests that the pass can transform execution with model parallelism and no +// replication. +// +// The following TPUCompileMetadataProto is used: +// args { +// dtype: DT_FLOAT +// shape { +// dim { +// size: 128 +// } +// } +// } +// num_replicas: 1 +// num_cores_per_replica: 2 + +// CHECK-LABEL: func @parallel_execute +func.func @parallel_execute(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) { + // CHECK: %[[COMPILE:.*]]:3 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:3 = "tf_device.launch"() ({ + %1:3 = "tf._TPUCompileMlir"() {NumDynamicShapes = 0 : i64, metadata = "\0A\09\08\01\12\05\12\03\08\80\01\18\01 \02", mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1, %1#2 : tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#2) <{index = 0 : i64, is_output = false}> + // CHECK: %[[ITER:.*]]:2 = "tf.IteratorGetNext" + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<128xf32>, tensor<128xf32>) + // CHECK: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK: "tf_device.parallel_execute" + "tf_device.parallel_execute"() ({ + // CHECK-NEXT: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER]]#0, %[[LAYOUT0]]) + // CHECK-SAME: device = "/device:TPU:0" + // CHECK-NEXT: "tf_device.launch" + // CHECK-SAME: device = "/device:TPU:0" + // CHECK-NEXT: "tf.TPUExecute"(%[[COPY0]], %[[COMPILE]]#1) + // CHECK-NEXT: tf_device.return + "tf_device.launch"() ({ + "tf.TPUExecute"(%2#0, %compile#1) : (tensor<128xf32>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "/device:TPU:0"} : () -> () + tf_device.return + }, + { + // CHECK: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER]]#1, %[[LAYOUT1]]) + // CHECK-SAME: device = "/device:TPU:1" + // CHECK-NEXT: "tf_device.launch" + // CHECK-SAME: device = "/device:TPU:1" + // CHECK-NEXT: "tf.TPUExecute"(%[[COPY1]], %[[COMPILE]]#2) + // CHECK-NEXT: tf_device.return + "tf_device.launch"() ({ + "tf.TPUExecute"(%2#1, %compile#2) : (tensor<128xf32>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "/device:TPU:1"} : () -> () + tf_device.return + }) {} : () -> () + func.return +} + +// ----- + +// Tests that the pass can transform execution with model parallelism and +// replication. +// +// The following TPUCompileMetadataProto is used: +// args { +// dtype: DT_FLOAT +// shape { +// dim { +// size: 128 +// } +// } +// } +// num_replicas: 2 +// num_cores_per_replica: 2 + +// CHECK-LABEL: func @replicated_parallel_execute +// CHECK-SAME: %[[ARG0:[a-z0-9]+]]: tensor<*x!tf_type.resource> +// CHECK-SAME: %[[ARG1:[a-z0-9]+]]: tensor<*x!tf_type.resource> +func.func @replicated_parallel_execute(%arg0: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}, %arg1: tensor<*x!tf_type.resource> {tf.device = "/device:CPU:0"}) { + // CHECK: %[[COMPILE:.*]]:3 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:3 = "tf_device.launch"() ({ + %1:3 = "tf._TPUCompileMlir"() {NumDynamicShapes = 0 : i64, metadata = "\0A\09\08\01\12\05\12\03\08\80\01\18\02 \02", mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) + tf_device.return %1#0, %1#1, %1#2 : tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) + // CHECK-DAG: %[[LAYOUT0:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#1) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[LAYOUT1:.*]] = "tf.TPUGetLayoutOp"(%[[COMPILE]]#2) <{index = 0 : i64, is_output = false}> + // CHECK-DAG: %[[ITER0:.*]]:2 = "tf.IteratorGetNext"(%[[ARG0]]) + // CHECK-DAG: %[[ITER1:.*]]:2 = "tf.IteratorGetNext"(%[[ARG1]]) + %2:2 = "tf.IteratorGetNext"(%arg0) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<128xf32>, tensor<128xf32>) + %3:2 = "tf.IteratorGetNext"(%arg1) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<128xf32>, tensor<128xf32>) + // CHECK: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK-DAG: %[[COPY0:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#0, %[[LAYOUT0]]) {device = "/device:TPU:0"} + // CHECK-DAG: %[[COPY1:.*]] = "tf.TPUCopyWithLayout"(%[[ITER1]]#0, %[[LAYOUT0]]) {device = "/device:TPU:1"} + // CHECK-DAG: %[[COPY2:.*]] = "tf.TPUCopyWithLayout"(%[[ITER0]]#1, %[[LAYOUT1]]) {device = "/device:TPU:2"} + // CHECK-DAG: %[[COPY3:.*]] = "tf.TPUCopyWithLayout"(%[[ITER1]]#1, %[[LAYOUT1]]) {device = "/device:TPU:3"} + // CHECK-NEXT: tf_device.replicate + // CHECK-SAME: ([%[[COPY0]], %[[COPY1]]] as %[[R0:[a-z0-9]+]]: tensor<128xf32>, [%[[COPY2]], %[[COPY3]]] as %[[R1:[a-z0-9]+]]: tensor<128xf32>) + tf_device.replicate([%2#0, %3#0] as %r0: tensor<128xf32>, [%2#1, %3#1] as %r1: tensor<128xf32>) {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"], TPU_REPLICATED_CORE_1 = ["/device:TPU:2", "/device:TPU:3"]}} { + // CHECK: "tf_device.parallel_execute" + "tf_device.parallel_execute"() ({ + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK: "tf.TPUExecute"(%[[R0]], %[[COMPILE]]#1) + // CHECK-NEXT: tf_device.return + "tf_device.launch"() ({ + "tf.TPUExecute"(%r0, %compile#1) : (tensor<128xf32>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + }, + { + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_1" + // CHECK: "tf.TPUExecute"(%[[R1]], %[[COMPILE]]#2) + // CHECK-NEXT: tf_device.return + "tf_device.launch"() ({ + "tf.TPUExecute"(%r1, %compile#2) : (tensor<128xf32>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_1"} : () -> () + tf_device.return + }) {} : () -> () + tf_device.return + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-merge-variables-with-execute.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-merge-variables-with-execute.mlir new file mode 100644 index 0000000000000000000000000000000000000000..522deb93a6893b22c92e30cd2d78447d3a06d451 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-merge-variables-with-execute.mlir @@ -0,0 +1,403 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -tf-tpu-merge-variables-with-execute %s | FileCheck %s + +// Tests that the pass merges only variable reads/writes on the same device. + +// CHECK-LABEL: func @merge_same_device_variables +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_2:.*]]: tensor<*x!tf_type.resource>> +func.func @merge_same_device_variables( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg2: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) { + // CHECK-NEXT: %[[ID_0:.*]] = "tf.IdentityN"(%[[ARG_0]]) + %id0 = "tf.IdentityN"(%arg0) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} + : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + // CHECK-NEXT: %[[READ_2:.*]] = "tf.ReadVariableOp"(%[[ARG_2]]) + %read0 = "tf.ReadVariableOp"(%id0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<64xf32> + %read2 = "tf.ReadVariableOp"(%arg2) : (tensor<*x!tf_type.resource>>) -> tensor<16xf32> + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + %compile:2 = "tf_device.launch"() ({ + // CHECK: tf._TPUCompileMlir + %0:2 = "tf._TPUCompileMlir"() { + metadata = "", + mlir_module = "module attributes {tf.versions = {producer = 888 : i32}} {\0A func @main(%arg0: tensor<32xf32>, %arg1: tensor<64xf32>, %arg2: tensor<16xf32>) -> (tensor<32xf32>, tensor<16xf32>) {\0A %0:2 = \22tf.A\22(%arg0, %arg1, %arg2) : (tensor<32xf32>, tensor<64xf32>, tensor<16xf32>) -> (tensor<32xf32>, tensor<16xf32>)\0A return %0#0, %0#1 : tensor<32xf32>, tensor<16xf32>\0A }\0A}" + } : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %0#0, %0#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK: %[[EXE:.*]] = "tf_device.launch" + // CHECK-SAME: <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[ID_0]], %[[ARG_1]], %[[READ_2]], %[[COMPILE]]#1) + // CHECK-SAME: device_var_reads_indices = [0, 1], + // CHECK-SAME: device_var_updates_indices = [0, -1] + %execute:2 = "tf_device.launch"() ({ + %0:2 = "tf.TPUExecute"(%read0, %read1, %read2, %compile#1) { + Targs = [tensor<32xf32>, tensor<64xf32>, tensor<16xf32>], + Tresults = [tensor<32xf32>, tensor<16xf32>]} + : (tensor<32xf32>, tensor<64xf32>, tensor<16xf32>, tensor<2x!tf_type.string>) -> (tensor<32xf32>, tensor<16xf32>) + tf_device.return %0#0, %0#1 : tensor<32xf32>, tensor<16xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> (tensor<32xf32>, tensor<16xf32>) + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + "tf.AssignVariableOp"(%id0, %execute#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_2]], %[[EXE]]) + "tf.AssignVariableOp"(%arg2, %execute#1) : (tensor<*x!tf_type.resource>>, tensor<16xf32>) -> () + // CHECK-NEXT: return + func.return +} + +// ----- + +// Tests that the pass do not check devices for replicated region. + +// CHECK-LABEL: func @merge_replicated_variables +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>>, %[[ARG_1:.*]]: tensor<*x!tf_type.resource>>, +// CHECK-SAME: %[[ARG_2:.*]]: tensor<*x!tf_type.resource>> +func.func @merge_replicated_variables( + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: %[[READ_0:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + %compile:2 = "tf_device.launch"() ({ + // CHECK: tf._TPUCompileMlir + %0:2 = "tf._TPUCompileMlir"() { + metadata = "", + mlir_module = "module attributes {tf.versions = {producer = 888 : i32}} {\0A func @main(%arg0: tensor<32xf32>, %arg1: tensor<32xf32>) -> (tensor<32xf32>) {\0A %0 = \22tf.A\22(%arg0, %arg1) : (tensor<32xf32>, tensor<32xf32>) -> (tensor<32xf32>)\0A return %0 : tensor<32xf32>\0A }\0A}" + } : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %0#0, %0#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK: tf_device.replicate([%[[ARG_1]], %[[ARG_2]]] as %[[R_ARG:.*]]: tensor<*x!tf_type.resource>>) + tf_device.replicate([%arg1, %arg2] as %r: tensor<*x!tf_type.resource>>) {n = 2 : i32} { + // CHECK-NEXT: "tf_device.launch" + // CHECK-SAME: <{device = ""}> + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[READ_0]], %[[R_ARG]], %[[COMPILE]]#1) + // CHECK-SAME: device_var_reads_indices = [1], + // CHECK-SAME: device_var_updates_indices = [0] + %read1 = "tf.ReadVariableOp"(%r) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %execute = "tf_device.launch"() ({ + %0 = "tf.TPUExecute"(%read0, %read1, %compile#1) + : (tensor<32xf32>, tensor<32xf32>, tensor<2x!tf_type.string>) -> tensor<32xf32> + tf_device.return %0 : tensor<32xf32> + }) {device = ""} : () -> tensor<32xf32> + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + "tf.AssignVariableOp"(%r, %execute) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: tf_device.return + tf_device.return + // CHECK-NEXT: } + } + // CHECK-NEXT: return + func.return +} + +// ----- + +// Tests that the pass do not merge reads/assigns if there are interfering +// accesses in between. + +// CHECK-LABEL: func @interfering_accesses +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_2:.*]]: tensor<32xf32> +// CHECK-SAME: %[[ARG_4:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_5:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_6:.*]]: tensor<2xf32> +func.func @interfering_accesses( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg2: tensor<32xf32>, + %arg4: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg5: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg6: tensor<2xf32>) -> (tensor<8xf32>) { + // CHECK-NEXT: %[[READ_0:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // CHECK-NEXT: %[[READ_5:.*]] = "tf.ReadVariableOp"(%[[ARG_5]]) + %read5 = "tf.ReadVariableOp"(%arg5) : (tensor<*x!tf_type.resource>>) -> tensor<2xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_0]], %[[ARG_2]]) + "tf.AssignVariableOp"(%arg0, %arg2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_5]], %[[ARG_6]]) + "tf.AssignVariableOp"(%arg5, %arg6) : (tensor<*x!tf_type.resource>>, tensor<2xf32>) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<64xf32> + %read2 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + %compile:2 = "tf_device.launch"() ({ + // CHECK: tf._TPUCompileMlir + %0:2 = "tf._TPUCompileMlir"() { + metadata = "", + mlir_module = "module attributes {tf.versions = {producer = 888 : i32}} {\0A func @main(%arg0: tensor<32xf32>, %arg1: tensor<32xf32>) -> (tensor<32xf32>) {\0A %0 = \22tf.A\22(%arg0, %arg1) : (tensor<32xf32>, tensor<32xf32>) -> (tensor<32xf32>)\0A return %0 : tensor<32xf32>\0A }\0A}" + } : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %0#0, %0#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK: %[[EXE:.*]]:2 = "tf_device.launch" + // CHECK-SAME: <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[READ_0]], %[[ARG_1]], %[[ARG_4]], %[[READ_5]], %[[COMPILE]]#1) + // CHECK-SAME: device_var_reads_indices = [1, 2], + // CHECK-SAME: device_var_updates_indices = [1, -1] + %execute:3 = "tf_device.launch"() ({ + %0:3 = "tf.TPUExecute"(%read0, %read1, %read2, %read5, %compile#1) { + Targs = [tensor<32xf32>, tensor<64xf32>, tensor<8xf32>, tensor<2xf32>], + Tresults = [tensor<32xf32>, tensor<64xf32>, tensor<8xf32>]} + : (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>, tensor<2xf32>, tensor<2x!tf_type.string>) + -> (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>) + tf_device.return %0#0, %0#1, %0#2 : tensor<32xf32>, tensor<64xf32>, tensor<8xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>) + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + "tf.AssignVariableOp"(%arg1, %execute#1) : (tensor<*x!tf_type.resource>>, tensor<64xf32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_0]], %[[EXE]]#0) + "tf.AssignVariableOp"(%arg0, %execute#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: %[[READ_3:.*]] = "tf.ReadVariableOp"(%[[ARG_4]]) + %read3 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_4]], %[[EXE]]#1) + "tf.AssignVariableOp"(%arg4, %execute#2) : (tensor<*x!tf_type.resource>>, tensor<8xf32>) -> () + // CHECK-NEXT: return %[[READ_3]] + func.return %read3 : tensor<8xf32> +} + +// ----- + +// Tests that the pass does merge reads/assigns if there are resource accesses +// in between which are not interfering. +// For %arg0 and %arg1 both reads and assigns are merged, for %arg3 and %arg4 +// only reads are merged. + +// CHECK-LABEL: func @non_interfering_accesses +// CHECK-SAME: %[[ARG_0:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_2:[a-z0-9]+]]: tensor<32xf32> +// CHECK-SAME: %[[ARG_3:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_4:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_5:[a-z0-9]+]]: tensor<2xf32> +// CHECK-SAME: %[[ARG_6:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_7:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +func.func @non_interfering_accesses( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg2: tensor<32xf32>, + %arg3: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg4: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg5: tensor<2xf32>, + %arg6: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg7: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor<8xf32>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %read5 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor<2xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_6]], %[[ARG_2]]) + "tf.AssignVariableOp"(%arg6, %arg2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_7]], %[[ARG_5]]) + "tf.AssignVariableOp"(%arg7, %arg5) : (tensor<*x!tf_type.resource>>, tensor<2xf32>) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<64xf32> + %read2 = "tf.ReadVariableOp"(%arg3) : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + %compile:2 = "tf_device.launch"() ({ + // CHECK: tf._TPUCompileMlir + %0:2 = "tf._TPUCompileMlir"() { + metadata = "", + mlir_module = "module attributes {tf.versions = {producer = 888 : i32}} {\0A func @main(%arg0: tensor<32xf32>, %arg1: tensor<32xf32>) -> (tensor<32xf32>) {\0A %0 = \22tf.A\22(%arg0, %arg1) : (tensor<32xf32>, tensor<32xf32>) -> (tensor<32xf32>)\0A return %0 : tensor<32xf32>\0A }\0A}" + } : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %0#0, %0#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + // CHECK: %[[EXE:.*]] = "tf_device.launch" + // CHECK-SAME: <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[ARG_0]], %[[ARG_1]], %[[ARG_3]], %[[ARG_4]], %[[COMPILE]]#1) + // CHECK-SAME: device_var_reads_indices = [0, 1, 2, 3], + // CHECK-SAME: device_var_updates_indices = [0, 1, -1, -1] + %execute:3 = "tf_device.launch"() ({ + %0:3 = "tf.TPUExecute"(%read0, %read1, %read2, %read5, %compile#1) { + Targs = [tensor<32xf32>, tensor<64xf32>, tensor<8xf32>, tensor<2xf32>], + Tresults = [tensor<32xf32>, tensor<64xf32>, tensor<8xf32>]} + : (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>, tensor<2xf32>, tensor<2x!tf_type.string>) + -> (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>) + tf_device.return %0#0, %0#1, %0#2 : tensor<32xf32>, tensor<64xf32>, tensor<8xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> (tensor<32xf32>, tensor<64xf32>, tensor<8xf32>) + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + // CHECK-NEXT: %[[READ:.*]] = "tf.ReadVariableOp"(%[[ARG_3]]) + %read3 = "tf.ReadVariableOp"(%arg3) : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_3]], %[[EXE]]) + "tf.AssignVariableOp"(%arg3, %execute#2) : (tensor<*x!tf_type.resource>>, tensor<8xf32>) -> () + "tf.AssignVariableOp"(%arg1, %execute#1) : (tensor<*x!tf_type.resource>>, tensor<64xf32>) -> () + "tf.AssignVariableOp"(%arg0, %execute#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: return %[[READ]] + func.return %read3 : tensor<8xf32> +} + +// ----- + +// Tests that the pass does not merge for an execute node that has multiple +// inputs read from the same variable. + +// CHECK-LABEL: func @do_not_merge_multi_read +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:.*]]: tensor +func.func @do_not_merge_multi_read( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor) { + // CHECK-NEXT: %[[READ_0:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // CHECK-NEXT: %[[READ_1:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) + %read1 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // CHECK-NEXT: %[[EXE:.*]] = "tf_device.launch" + // CHECK-SAME: <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_0]], %[[READ_1]], %[[ARG_1]]) + %execute = "tf_device.launch"() ({ + %0 = "tf.TPUExecute"(%read0, %read1, %arg1) { + Targs = [tensor<32xf32>, tensor<32xf32>], Tresults = [tensor<32xf32>]} + : (tensor<32xf32>, tensor<32xf32>, tensor) -> (tensor<32xf32>) + tf_device.return %0 : tensor<32xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> tensor<32xf32> + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_0]], %[[EXE]]) + "tf.AssignVariableOp"(%arg0, %execute) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: return + func.return +} + +// ----- + +// Tests that the pass do not merge for an execute node that has multiple +// outputs used to update a variable. + +// CHECK-LABEL: func @do_not_merge_multi_assign +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:.*]]: tensor +func.func @do_not_merge_multi_assign( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor) { + // CHECK-NEXT: %[[READ_0:.*]] = "tf.ReadVariableOp"(%[[ARG_0]]) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + // CHECK-NEXT: %[[EXE:.*]]:2 = "tf_device.launch" + // CHECK-SAME: <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_0]], %[[ARG_1]]) + %execute:2 = "tf_device.launch"() ({ + %0:2 = "tf.TPUExecute"(%read0, %arg1) { + Targs = [tensor<32xf32>], Tresults = [tensor<32xf32>, tensor<32xf32>]} + : (tensor<32xf32>, tensor) -> (tensor<32xf32>, tensor<32xf32>) + tf_device.return %0#0, %0#1 : tensor<32xf32>, tensor<32xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> (tensor<32xf32>, tensor<32xf32>) + // CHECK-NEXT: tf_device.return + // CHECK-NEXT: }) + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_0]], %[[EXE]]#0) + "tf.AssignVariableOp"(%arg0, %execute#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: "tf.AssignVariableOp"(%[[ARG_0]], %[[EXE]]#1) + "tf.AssignVariableOp"(%arg0, %execute#1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + // CHECK-NEXT: return + func.return +} + +// ----- + +// Tests that the pass merges only variable reads/writes on the same device, +// with TPUExecutes in a tf_device.parallel_execute. + +// CHECK-LABEL: func @parallel_execute +// CHECK-SAME: %[[ARG_0:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:.*]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_2:.*]]: tensor +func.func @parallel_execute( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:1"}, + %arg2: tensor) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<64xf32> + // CHECK-NOT: "tf.ReadVariableOp" + // CHECK: "tf_device.parallel_execute" + %pe:2 = "tf_device.parallel_execute"() ({ + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %execute0 = "tf_device.launch"() ({ + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[ARG_0]], %[[ARG_2]]) + %0 = "tf.TPUExecute"(%read0, %arg2) : (tensor<32xf32>, tensor) -> tensor<32xf32> + // CHECK-NEXT: tf_device.return + tf_device.return %0 : tensor<32xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> tensor<32xf32> + tf_device.return %execute0 : tensor<32xf32> + }, { + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:1" + %execute1 = "tf_device.launch"() ({ + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[ARG_1]], %[[ARG_2]]) + %1 = "tf.TPUExecute"(%read1, %arg2) : (tensor<64xf32>, tensor) -> tensor<64xf32> + // CHECK-NEXT: tf_device.return + tf_device.return %1 : tensor<64xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:1"} : () -> tensor<64xf32> + tf_device.return %execute1 : tensor<64xf32> + }) : () -> (tensor<32xf32>, tensor<64xf32>) + // CHECK-NOT: "tf.AssignVariableOp" + "tf.AssignVariableOp"(%arg0, %pe#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %pe#1) : (tensor<*x!tf_type.resource>>, tensor<64xf32>) -> () + func.return +} + +// ----- + +// Tests that the pass merges variable reads/writes for TPUExecutes in a +// tf_device.parallel_execute that is replicated (tf_device.replicate). + +// CHECK-LABEL: func @replicated_parallel_execute +// CHECK-SAME: %[[ARG_0:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_1:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_2:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_3:[a-z0-9]+]]: tensor<*x!tf_type.resource>> +// CHECK-SAME: %[[ARG_4:[a-z0-9]+]]: tensor +func.func @replicated_parallel_execute( + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<*x!tf_type.resource>>, + %arg3: tensor<*x!tf_type.resource>>, + %arg4: tensor) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]+]]: tensor<*x!tf_type.resource>> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]+]]: tensor<*x!tf_type.resource>> + tf_device.replicate([%arg0, %arg1] as %ri0: tensor<*x!tf_type.resource>>, + [%arg2, %arg3] as %ri1: tensor<*x!tf_type.resource>>) {n = 2 : i32} { + // CHECK-NOT: "tf.ReadVariableOp" + %read0 = "tf.ReadVariableOp"(%ri0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %read1 = "tf.ReadVariableOp"(%ri1) : (tensor<*x!tf_type.resource>>) -> tensor<64xf32> + // CHECK: "tf_device.parallel_execute" + %pe:2 = "tf_device.parallel_execute"() ({ + // CHECK: "tf_device.launch" + %execute0 = "tf_device.launch"() ({ + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[RI_0]], %[[ARG_4]]) + %0 = "tf.TPUExecute"(%read0, %arg4) : (tensor<32xf32>, tensor) -> tensor<32xf32> + // CHECK-NEXT: tf_device.return + tf_device.return %0 : tensor<32xf32> + }) {device = ""} : () -> tensor<32xf32> + tf_device.return %execute0 : tensor<32xf32> + }, { + // CHECK: "tf_device.launch" + %execute1 = "tf_device.launch"() ({ + // CHECK-NEXT: "tf.TPUExecuteAndUpdateVariables"(%[[RI_1]], %[[ARG_4]]) + %1 = "tf.TPUExecute"(%read1, %arg4) : (tensor<64xf32>, tensor) -> tensor<64xf32> + // CHECK-NEXT: tf_device.return + tf_device.return %1 : tensor<64xf32> + }) {device = ""} : () -> tensor<64xf32> + tf_device.return %execute1 : tensor<64xf32> + }) : () -> (tensor<32xf32>, tensor<64xf32>) + // CHECK-NOT: "tf.AssignVariableOp" + "tf.AssignVariableOp"(%ri0, %pe#0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%ri1, %pe#1) : (tensor<*x!tf_type.resource>>, tensor<64xf32>) -> () + } + func.return +} + +// ----- + +// Tests that resource variables not hoisted are flagged. + +func.func @missing_read_write( + %arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, + %arg1: tensor) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %execute:2 = "tf_device.launch"() ({ + // expected-error @+1 {{resource that was neither read nor written to}} + %0:2 = "tf.TPUExecute"(%arg0, %read0, %arg1) + : (tensor<*x!tf_type.resource>>, tensor<32xf32>, tensor) -> (tensor<32xf32>, tensor<32xf32>) + tf_device.return %0#0, %0#1 : tensor<32xf32>, tensor<32xf32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> (tensor<32xf32>, tensor<32xf32>) + "tf.AssignVariableOp"(%arg0, %execute#1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-multiple-while-body-func.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-multiple-while-body-func.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5ead703e108d62e11c6a13c02a257d22089d5843 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-multiple-while-body-func.mlir @@ -0,0 +1,34 @@ +// RUN: tf-opt %s -tf-replicated-clustering-bridge-v2 -tfrt-lower-cluster-to-runtime-ops-tpu 2>&1 | FileCheck %s +// RUN: tf-opt %s -tf-replicated-clustering-bridge-v1 -tfrt-lower-cluster-to-runtime-ops-tpu -tf-dialect-to-executor-v1 2>&1 | FileCheck %s + +// This test verifies there is no warning about shape inference failure in TPU +// bridge in handling multiple usage of the same function. + +// Since it is possible that this warning may become an error in the future, +// only check the message content here. + +// CHECK-NOT: expected control flow function {{.*}} to have exactly 1 use, found 2 + +"builtin.module"() ({ + "func.func"() ({ + ^bb0(%arg0: tensor): + %1617 = "tf.While"(%arg0) {_lower_using_switch_merge = true, _num_original_outputs = 7 : i64, _read_only_resource_inputs = [], body = @main_while_body_4225150, cond = @main_while_cond_4225140, device = "", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor) -> (tensor) + "func.return"() : () -> () + }) {sym_name = "__inference_wrapped_function_4260250", sym_visibility = "private", tf._input_shapes = [#tf_type.shape<>], tf.signature.is_stateful, function_type = (tensor) -> ()} : () -> () + "func.func"() ({ + ^bb0(%arg0: tensor): + %1617 = "tf.While"(%arg0) {_lower_using_switch_merge = true, _num_original_outputs = 7 : i64, _read_only_resource_inputs = [], body = @main_while_body_4225150, cond = @main_while_cond_4225140, device = "", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor) -> (tensor) + "func.return"() : () -> () + }) {sym_name = "__inference_wrapped_function_4260250_0", sym_visibility = "private", tf._input_shapes = [#tf_type.shape<>], tf.signature.is_stateful, function_type = (tensor) -> ()} : () -> () + "func.func"() ({ + ^bb0(%arg0: tensor): + "func.return"(%arg0) : (tensor) -> () + }) {sym_name = "main_while_body_4225150", sym_visibility = "private", tf._input_shapes = [#tf_type.shape<>], tf.signature.is_stateful, function_type = (tensor) -> (tensor)} : () -> () + "func.func"() ({ + ^bb0(%arg0: tensor): + %0 = "tf.Less"(%arg0, %arg0) {device = ""} : (tensor, tensor) -> tensor<*xi1> + %1 = "tf.Less"(%arg0, %arg0) {device = ""} : (tensor, tensor) -> tensor<*xi1> + %2 = "tf.LogicalAnd"(%0, %1) {device = ""} : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + "func.return"(%2) : (tensor<*xi1>) -> () + }) {sym_name = "main_while_cond_4225140", sym_visibility = "private", tf._input_shapes = [#tf_type.shape<>], function_type = (tensor) -> tensor<*xi1>} : () -> () +}) {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 651 : i32}, tf_saved_model.semantics} : () -> () diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-resource-read-for-write.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-resource-read-for-write.mlir new file mode 100644 index 0000000000000000000000000000000000000000..392057672695f99e646278d6e17bcf6cc2f90c8d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-resource-read-for-write.mlir @@ -0,0 +1,91 @@ +// RUN: tf-opt -tf-tpu-resource-read-for-write %s | FileCheck %s --dump-input=always + +// CHECK-LABEL: func @write_only_resource +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor<*x!tf_type.resource>>) +func.func @write_only_resource(%arg0: tensor, %arg1: tensor, %arg2: tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: [[READ:%.*]] = "tf.ReadVariableOp"([[ARG2]]) + // CHECK-NEXT: [[CLUSTER:%.*]]:2 = "tf_device.cluster_func"([[ARG0]], [[ARG1]], [[READ]]) + // CHECK-SAME: _replication_info = "write", _xla_compile_device_type = "TPU" + %0:2 = "tf_device.cluster_func"(%arg0, %arg1) {_replication_info = "write", _xla_compile_device_type = "TPU", func = @write_func} : (tensor, tensor) -> (tensor, tensor) + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG2]], [[CLUSTER]]#1) + "tf.AssignVariableOp"(%arg2, %0#1) : (tensor<*x!tf_type.resource>>, tensor) -> () + // CHECK-NEXT: return + func.return +} + +// CHECK-LABEL: func @write_func +// CHECK-SAME: ({{%.*}}: tensor, {{%.*}}: tensor, {{%.*}}: tensor) -> (tensor, tensor) +func.func @write_func(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + func.return %arg1, %arg0 : tensor, tensor +} + +// CHECK-LABEL: func @read_write_resource +func.func @read_write_resource(%arg0: tensor, %arg1: tensor, %arg2: tensor<*x!tf_type.resource>>) { + // CHECK-COUNT-1: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg2) : (tensor<*x!tf_type.resource>>) -> tensor + %1:2 = "tf_device.cluster_func"(%arg0, %arg1, %0) {_xla_compile_device_type = "TPU", _replication_info = "read_write", func = @read_write_func} : (tensor, tensor, tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg2, %1#1) : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @read_write_func +// CHECK-SAME: ({{%.*}}: tensor, {{%.*}}: tensor) -> (tensor, tensor) +func.func @read_write_func(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + func.return %arg1, %arg0 : tensor, tensor +} + +// CHECK-LABEL: func @multiple_write_resource +func.func @multiple_write_resource(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>) { + // CHECK-NOT: tf.ReadVariableOp + %0:2 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "multiple_write", func = @multiple_write_func} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg1, %0#0) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %0#1) : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @multiple_write_func +// CHECK-SAME: ({{%.*}}: tensor) -> (tensor, tensor) +func.func @multiple_write_func(%arg0: tensor) -> (tensor, tensor) { + func.return %arg0, %arg0 : tensor, tensor +} + +// CHECK-LABEL: func @multiple_result_user +func.func @multiple_result_user(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>>) -> tensor { + // CHECK-NOT: tf.ReadVariableOp + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "multiple_uses", func = @multiple_result_user_func} : (tensor) -> tensor + "tf.AssignVariableOp"(%arg1, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + func.return %0 : tensor +} + +// CHECK-LABEL: func @multiple_result_user_func +// CHECK-SAME: ({{%.*}}: tensor) -> tensor +func.func @multiple_result_user_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// CHECK-LABEL: @reads_outside_replicate_op +func.func @reads_outside_replicate_op(%arg0: tensor<*x!tf_type.resource>> {tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}) { +// CHECK-COUNT-1: tf.ReadVariableOp +// CHECK: tf_device.replicate +// CHECK-NOT: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<1xf32> + %cst = "tf.Const"() <{value = dense<0.000000e+00> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<1> : tensor<1xi64>}> : () -> tensor<1xi64> + %fill = "tf.Fill"(%cst_0, %cst) : (tensor<1xi64>, tensor) -> tensor<1xf32> + tf_device.replicate([%0, %fill] as %arg_r0: tensor<1xf32>) {n = 2 : i32} { + %1 = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %2 = "tf.Identity"(%arg_r0) : (tensor<1xf32>) -> tensor<1xf32> + tf_device.return %2 : tensor<1xf32> + }) : () -> tensor<1xf32> + %3 = "tf_device.cluster_func"(%1) <{func = @write_chain_func}> {_replication_info = "cluster__train_helper", _xla_compile_device_type = "TPU", num_cores_per_replica = 1 : i64} : (tensor<1xf32>) -> tensor<1xf32> + "tf.AssignVariableOp"(%arg0, %3) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<1xf32>) -> () + tf_device.return + } + func.return +} + +func.func private @write_chain_func(%arg0: tensor<1xf32>) -> (tensor<1xf32>) { + %cst = "tf.Const"() <{value = dense<[[0, 1]]> : tensor<1x2xi32>}> : () -> tensor<1x2xi32> + %0 = "tf.XlaAllReduce"(%arg0, %cst) <{mode = "CrossReplica", reduce_op = "Add"}> : (tensor<1xf32>, tensor<1x2xi32>) -> tensor<1xf32> + return %0 : tensor<1xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-variable-runtime-reformatting.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-variable-runtime-reformatting.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0125ba8e5bc333e6d30b21d40dafd593596b59ca --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu-variable-runtime-reformatting.mlir @@ -0,0 +1,422 @@ +// RUN: tf-opt %s -split-input-file -tf-tpu-variable-runtime-reformatting| FileCheck %s + +// Tests that the pass can correctly transform a training loop with 2 replicas. + +!tf_res_f32 = tensor<*x!tf_type.resource>> +!tf_res_md_f32 = tensor<*x!tf_type.resource>> // Multi-dim f32 + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:0"}, + // CHECK-SAME: %[[ARG1:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:1"}, + // CHECK-SAME: %[[ARG2:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:0"}, + // CHECK-SAME: %[[ARG3:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:1"}) + func.func @main(%arg0: !tf_res_f32 {tf.device = "/device:TPU:0"}, + %arg1: !tf_res_f32 {tf.device = "/device:TPU:1"}, + %arg2: !tf_res_md_f32 {tf.device = "/device:TPU:0"}, + %arg3: !tf_res_md_f32 {tf.device = "/device:TPU:1"}) { + + %0 = "tf.Const"() {value = dense<100> : tensor} : () -> tensor + // CHECK: %[[STATE0:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:0" + // CHECK: %[[STATE1:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:1" + // CHECK: %[[WHILE:.*]] = "tf.WhileRegion"( + %1 = "tf.WhileRegion"(%0) ({ + // Condition region + // CHECK: ^bb + // CHECK: "tf.Yield" + ^bb0(%carg0: tensor): + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %c1 = "tf.GreaterEqual"(%carg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + "tf.Yield"(%c1) : (tensor) -> () + }, { + // Body region + // CHECK: ^bb0 + ^bb0(%barg0: tensor): + %b0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %b1 = "tf.AddV2"(%barg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %b2:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and 2 return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %b2#0, %b2#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG0]], %[[ARG1]]] as %[[R0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[ARG2]], %[[ARG3]]] as %[[R1:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[R_STATE:.*]]: tensor>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + %rep:2 = tf_device.replicate([%arg0, %arg1] as %arg30: tensor<*x!tf_type.resource>>, + [%arg2, %arg3] as %arg31: tensor<*x!tf_type.resource>>) + {_mirrored_variable_indices = [0, 1], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + // CHECK: %[[ID:.*]] = "tf.Identity"(%[[R0]]) + %id = "tf.Identity"(%arg30) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1, %[[R_STATE]]) + // CHECK-NEXT: tf_device.return + // CHECK: "tf.TPUExecuteAndUpdateVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%id, %arg31, %compile#1) + {device_var_reads_indices = [0, 1], device_var_updates_indices = [0, 1]} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + %ret = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + tf_device.return %ret : tensor + } + // CHECK: "tf.Yield" + "tf.Yield"(%b1) : (tensor) -> () + }) {device = "", is_stateless = false} : (tensor) -> (tensor) + // CHECK: %[[DEFAULT:.*]] = "tf.Const"() + // CHECK: tf_device.replicate + // CHECK-SAME: as %[[V0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: as %[[V1:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[STATE:.*]]: tensor>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[V0]], %[[V1]], %[[DEFAULT]], %[[STATE]]) + // CHECK-NEXT: tf_device.return + func.return + } +} + + +// ----- + +// Tests that the pass does not format variables with other uses. + +!tf_res_f32 = tensor<*x!tf_type.resource>> +!tf_res_md_f32 = tensor<*x!tf_type.resource>> // Multi-dim f32 + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func @main + // CHECK-NOT: TPUReshardVariables + func.func @main(%arg0: !tf_res_f32 {tf.device = "/device:TPU:0"}, + %arg1: !tf_res_f32 {tf.device = "/device:TPU:1"}, + %arg2: !tf_res_md_f32 {tf.device = "/device:TPU:0"}, + %arg3: !tf_res_md_f32 {tf.device = "/device:TPU:1"}, + %arg4: !tf_res_f32 {tf.device = "/device:TPU:1"}) { + + %0 = "tf.Const"() {value = dense<100> : tensor} : () -> tensor + %1 = "tf.WhileRegion"(%0) ({ + // Condition region + ^bb0(%carg0: tensor): + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %c1 = "tf.GreaterEqual"(%carg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + "tf._UnknownOp1_"(%arg1) : (!tf_res_f32) -> () + "tf.Yield"(%c1) : (tensor) -> () + }, { + // Body region + ^bb0(%barg0: tensor): + %b0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %b1 = "tf.AddV2"(%barg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + %compile:2 = "tf_device.launch"() ({ + %b2:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 3 parameter and 3 return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %b2#0, %b2#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %id0 = "tf.Identity"(%arg3) : (!tf_res_md_f32) -> !tf_res_md_f32 + "tf._Unknown_"(%id0) : (!tf_res_md_f32) -> () + %newvar = "tf._SomeOp"() : () -> !tf_res_f32 + %rep:2 = tf_device.replicate([%arg0, %arg1] as %arg30: !tf_res_f32, + [%arg2, %arg3] as %arg31: !tf_res_md_f32, + [%newvar, %arg4] as %arg32 : !tf_res_f32) + {_mirrored_variable_indices = [0, 1, 2], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + // %arg30 is used in the cond function, %arg31 has other uses (%id0), and + // %arg32 is not a pass-through. + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%arg30, %arg31, %arg32, %compile#1) + {device_var_reads_indices = [0, 1, 2], device_var_updates_indices = [0, 1, 2]} + : (!tf_res_f32, !tf_res_md_f32, !tf_res_f32, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + %ret = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + tf_device.return %ret : tensor + } + "tf.Yield"(%b1) : (tensor) -> () + }) {device = "", is_stateless = false} : (tensor) -> (tensor) + func.return + } +} + +// ----- + +// Tests that the pass does not format variables when model parallelism is +// present. Model parallelism is present when there is more than 1 +// TPUExecuteAndUpdateVariables in a parallel_execute. + +!tf_res_f32 = tensor<*x!tf_type.resource>> +!tf_res_md_f32 = tensor<*x!tf_type.resource>> // Multi-dim f32 + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func @main + // CHECK-NOT: TPUReshardVariables + func.func @main( + %arg0: !tf_res_f32 {tf.device = "/device:TPU:0"}, + %arg1: !tf_res_f32 {tf.device = "/device:TPU:1"}, + %arg2: !tf_res_md_f32 {tf.device = "/device:TPU:0"}, + %arg3: !tf_res_md_f32 {tf.device = "/device:TPU:1"}) { + + %0 = "tf.Const"() {value = dense<100> : tensor} : () -> tensor + %1 = "tf.WhileRegion"(%0) ({ + // Condition region + ^bb0(%carg0: tensor): + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %c1 = "tf.GreaterEqual"(%carg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + "tf.Yield"(%c1) : (tensor) -> () + }, { + // Body region + ^bb0(%barg0: tensor): + %b0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %b1 = "tf.AddV2"(%barg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + %compile:2 = "tf_device.launch"() ({ + %b2:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and 2 return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %b2#0, %b2#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + %rep:2 = tf_device.replicate( + [%arg0, %arg1] as %arg30: tensor<*x!tf_type.resource>>, + [%arg2, %arg3] as %arg31: tensor<*x!tf_type.resource>>) + {_mirrored_variable_indices = [0, 1], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + %id = "tf.Identity"(%arg30) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + "tf_device.parallel_execute"() ({ + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%id, %arg31, %compile#1) + {device_var_reads_indices = [0, 1], device_var_updates_indices = [0, 1]} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + }, { + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%compile#1) + {device_var_reads_indices = [], device_var_updates_indices = []} + : (tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_1"} : () -> () + tf_device.return + }) {} : () -> () + %ret = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + tf_device.return %ret : tensor + } + "tf.Yield"(%b1) : (tensor) -> () + }) {device = "", is_stateless = false} : (tensor) -> (tensor) + func.return + } +} + +// ----- + +// Tests that the pass formats variables when there is a parallel_execute but +// only 1 TPUExecuteAndUpdateVariables. This can happen when there is outside +// compilation and no model parallelism. + +!tf_res_f32 = tensor<*x!tf_type.resource>> +!tf_res_md_f32 = tensor<*x!tf_type.resource>> // Multi-dim f32 + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:0"}, + // CHECK-SAME: %[[ARG1:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:1"}, + // CHECK-SAME: %[[ARG2:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:0"}, + // CHECK-SAME: %[[ARG3:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:1"}) + func.func @main(%arg0: !tf_res_f32 {tf.device = "/device:TPU:0"}, + %arg1: !tf_res_f32 {tf.device = "/device:TPU:1"}, + %arg2: !tf_res_md_f32 {tf.device = "/device:TPU:0"}, + %arg3: !tf_res_md_f32 {tf.device = "/device:TPU:1"}) { + + %0 = "tf.Const"() {value = dense<100> : tensor} : () -> tensor + // CHECK: %[[STATE0:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:0" + // CHECK: %[[STATE1:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:1" + // CHECK: %[[WHILE:.*]] = "tf.WhileRegion"( + %1 = "tf.WhileRegion"(%0) ({ + // Condition region + // CHECK: ^bb + // CHECK: "tf.Yield" + ^bb0(%carg0: tensor): + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %c1 = "tf.GreaterEqual"(%carg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + "tf.Yield"(%c1) : (tensor) -> () + }, { + // Body region + // CHECK: ^bb0 + ^bb0(%barg0: tensor): + %b0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %b1 = "tf.AddV2"(%barg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %b2:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and 2 return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %b2#0, %b2#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG0]], %[[ARG1]]] as %[[R0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[ARG2]], %[[ARG3]]] as %[[R1:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[R_STATE:.*]]: tensor>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + %rep:2 = tf_device.replicate([%arg0, %arg1] as %arg30: tensor<*x!tf_type.resource>>, + [%arg2, %arg3] as %arg31: tensor<*x!tf_type.resource>>) + {_mirrored_variable_indices = [0, 1], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + // CHECK: %[[ID:.*]] = "tf.Identity"(%[[R0]]) + %id = "tf.Identity"(%arg30) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + // CHECK: "tf_device.parallel_execute" + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1, %[[R_STATE]]) + // CHECK-NEXT: tf_device.return + // CHECK: "tf.TPUExecuteAndUpdateVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1) + "tf_device.parallel_execute"() ({ + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%id, %arg31, %compile#1) + {device_var_reads_indices = [0, 1], device_var_updates_indices = [0, 1]} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + }, { + tf_device.return + }) {} : () -> () + %ret = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + tf_device.return %ret : tensor + } + // CHECK: "tf.Yield" + "tf.Yield"(%b1) : (tensor) -> () + }) {device = "", is_stateless = false} : (tensor) -> (tensor) + // CHECK: %[[DEFAULT:.*]] = "tf.Const"() + // CHECK: tf_device.replicate + // CHECK-SAME: as %[[V0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: as %[[V1:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[STATE:.*]]: tensor>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[V0]], %[[V1]], %[[DEFAULT]], %[[STATE]]) + // CHECK-NEXT: tf_device.return + func.return + } +} + +// ----- + +// Tests that the pass can correctly transform a training loop with a packed +// variable. +!tf_res_f32 = tensor<*x!tf_type.resource>> +!tf_res_md_f32 = tensor<*x!tf_type.resource>> // Multi-dim f32 + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:0"}, + // CHECK-SAME: %[[ARG1:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:TPU:1"}, + // CHECK-SAME: %[[ARG2:.*]]: tensor<*x!tf_type.resource>> {tf.device = "/device:COMPOSITE:0"}) + func.func @main(%arg0: !tf_res_f32 {tf.device = "/device:TPU:0"}, + %arg1: !tf_res_f32 {tf.device = "/device:TPU:1"}, + %arg2: !tf_res_md_f32 {tf.device = "/device:COMPOSITE:0"}) { + %0 = "tf.Const"() {value = dense<100> : tensor} : () -> tensor + // CHECK: %[[STATE0:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:0" + // CHECK: %[[STATE1:.*]] = "tf.VarHandleOp"() + // CHECK-SAME: device = "/device:TPU:1" + // CHECK: %[[WHILE:.*]] = "tf.WhileRegion"( + %1 = "tf.WhileRegion"(%0) ({ + // Condition region + // CHECK: ^bb + // CHECK: "tf.Yield" + ^bb0(%carg0: tensor): + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %c1 = "tf.GreaterEqual"(%carg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + "tf.Yield"(%c1) : (tensor) -> () + }, { + // Body region + // CHECK: ^bb0 + ^bb0(%barg0: tensor): + %b0 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %b1 = "tf.AddV2"(%barg0, %0) {T = i32, device = ""} : (tensor, tensor) -> tensor + // CHECK: %[[COMPILE:.*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + %compile:2 = "tf_device.launch"() ({ + %b2:2 = "tf._TPUCompileMlir"() { + NumDynamicShapes = 0 : i64, + // The metadata encodes 2 parameter and 2 return values. + metadata = "\0A\0E\08\01\18\01\22\08\08\01\1A\01\01\22\01\00\0A \08\01\12\10\12\02\08\03\12\02\08\03\12\02\08\01\12\02\08 \18\01\22\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\12\0A\0A\08\08\01\1A\01\01\22\01\00\18\02 \01", + mlir_module = "..."} : () -> (tensor, tensor<2x!tf_type.string>) + tf_device.return %b2#0, %b2#1 : tensor, tensor<2x!tf_type.string> + }) {device = "/device:CPU:0"} : () -> (tensor, tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUCompileSucceededAssert"(%compile#0) : (tensor) -> () + tf_device.return + }) {device = "/device:CPU:0"} : () -> () + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG0]], %[[ARG1]]] as %[[R0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[R_STATE:.*]]: tensor>> + // CHECK-SAME: %[[ARG2]] as %[[R1:.*]]: tensor<*x!tf_type.resource>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + %rep:2 = tf_device.replicate([%arg0, %arg1] as %arg30: tensor<*x!tf_type.resource>>, + %arg2 as %arg31: tensor<*x!tf_type.resource>>) + {_mirrored_variable_indices = [0, 1], _packed_input_indices = [1], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + // CHECK: %[[ID:.*]] = "tf.Identity"(%[[R0]]) + %id = "tf.Identity"(%arg30) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1, %[[R_STATE]]) + // CHECK-NEXT: tf_device.return + // CHECK: "tf.TPUExecuteAndUpdateVariables"(%[[ID]], %[[R1]], %[[COMPILE]]#1) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%id, %arg31, %compile#1) + {device_var_reads_indices = [0, 1], device_var_updates_indices = [0, 1]} + : (tensor<*x!tf_type.resource>>, tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + %ret = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + tf_device.return %ret : tensor + } + // CHECK: "tf.Yield" + "tf.Yield"(%b1) : (tensor) -> () + }) {device = "", is_stateless = false} : (tensor) -> (tensor) + // CHECK: %[[DEFAULT:.*]] = "tf.Const"() + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG0]], %[[ARG1]]] as %[[V0:.*]]: tensor<*x!tf_type.resource>>, + // CHECK-SAME: [%[[STATE0]], %[[STATE1]]] as %[[STATE:.*]]: tensor>> + // CHECK-SAME: %[[ARG2]] as %[[V1:.*]]: tensor<*x!tf_type.resource>> + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"] + // CHECK: "tf_device.launch" + // CHECK-SAME: device = "TPU_REPLICATED_CORE_0" + // CHECK-NEXT: "tf.TPUReshardVariables"(%[[V0]], %[[V1]], %[[DEFAULT]], %[[STATE]]) + // CHECK-NEXT: tf_device.return + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..3eb601a84e64374c384e95a7994f6edeb99525ad --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/BUILD @@ -0,0 +1,31 @@ +load("//tensorflow:tensorflow.default.bzl", "filegroup") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +licenses(["notice"]) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "@llvm-project//mlir:run_lit.sh", + exclude = [ + # TODO(b/305738491): Temporarily excluded until Bridge V1 has all passes as pass pipelines. + "end_to_end.mlir", + ], + test_file_exts = [ + "mlir", + "pbtxt", + ], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/end_to_end.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/end_to_end.mlir new file mode 100644 index 0000000000000000000000000000000000000000..991cee81de6dc0ff6eb55576151426a805b9cef9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_bridge_v1/end_to_end.mlir @@ -0,0 +1,30 @@ +// RUN: tf-opt %s -tf-replicated-clustering-bridge-v1 | FileCheck %s + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"], tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 296 : i32}} { + func.func @main() { +// CHECK: arith.constant +// CHECK: TPUCompile +// CHECK: TPUExecute +// CHECK-NOT: func @_func + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "arith.constant"() {value = dense<2.000000e+00> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "arith.constant"() {value = dense<3.000000e+00> : tensor} : () -> tensor + %control_2 = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], name = "TPUReplicateMetadata", num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_tpu = true, use_spmd_for_xla_partitioning = false} : () -> () + %outputs_3, %control_4 = tf_executor.island wraps "tf.Placeholder"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "x", shape = "tfshape$dim { }"} : () -> tensor<0xf32> + %outputs_5, %control_6 = tf_executor.island wraps "tf.TPUReplicatedInput"(%outputs_3) {N = 1 : i64, T = "tfdtype$DT_FLOAT", device = "", name = "input0"} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_7, %control_8 = tf_executor.island wraps "tf.Identity"(%outputs_5) {T = "tfdtype$DT_FLOAT", _tpu_input_identity = true, _tpu_replicate = "cluster", device = "", name = "replicated_input_0"} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_9, %control_10 = tf_executor.island wraps "tf.Mul"(%outputs_7, %outputs) {T = "tfdtype$DT_FLOAT", _tpu_replicate = "cluster", device = "", name = "mul"} : (tensor<0xf32>, tensor) -> tensor<0xf32> + %outputs_11, %control_12 = tf_executor.island wraps "tf.Placeholder"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "y", shape = "tfshape$dim { }"} : () -> tensor<0xf32> + %outputs_13, %control_14 = tf_executor.island wraps "tf.TPUReplicatedInput"(%outputs_11) {N = 1 : i64, T = "tfdtype$DT_FLOAT", device = "", name = "input1"} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_15, %control_16 = tf_executor.island wraps "tf.Identity"(%outputs_13) {T = "tfdtype$DT_FLOAT", _tpu_input_identity = true, _tpu_replicate = "cluster", device = "", name = "replicated_input_1"} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_17, %control_18 = tf_executor.island wraps "tf.Mul"(%outputs_15, %outputs_0) {T = "tfdtype$DT_FLOAT", _tpu_replicate = "cluster", device = "", name = "mul_1"} : (tensor<0xf32>, tensor) -> tensor<0xf32> + %outputs_19, %control_20 = tf_executor.island wraps "tf.AddV2"(%outputs_9, %outputs_17) {T = "tfdtype$DT_FLOAT", _tpu_replicate = "cluster", device = "", name = "add"} : (tensor<0xf32>, tensor<0xf32>) -> tensor<0xf32> + %outputs_21, %control_22 = tf_executor.island wraps "tf.Identity"(%outputs_19) {T = "tfdtype$DT_FLOAT", _tpu_output_identity = true, _tpu_replicate = "cluster", device = "/device:TPU_REPLICATED_CORE:0", name = "Identity"} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_23, %control_24 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%outputs_21) {T = "tfdtype$DT_FLOAT", device = "", name = "output0", num_replicas = 1 : i64} : (tensor<0xf32>) -> tensor<0xf32> + %outputs_25, %control_26 = tf_executor.island wraps "tf.Identity"(%outputs_23) {T = "tfdtype$DT_FLOAT", device = "", name = "output_0_shard_0"} : (tensor<0xf32>) -> tensor<0xf32> + %control_27 = tf_executor.island(%control_2, %control_26) wraps "tf.NoOp"() : () -> () + tf_executor.fetch %control_27 : !tf_executor.control + } + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir new file mode 100644 index 0000000000000000000000000000000000000000..656f55a6c4702145424780b2c34bbc9032d44d47 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir @@ -0,0 +1,934 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-cluster-formation | FileCheck %s + + +// Test ops in cluster only have `_replication_info` and `device` attributes +// removed when moved to a `tf_device.cluster`. +// CHECK-LABEL: func @cluster_ops_removed_attrs +func.func @cluster_ops_removed_attrs() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: "tf.opA" +// CHECK-SAME: name = "name" +// CHECK-NOT: _replication_info = "replicate" +// CHECK-NOT: device = "/device:TPU:0" +// CHECK: tf_device.return + + +// Test TPUReplicateMetadata ops `name` and `num_replicas` attributes are not +// copied over to `tf_device.cluster`. +// CHECK-LABEL: func @removed_metadata_attrs +func.func @removed_metadata_attrs() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK-NOT: name = "name" +// CHECK-NOT: num_replicas = 1 + + +// Test TPUReplicateMetadata op is removed when forming clusters. +// CHECK-LABEL: func @metadata_op_removed +func.func @metadata_op_removed() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK-NOT: "tf.TPUReplicateMetadata" + + +// Test ops in a function body with the same `_replication_info` attribute are +// merged under a `tf_device.cluster` op. +// CHECK-LABEL: func @ops_in_func_body +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) +func.func @ops_in_func_body(%arg0 : tensor) -> (tensor, tensor, tensor) { + %0 = "tf.opA"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %1 = "tf.opB"() {is_stateless = true} : () -> tensor + %2 = "tf.opC"(%0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + %3 = "tf.opD"(%2) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %4 = "tf.opE"() {is_stateless = true} : () -> tensor + %5 = "tf.opF"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + func.return %2, %3, %5 : tensor, tensor, tensor +} + +// CHECK: "tf.opB" +// CHECK: "tf.opE" +// CHECK: %[[CLUSTER:[0-9]*]]:3 = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_C:[0-9]*]] = "tf.opC"(%[[OP_A]]) +// CHECK-NEXT: %[[OP_D:[0-9]*]] = "tf.opD"(%[[OP_C]]) +// CHECK-NEXT: %[[OP_F:[0-9]*]] = "tf.opF"(%[[ARG_0]]) +// CHECK-NEXT: tf_device.return %[[OP_C]], %[[OP_D]], %[[OP_F]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: return %[[CLUSTER]]#0, %[[CLUSTER]]#1, %[[CLUSTER]]#2 + + +// Test a nested user of an op in a cluster has its operand be updated to +// `tf_device.cluster` result. +// CHECK-LABEL: func @nested_cluster_op_user +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) +func.func @nested_cluster_op_user(%arg0 : tensor) -> (tensor) { + %0 = "tf.opA"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %1 = "tf_device.launch"() ({ + tf_device.return %0 : tensor + }) {device = "/device:TPU:0"} : () -> tensor + %2 = "tf.opB"(%0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return %2 : tensor +} + +// CHECK: %[[CLUSTER:[0-9]*]]:2 = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_B:[0-9]*]] = "tf.opB"(%[[OP_A]]) +// CHECK-NEXT: tf_device.return %[[OP_A]], %[[OP_B]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: tf_device.launch +// CHECK-NEXT: tf_device.return %[[CLUSTER]]#0 +// CHECK: return %[[CLUSTER]]#1 + + +// Test nested op of a cluster with an operand from an op of the same cluster +// retains its original operand. +// CHECK-LABEL: func @nested_cluster_op +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) +func.func @nested_cluster_op(%arg0 : tensor) -> (tensor) { + %0 = "tf.opA"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %1 = "tf.opB"() ({ + "tf.opC"(%0) {is_stateless = true} : (tensor) -> tensor + }) {_xla_compile_device_type = "TPU", _replication_info = "replicate"} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return %1 : tensor +} + +// CHECK: %[[CLUSTER:[0-9]*]] = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_B:[0-9]*]] = "tf.opB"() ({ +// CHECK-NEXT: "tf.opC"(%[[OP_A]]) +// CHECK: tf_device.return %[[OP_B]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: return %[[CLUSTER]] + + +// Test multiple clusters interleaved. +// CHECK-LABEL: func @interleaved_clusters +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) +func.func @interleaved_clusters(%arg0 : tensor) -> (tensor, tensor) { + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate_1", device = "device_1", num_replicas = 1, topology = "topology_1"} : () -> () + %0 = "tf.opA"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate_0", is_stateless = true} : (tensor) -> tensor + %1 = "tf.opB"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "replicate_1", is_stateless = true} : (tensor) -> tensor + %2 = "tf.opC"(%0) {_xla_compile_device_type = "TPU", _replication_info = "replicate_0", is_stateless = true} : (tensor) -> tensor + %3 = "tf.opD"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate_1", is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate_0", device = "device_0", num_replicas = 1, topology = "topology_0"} : () -> () + func.return %2, %3 : tensor, tensor +} + +// CHECK: %[[CLUSTER_0:[0-9]*]] = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_C:[0-9]*]] = "tf.opC"(%[[OP_A]]) +// CHECK-NEXT: tf_device.return %[[OP_C]] +// CHECK-NEXT: _replication_info = "replicate_0" +// CHECK-SAME: device = "device_0" +// CHECK-SAME: topology = "topology_0" +// CHECK: %[[CLUSTER_1:[0-9]*]] = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_B:[0-9]*]] = "tf.opB"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_D:[0-9]*]] = "tf.opD"(%[[OP_B]]) +// CHECK-NEXT: tf_device.return %[[OP_D]] +// CHECK-NEXT: _replication_info = "replicate_1" +// CHECK-SAME: device = "device_1" +// CHECK-SAME: topology = "topology_1" +// CHECK: return %[[CLUSTER_0]], %[[CLUSTER_1]] + + +// Test operands and results of ops of a cluster that are interleaved between +// other ops of the same cluster are moved before and after the cluster +// properly. +// CHECK-LABEL: func @interleaved_cluster_operands_results +func.func @interleaved_cluster_operands_results() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : () -> tensor + %1 = "tf.opB"(%0) {is_stateless = true} : (tensor) -> tensor + %2 = "tf.opC"() {is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + %3 = "tf.opD"(%1) {is_stateless = true} : (tensor) -> tensor + %4 = "tf.opE"(%2) {is_stateless = true} : (tensor) -> tensor + %5 = "tf.opF"(%4) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + func.return +} + +// CHECK: %[[OP_C:[0-9]*]] = "tf.opC" +// CHECK: %[[OP_E:[0-9]*]] = "tf.opE"(%[[OP_C]]) +// CHECK: %[[CLUSTER:[0-9]*]] = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA" +// CHECK-NEXT: "tf.opF"(%[[OP_E]]) +// CHECK-NEXT: tf_device.return %[[OP_A]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: %[[OP_B:[0-9]*]] = "tf.opB"(%[[CLUSTER]]) +// CHECK: "tf.opD"(%[[OP_B]]) + + +// Test one replica cluster results in removing of TPUReplicatedInput and +// TPUReplicatedOutput nodes and operands are forwarded to results. +// CHECK-LABEL: func @one_replica +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) +func.func @one_replica(%arg0: tensor) -> tensor { + %ri = "tf.TPUReplicatedInput"(%arg0) : (tensor) -> tensor + %0 = "tf.opA"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %1 = "tf.opB"(%0) {is_stateless = true} : (tensor) -> tensor + %2 = "tf.opC"() {is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + %3 = "tf.opD"(%1) {is_stateless = true} : (tensor) -> tensor + %4 = "tf.opE"(%2) {is_stateless = true} : (tensor) -> tensor + %5 = "tf.opF"(%4) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %ro = "tf.TPUReplicatedOutput"(%5) : (tensor) -> tensor + func.return %ro : tensor +} + +// CHECK: %[[OP_C:[0-9]*]] = "tf.opC" +// CHECK: %[[OP_E:[0-9]*]] = "tf.opE"(%[[OP_C]]) +// CHECK: %[[CLUSTER:[0-9]*]]:2 = "tf_device.cluster"() ({ +// CHECK-NEXT: %[[OP_A:[0-9]*]] = "tf.opA"(%[[ARG_0]]) +// CHECK-NEXT: %[[OP_F:[0-9]*]] = "tf.opF"(%[[OP_E]]) +// CHECK-NEXT: tf_device.return %[[OP_A]], %[[OP_F]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: %[[OP_B:[0-9]*]] = "tf.opB"(%[[CLUSTER]]#0) +// CHECK: "tf.opD"(%[[OP_B]]) +// CHECK: return %[[CLUSTER]]#1 +// CHECK-NOT: "tf.TPUReplicatedInput" +// CHECK-NOT: "tf.TPUReplicatedOutput" + + +// Test replication with replicated operands and replicated results. The cluster +// will be wrapped in a `tf_device.cluster` first and then by a replicate. +// TPUReplicatedInput and TPUReplicatedOutput nodes will be replaced by the +// replicate operands and results. +// CHECK-LABEL: func @replication +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor, %[[ARG_1:[a-z0-9]*]]: tensor, %[[ARG_2:[a-z0-9]*]]: tensor) +func.func @replication(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> (tensor, tensor) { + %0 = "tf.opA"() {is_stateless = true} : () -> tensor + %ri_0 = "tf.TPUReplicatedInput"(%arg0, %0) : (tensor, tensor) -> tensor + %1 = "tf.opB"() {is_stateless = true} : () -> tensor + %ri_1 = "tf.TPUReplicatedInput"(%1, %arg1) : (tensor, tensor) -> tensor + %2 = "tf.opC"() {is_stateless = true} : () -> tensor + %3 = "tf.opD"(%ri_0, %ri_1, %arg2, %2) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor, tensor, tensor, tensor) -> tensor + %ro_0:2 = "tf.TPUReplicatedOutput"(%3) : (tensor) -> (tensor, tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %7 = "tf.opE"(%3, %ri_0, %ri_1, %arg2, %2) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor, tensor, tensor, tensor, tensor) -> tensor + %ro_1:2 = "tf.TPUReplicatedOutput"(%7) : (tensor) -> (tensor, tensor) + func.return %ro_0#0, %ro_1#1 : tensor, tensor +} + +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA" +// CHECK: %[[OP_B:[0-9]*]] = "tf.opB" +// CHECK: %[[OP_C:[0-9]*]] = "tf.opC" +// CHECK: %[[REPLICATE:[0-9]*]]:4 = tf_device.replicate +// CHECK-DAG: [%[[ARG_0]], %[[OP_A]]] as %[[RI_0:[a-z0-9]*]]: tensor +// CHECK-DAG: [%[[OP_B]], %[[ARG_1]]] as %[[RI_1:[a-z0-9]*]]: tensor +// CHECK-SAME: n = 2 : i32 +// CHECK-NEXT: %[[CLUSTER:[0-9]*]]:2 = "tf_device.cluster"() ({ +// CHECK: %[[OP_D:[0-9]*]] = "tf.opD"(%[[RI_0]], %[[RI_1]], %[[ARG_2]], %[[OP_C]]) +// CHECK: %[[OP_E:[0-9]*]] = "tf.opE"(%[[OP_D]], %[[RI_0]], %[[RI_1]], %[[ARG_2]], %[[OP_C]]) +// CHECK: tf_device.return %[[OP_D]], %[[OP_E]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: tf_device.return %[[CLUSTER]]#0, %[[CLUSTER]]#1 +// CHECK: return %[[REPLICATE]]#0, %[[REPLICATE]]#3 + + +// Test replication with model parallelism using partitioned resource inputs. +// The cluster will be wrapped in a `tf_device.cluster` first and then by a +// replicate. +// TPUPartitionedInputV2 nodes would be inside the replicate but outside the +// cluster. +// TPUReplicatedInput and TPUReplicatedOutput nodes will be replaced by the +// replicate operands and results. +// CHECK-LABEL: func @replication_with_model_parallelism +// CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor>>, %[[ARG_1:[a-z0-9]*]]: tensor>>, %[[ARG_2:[a-z0-9]*]]: tensor>>, %[[ARG_3:[a-z0-9]*]]: tensor>>) +!rtype = tensor>> + +func.func @replication_with_model_parallelism(%arg0: !rtype, %arg1: !rtype, %arg2: !rtype, %arg3: !rtype) -> (tensor<10x3xf32>, tensor) { + %0 = "tf.opA"() {is_stateless = true} : () -> tensor + %1 = "tf.opB"() {is_stateless = true} : () -> tensor + %2 = "tf.TPUReplicatedInput"(%arg0, %arg2) : (!rtype, !rtype) -> !rtype + %3 = "tf.TPUReplicatedInput"(%arg1, %arg3) : (!rtype, !rtype) -> !rtype + %4 = "tf.TPUPartitionedInputV2"(%2, %3) {_XlaSharding = "", device = "", partition_dims = []} : (!rtype, !rtype) -> !rtype + %5 = "tf.TPUReplicatedInput"(%0, %1) : (tensor, tensor) -> tensor + %6 = "tf.opC"(%4) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (!rtype) -> tensor<10x3xf32> + %7:2 = "tf.TPUReplicatedOutput"(%6) : (tensor<10x3xf32>) -> (tensor<10x3xf32>, tensor<10x3xf32>) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 2 : i64, topology = "topology"} : () -> () + %8 = "tf.opD"(%5) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %9:2 = "tf.TPUReplicatedOutput"(%8) : (tensor) -> (tensor, tensor) + func.return %7#0, %9#1 : tensor<10x3xf32>, tensor +} + +// CHECK: %[[OP_A:[0-9]*]] = "tf.opA" +// CHECK: %[[OP_B:[0-9]*]] = "tf.opB" +// CHECK: %[[REPLICATE:[0-9]*]]:4 = tf_device.replicate +// CHECK-DAG: [%[[ARG_0]], %[[ARG_2]]] as %[[RI_0:[a-z0-9]*]]: tensor>> +// CHECK-DAG: [%[[ARG_1]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor>> +// CHECK-DAG: [%[[OP_A]], %[[OP_B]]] as %[[RI_2:[a-z0-9]*]]: tensor +// CHECK-SAME: n = 2 : i32 +// CHECK: %[[PI:[0-9]*]] = "tf.TPUPartitionedInputV2"(%[[RI_0]], %[[RI_1]]) +// CHECK-NEXT: %[[CLUSTER:[0-9]*]]:2 = "tf_device.cluster"() ({ +// CHECK: %[[OP_C:[0-9]*]] = "tf.opC"(%[[PI]]) +// CHECK: %[[OP_D:[0-9]*]] = "tf.opD"(%[[RI_2]]) +// CHECK: tf_device.return %[[OP_C]], %[[OP_D]] +// CHECK-NEXT: _replication_info = "replicate" +// CHECK-SAME: device = "/device:TPU:0" +// CHECK-SAME: topology = "topology" +// CHECK: tf_device.return %[[CLUSTER]]#0, %[[CLUSTER]]#1 +// CHECK: return %[[REPLICATE]]#0, %[[REPLICATE]]#3 + + +// Test TPUReplicatedInputs with non contiguous `index` attributes. +// CHECK-LABEL: func @non_contigous_indices +// CHECK-SAME: (%[[ARG_0:.*]]: tensor, %[[ARG_1:.*]]: tensor, %[[ARG_2:.*]]: tensor, %[[ARG_3:.*]]: tensor, %[[ARG_4:.*]]: tensor, %[[ARG_5:.*]]: tensor) +func.func @non_contigous_indices(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor) { + %0 = "tf.TPUReplicatedInput"(%arg0, %arg0) {index = 8 : i64} : (tensor, tensor) -> tensor + "tf.opA"(%0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + %1 = "tf.TPUReplicatedInput"(%arg1) {index = 6 : i64, is_packed = true} : (tensor) -> tensor + "tf.opA"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + %2 = "tf.TPUReplicatedInput"(%arg2, %arg2) : (tensor, tensor) -> tensor + "tf.opB"(%2) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + %3 = "tf.TPUReplicatedInput"(%arg3) {is_packed = true} : (tensor) -> tensor + "tf.opB"(%3) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + %4 = "tf.TPUReplicatedInput"(%arg4, %arg4) {index = 2 : i64} : (tensor, tensor) -> tensor + "tf.opC"(%4) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + %5 = "tf.TPUReplicatedInput"(%arg5) {index = 4 : i64, is_packed = true} : (tensor) -> tensor + "tf.opC"(%5) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// CHECK: tf_device.replicate +// CHECK-SAME: [%[[ARG_0]], %[[ARG_0]]] as %{{[a-z0-9]*}} +// CHECK-SAME: [%[[ARG_2]], %[[ARG_2]]] as %{{[a-z0-9]*}} +// CHECK-SAME: [%[[ARG_4]], %[[ARG_4]]] as %{{[a-z0-9]*}} +// CHECK-SAME: %[[ARG_1]] as %{{[a-z0-9]*}} +// CHECK-SAME: %[[ARG_3]] as %{{[a-z0-9]*}} +// CHECK-SAME: %[[ARG_5]] as %{{[a-z0-9]*}} + + +// Test that the `is_mirrored_variable` attribute is preserved in the +// tf_device.replicate op. +// CHECK-LABEL: func @mirrored_variables +// CHECK-SAME: (%[[ARG_0:.*]]: tensor>>, %[[ARG_1:.*]]: tensor>>, %[[ARG_2:.*]]: tensor>>, %[[ARG_3:.*]]: tensor>>, %[[ARG_4:.*]]: tensor>>) +func.func @mirrored_variables(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>, %arg4: tensor>>) { + %0 = "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 0 : i64} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.TPUReplicatedInput"(%arg2, %arg3) {index = 1 : i64, is_mirrored_variable = true} : (tensor>>, tensor>>) -> tensor>> + %2 = "tf.TPUReplicatedInput"(%arg4) {index = 2 : i64, is_mirrored_variable = true, is_packed = true} : (tensor>>) -> tensor>> + "tf.opA"(%0, %1, %2) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", is_stateless = true} : (tensor>>, tensor>>, tensor>>) -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// CHECK: tf_device.replicate +// CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %{{[a-z0-9]*}} +// CHECK-SAME: %[[ARG_4]] as %{{[a-z0-9]*}} +// CHECK-SAME: _mirrored_variable_indices = [1, 2] + + +// Test resource usage after resource use in cluster is moved to after the +// cluster. +// CHECK-LABEL: func @resource_after_cluster +// CHECK-SAME: ([[USED_RESOURCE:%.*]]: tensor<*x!tf_type.resource>>, [[UNUSED_RESOURCE:%.*]]: tensor<*x!tf_type.resource>>) +func.func @resource_after_cluster(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) { + // CHECK-NEXT: [[CONST:%.*]] = "tf.Const" + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + + // CHECK-NEXT: "tf.AssignSubVariableOp"([[UNUSED_RESOURCE]], [[CONST]]) + + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.ReadVariableOp"([[USED_RESOURCE]]) + // CHECK-NEXT: "tf.NoOp" + // CHECK-NEXT: tf_device.return + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster_test_fn", allow_soft_placement = false, computation_shape = [], device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + %1 = "tf.ReadVariableOp"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster_test_fn"} : (tensor<*x!tf_type.resource>>) -> tensor + + "tf.AssignSubVariableOp"(%arg1, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + + // CHECK: "tf.AssignAddVariableOp"([[USED_RESOURCE]], [[CONST]]) + "tf.AssignAddVariableOp"(%arg0, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + + "tf.NoOp"() {_xla_compile_device_type = "TPU", _replication_info = "cluster_test_fn"} : () -> () + func.return +} + + +// Test resource not used by cluster is moved to before the cluster. +// CHECK-LABEL: func @resource_before_cluster +func.func @resource_before_cluster() { + // CHECK-NEXT: [[CONST:%.*]] = "tf.Const" + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + + // CHECK-NEXT: [[UNUSED_RESOURCE:%.*]] = "tf.VarHandleOp" + // CHECK-NEXT: "tf.AssignAddVariableOp"([[UNUSED_RESOURCE]], [[CONST]]) + + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.NoOp" + // CHECK-NEXT: tf_device.return + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster_test_fn", allow_soft_placement = false, computation_shape = [], device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + + %1 = "tf.VarHandleOp"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor<*x!tf_type.resource>> + "tf.AssignAddVariableOp"(%1, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + + "tf.NoOp"() {_xla_compile_device_type = "TPU", _replication_info = "cluster_test_fn"} : () -> () + func.return +} + + +// Test cluster formation with ops with attached regions within a cluster. +// Nested op's that are moved should get their _replication_info and device +// attributes cleared. +// CHECK-LABEL: func @cluster_ops_with_regions +func.func @cluster_ops_with_regions() { + %0 = "tf.opA"() ({ + %1 = "tf.opB"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "nameB", is_stateless = true} : () -> (tensor) + }) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "nameA"} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: "tf.opA"() ({ +// CHECK-NEXT: "tf.opB" +// CHECK-NOT: _replication_info = "replicate" +// CHECK-NOT: device = "/device:TPU:0" +// CHECK-SAME: name = "nameB" +// CHECK: }) +// CHECK-NOT: _replication_info = "replicate" +// CHECK-NOT: device = "/device:TPU:0" +// CHECK: name = "nameA" +// CHECK: tf_device.return + +// A nested cluster op using result of another cluster op. In the below, opA and +// opB go in a cluster, and opD stays outside. +// CHECK-LABEL: func @cluster_nested_op_using_other_op +func.func @cluster_nested_op_using_other_op() { + %0 = "tf.opA"() { _xla_compile_device_type = "TPU", _replication_info = "foo" , is_stateless = true} : () -> tensor + "tf.opB"() ({ + "tf.opC"(%0) {is_stateless = true} : (tensor) -> () + }) { _xla_compile_device_type = "TPU", _replication_info = "foo" } : () -> () + "tf.opD"(%0) {is_stateless = true} : (tensor) -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "foo", device = "CPU", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: [[CLUSTER:%.*]] = "tf_device.cluster"() ({ +// CHECK: [[OPA:%.*]] = "tf.opA"() {is_stateless = true} : () -> tensor +// CHECK: "tf.opB"() ({ +// CHECK: "tf.opC"([[OPA]]) +// CHECK: tf_device.return [[OPA]] +// CHECK: "tf.opD"([[CLUSTER]]) + +// Preceding user is using resource updated by a nested op. +!tf_res = tensor<*x!tf_type.resource>> +// CHECK-LABEL: func @cluster_nested_op_updating_resource +func.func @cluster_nested_op_updating_resource() { + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> !tf_res + + "tf.opA"() ({ + "tf.AssignAddVariableOp"(%1, %0) : (!tf_res, tensor) -> () + "tf.terminator"() : () -> () + }) { _xla_compile_device_type = "TPU", _replication_info = "foo" } : () -> () + "tf.AssignAddVariableOp"(%1, %0) : (!tf_res, tensor) -> () + "tf.opB"() { _xla_compile_device_type = "TPU", _replication_info = "foo" , is_stateless = true} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "foo", device = "CPU", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: [[CONST:%.*]] = "tf.Const" +// CHECK: [[VAR:%.*]] = "tf.VarHandleOp" +// CHECK: "tf_device.cluster"() ({ +// CHECK: "tf.opA"() ({ +// CHECK: "tf.AssignAddVariableOp"([[VAR]], [[CONST]]) +// CHECK: }) +// CHECK: "tf.opB"() +// CHECK: tf_device.return +// CHECK: }) +// CHECK-SAME: _replication_info = "foo" +// CHECK: "tf.AssignAddVariableOp"([[VAR]], [[CONST]]) + +// Preceding user is using resource updated by the cluster within a nested op. +// Resource is updated by a cluster op, and opA (not in cluster) is using the +// resource in a nested op. We expect opA to be after the cluster. +// CHECK-LABEL: func @cluster_nested_op_using_resource +func.func @cluster_nested_op_using_resource() { + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> !tf_res + "tf.AssignAddVariableOp"(%1, %0) { _xla_compile_device_type = "TPU", _replication_info = "foo" } : (!tf_res, tensor) -> () + "tf.opA"() ({ + "tf.AssignAddVariableOp"(%1, %0) : (!tf_res, tensor) -> () + "tf.terminator"() : () -> () + }) : () -> () + "tf.opB"() { _xla_compile_device_type = "TPU", _replication_info = "foo" , is_stateless = true} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "foo", device = "CPU", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: [[CONST:%.*]] = "tf.Const" +// CHECK: [[VAR:%.*]] = "tf.VarHandleOp" +// CHECK: "tf_device.cluster"() ({ +// CHECK: "tf.AssignAddVariableOp"([[VAR]], [[CONST]]) +// CHECK: "tf.opB"() +// CHECK: tf_device.return +// CHECK: }) +// CHECK-SAME: _replication_info = "foo" +// CHECK: "tf.opA"() ({ +// CHECK: "tf.AssignAddVariableOp"([[VAR]], [[CONST]]) + + +// ----- + + +!tf_res = tensor<*x!tf_type.resource>> + +// Test multiple replicated clusters interleaved and uses resource variables. +// CHECK-LABEL: func @multiple_replicated_interleaved +func.func @multiple_replicated_interleaved(%arg0: !tf_res) { + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "a", num_replicas = 2, topology = "topology"} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "b", num_replicas = 2, topology = "topology"} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "c", num_replicas = 2, topology = "topology"} : () -> () + %0 = "tf.TPUReplicatedInput"(%arg0, %arg0) : (!tf_res, !tf_res) -> !tf_res + %1 = "tf.TPUReplicatedInput"(%arg0, %arg0) : (!tf_res, !tf_res) -> !tf_res + %2 = "tf.TPUReplicatedInput"(%arg0, %arg0) : (!tf_res, !tf_res) -> !tf_res + %3 = "tf.ReadVariableOp"(%0) {_xla_compile_device_type = "TPU", _replication_info = "a"} : (!tf_res) -> tensor + %4 = "tf.ReadVariableOp"(%1) {_xla_compile_device_type = "TPU", _replication_info = "b"} : (!tf_res) -> tensor + %5 = "tf.ReadVariableOp"(%2) {_xla_compile_device_type = "TPU", _replication_info = "c"} : (!tf_res) -> tensor + %6 = "tf.Identity"(%3) {_xla_compile_device_type = "TPU", _replication_info = "a"} : (tensor) -> tensor + %7 = "tf.Identity"(%4) {_xla_compile_device_type = "TPU", _replication_info = "b"} : (tensor) -> tensor + %8 = "tf.Identity"(%5) {_xla_compile_device_type = "TPU", _replication_info = "c"} : (tensor) -> tensor + %9:2 = "tf.TPUReplicatedOutput"(%6) : (tensor) -> (tensor, tensor) + %10:2 = "tf.TPUReplicatedOutput"(%7) : (tensor) -> (tensor, tensor) + %11:2 = "tf.TPUReplicatedOutput"(%8) : (tensor) -> (tensor, tensor) + func.return +} + +// CHECK: tf_device.replicate +// CHECK: tf_device.replicate +// CHECK: tf_device.replicate + + +// ----- + + +// Test cluster that is replicated but has a non TPUReplicatedOutput consumer. +// CHECK-LABEL: func @replicated_non_replicated_output +func.func @replicated_non_replicated_output() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + %1 = "tf.opB"(%0) {is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// CHECK: [[REPLICATE:%.+]]:2 = tf_device.replicate +// CHECK: "tf.opB"([[REPLICATE]]#0) + +// ----- + +// TF produces Identity ops between TPUReplicatedOutput and +// TPUPartitionedOutputV2 ops. This test ensures that they are erased +// and not considered within the clustered computation. It also ensures that +// the expected interleaving pattern is present in the output. + +func.func @partitioned_outputs(%arg0: tensor) -> (tensor, tensor, tensor, tensor) { + %pi0 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %pi1 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %1 = "tf.TPUReplicatedInput"(%pi0, %pi1) {is_mirrored_variable = true, is_packed = false} : (tensor, tensor) -> (tensor) + %2 = "tf.opA"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", is_stateless = true} : (tensor) -> (tensor) + %3:2 = "tf.TPUReplicatedOutput"(%2) : (tensor) -> (tensor, tensor) + %4 = "tf.Identity"(%3#0) : (tensor) -> (tensor) + %5:2 = "tf.TPUPartitionedOutputV2"(%4) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + %6 = "tf.Identity"(%3#1) : (tensor) -> (tensor) + %7:2 = "tf.TPUPartitionedOutputV2"(%6) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, num_cores_per_replica = 2, topology = "topology"} : () -> () + func.return %5#0, %5#1, %7#0, %7#1 : tensor, tensor, tensor, tensor +} + +// CHECK: [[REPLICATE:%.+]]:4 = tf_device.replicate +// CHECK: return [[REPLICATE]]#0, [[REPLICATE]]#2, [[REPLICATE]]#1, [[REPLICATE]]#3 + +// ----- + +// Ensures that mixed partitioned and replicated outputs +// works in the multi-replica case. +func.func @mixed_partitioned_outputs(%arg0: tensor) -> (tensor, tensor) { + %pi0 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %pi1 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %1 = "tf.TPUReplicatedInput"(%pi0, %pi1) {is_mirrored_variable = true, is_packed = false} : (tensor, tensor) -> (tensor) + %2:2 = "tf.opA"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", is_stateless = true} : (tensor) -> (tensor, tensor) + %3:2 = "tf.TPUReplicatedOutput"(%2#0) : (tensor) -> (tensor, tensor) + %5:2 = "tf.TPUPartitionedOutputV2"(%3#0) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + %7:2 = "tf.TPUPartitionedOutputV2"(%3#1) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + %8:2 = "tf.TPUReplicatedOutput"(%2#1) : (tensor) -> (tensor, tensor) + %9 = "tf.opB"(%5#0, %5#1, %7#0, %7#1) : (tensor, tensor, tensor, tensor) -> (tensor) + %10 = "tf.opC"(%8#0, %8#1) : (tensor, tensor) -> (tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, num_cores_per_replica = 2, topology = "topology"} : () -> () + func.return %9, %10 : tensor, tensor +} + +// CHECK: [[REPLICATE:%.+]]:6 = tf_device.replicate +// CHECK: [[OP_B:%.+]] = "tf.opB"([[REPLICATE]]#0, [[REPLICATE]]#2, [[REPLICATE]]#1, [[REPLICATE]]#3) +// CHECK: [[OP_C:%.+]] = "tf.opC"([[REPLICATE]]#4, [[REPLICATE]]#5) + +// ----- + +// For the single replica case: +// - Ensures that Identity ops are ignored. +// - Checks that mixing TPUPartitionedOutputV2 and TPUReplicatedOutput works. + +func.func @single_replica_mixed_partitioned_outputs(%arg0: tensor) -> (tensor, tensor, tensor) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %1 = "tf.TPUReplicatedInput"(%0) {is_mirrored_variable = true, is_packed = false} : (tensor) -> (tensor) + %2:2 = "tf.opA"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", is_stateless = true} : (tensor) -> (tensor, tensor) + %3 = "tf.TPUReplicatedOutput"(%2#0) : (tensor) -> (tensor) + %4 = "tf.Identity"(%3) : (tensor) -> (tensor) + %5:2 = "tf.TPUPartitionedOutputV2"(%4) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + %6 = "tf.TPUReplicatedOutput"(%2#1) : (tensor) -> (tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, num_cores_per_replica = 2, topology = "topology"} : () -> () + func.return %5#0, %5#1, %6 : tensor, tensor, tensor +} + +// CHECK: [[CLUSTER:%.+]]:2 = "tf_device.cluster" +// CHECK: [[OUTPUT:%.+]]:2 = "tf.TPUPartitionedOutputV2"([[CLUSTER]]#0) +// CHECK: return [[OUTPUT]]#0, [[OUTPUT]]#1, [[CLUSTER]]#1 + +// ----- + +func.func @replica_mismatch(%arg0: tensor) { + %pi0 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %pi1 = "tf.TPUPartitionedInputV2"(%arg0) {N = 2, partition_dims = [], _XlaSharding = "", is_packed = true} : (tensor) -> (tensor) + %1 = "tf.TPUReplicatedInput"(%pi0, %pi1) {is_mirrored_variable = true, is_packed = false} : (tensor, tensor) -> (tensor) + %2 = "tf.opA"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", is_stateless = true} : (tensor) -> (tensor) + %3:2 = "tf.TPUReplicatedOutput"(%2) : (tensor) -> (tensor, tensor) + %4 = "tf.Identity"(%3#0) : (tensor) -> (tensor) + // expected-error@+1 {{expected zero or 2 'TPUPartitionedOutput' op(s), instead got 1}} + %5:2 = "tf.TPUPartitionedOutputV2"(%4) {_XlaSharding = "", partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, num_cores_per_replica = 2, topology = "topology"} : () -> () + func.return +} + +// ----- + + +// Test cluster with missing `num_replicas` attribute. +func.func @missing_num_replicas() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + // expected-error@+1 {{'tf.TPUReplicateMetadata' op requires attribute 'num_replicas'}} + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", topology = "topology"} : () -> () + func.return +} + + +// ----- + + +// Test cluster with bad `num_replicas` attribute. +func.func @bad_num_replicas() { + // expected-error@+1 {{requires 'num_replicas' int attribute to be at least 1}} + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 0, topology = "topology"} : () -> () + func.return +} + +// ----- + +// Test cluster with bad `num_cores_per_replica` attribute. +!rtype = tensor>> +func.func @replication_with_model_parallelism(%arg0: !rtype, %arg1: !rtype, %arg2: !rtype, %arg3: !rtype) -> (tensor<10x3xf32>) { + %2 = "tf.TPUReplicatedInput"(%arg0, %arg2) : (!rtype, !rtype) -> !rtype + %3 = "tf.TPUReplicatedInput"(%arg1, %arg3) : (!rtype, !rtype) -> !rtype + // expected-error@+1 {{'tf.TPUPartitionedInputV2' op requires 4 operands but found 2}} + %4 = "tf.TPUPartitionedInputV2"(%2, %3) {_XlaSharding = "", device = "", partition_dims = []} : (!rtype, !rtype) -> !rtype + %6 = "tf.opC"(%4) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (!rtype) -> tensor<10x3xf32> + %7:2 = "tf.TPUReplicatedOutput"(%6) : (tensor<10x3xf32>) -> (tensor<10x3xf32>, tensor<10x3xf32>) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_cores_per_replica = 4 : i64, num_replicas = 2 : i64, topology = "topology"} : () -> () + func.return %7#0 : tensor<10x3xf32> +} + +// ----- + + +// Test cluster with TPUReplicatedInput where the number of operands does not +// match associated `num_replicas` attribute. +func.func @mismatched_replicated_input(%arg0: tensor) { + // expected-error@+1 {{'tf.TPUReplicatedInput' op requires 2 operands}} + %0 = "tf.TPUReplicatedInput"(%arg0, %arg0, %arg0) : (tensor, tensor, tensor) -> tensor + %1 = "tf.opA"(%0) {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + + +// ----- + + +// Test cluster with TPUReplicatedOutput where the number of results does not +// match associated `num_replicas` attribute. +func.func @mismatched_replicated_output() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + // expected-error@+1 {{'tf.TPUReplicatedOutput' op requires 2 results}} + %1:3 = "tf.TPUReplicatedOutput"(%0) : (tensor) -> (tensor, tensor, tensor) + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + + +// ----- + + +// Test unused TPUReplicatedInput that has more than one operand. +func.func @leftover_replicated_input(%arg0: tensor) { + %0 = "tf.TPUReplicatedInput"(%arg0, %arg0) : (tensor, tensor) -> tensor + func.return +} + + +// ----- + + +// Test unused TPUReplicatedOutput that has more than one result. +func.func @leftover_replicated_output(%arg0: tensor) { + %0:2 = "tf.TPUReplicatedOutput"(%arg0) : (tensor) -> (tensor, tensor) + func.return +} + + +// ----- + +// CHECK-LABEL: func @cluster_ops_keep_replicated_core_attr +func.func @cluster_ops_keep_replicated_core_attr() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU_REPLICATED_CORE:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// CHECK: "tf.opA" +// CHECK-SAME-DAG: name = "name" +// CHECK-SAME-DAG: device = "/device:TPU_REPLICATED_CORE:0" +// CHECK: tf_device.return + +// ----- + +func.func @missing_replication_or_compilation_attribute() { + // expected-error@+1 {{'tf.opA' op is expected to have either both or none of '_replication_info' and '_xla_compile_device_type' attributes}} + %0 = "tf.opA"() { _replication_info = "replicate", device = "/device:TPU_REPLICATED_CORE:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +func.func @empty_replication_attribute() { + // expected-error@+1 {{'tf.opA' op has an empty '_replication_info' attribute}} + %0 = "tf.opA"() { _xla_compile_device_type = "TPU", _replication_info = "", device = "/device:TPU_REPLICATED_CORE:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +func.func @invalid_device_type() { + // expected-error@+1 {{'tf.opA' op has invalid '_xla_compile_device_type' value 'XPU'}} + %0 = "tf.opA"() {_xla_compile_device_type = "XPU", _replication_info = "replicate", device = "/device:TPU:0", name = "name", is_stateless = true} : () -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- +// expected-error@+1 {{found different '_xla_compile_device_type' attribute values (GPU,TPU) in same block which is not supported}} +func.func @invalid_compilation_cluster_mixed_device_types() { + "tf.opA"() { _xla_compile_device_type = "GPU", _replication_info = "replicate", is_stateless = true} : () -> () + "tf.opB"() { _xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +// expected-error@+1 {{found different '_xla_compile_device_type' attribute values (CPU,GPU) in same block which is not supported}} +func.func @invalid_compilation_replication_cluster_mixed_device_types() { + "tf.opA"() { _xla_compile_device_type = "CPU", _replication_info = "cluster", is_stateless = true} : () -> () + "tf.opB"() { _xla_compile_device_type = "GPU", _replication_info = "cluster", is_stateless = true} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +func.func @cyclic_control_dependency_no_replication() { + "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate"} : () -> () + // expected-warning-re@+1 {{Op has cyclic dependency with a compilation cluster{{.*}}}} + "tf.opB"() : () -> () + "tf.opC"() {_xla_compile_device_type = "TPU", _replication_info = "replicate"} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +func.func @cyclic_data_dependency_no_replication() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : () -> (tensor) + // expected-warning-re@+2 {{Op has cyclic dependency with a compilation cluster{{.*}}}} + // expected-error@+1 {{operand #0 does not dominate this use}} + %1 = "tf.opB"(%0) {is_stateless = true} : (tensor) -> (tensor) + // expected-note@+1 {{operand defined here (op in the same block)}} + "tf.opC"(%1) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} + +// ----- + +func.func @cyclic_control_dependency_replication() { + "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "cluster"} : () -> () + // expected-warning-re@+1 {{Op has cyclic dependency with a compilation cluster{{.*}}}} + "tf.opB"() : () -> () + "tf.opC"() {_xla_compile_device_type = "TPU", _replication_info = "cluster"} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + + +// ----- + +func.func @cyclic_data_dependency_replication() { + %0 = "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", is_stateless = true} : () -> (tensor) + // expected-warning-re@+2 {{Op has cyclic dependency with a compilation cluster{{.*}}}} + // expected-error@+1 {{operand #0 does not dominate this use}} + %1 = "tf.opB"(%0) {is_stateless = true} : (tensor) -> (tensor) + // expected-note@+1 {{operand defined here (op in the same block)}} + "tf.opC"(%1) {_xla_compile_device_type = "TPU", _replication_info = "cluster", is_stateless = true} : (tensor) -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// ----- + +// expected-warning@+1 {{TPUReplicateMetadata for associated '_replication_info' attribute 'cluster' is missing}} +func.func @missing_metadata() { + "tf.opA"() {_xla_compile_device_type = "TPU", _replication_info = "cluster"} : () -> () + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// ----- + +// CHECK-LABEL: func @const_with_attrs +func.func @const_with_attrs(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor, tensor) { + // CHECK: %{{[a-z0-9_]*}} = "tf.Const"() <{value = dense<-1> : tensor<1xi32>}> : () -> tensor<1xi32> + // CHECK-NEXT: %{{[a-z0-9_]*}} = "tf.Reshape"(%arg0 + // CHECK-NEXT: %{{.*}} = "tf_device.cluster"() ({ + %minus_one = "tf.Const"() {_replication_info = "cluster", + _xla_compile_device_type = "TPU", + value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + "tf.TPUReplicateMetadata"() {_replication_info = "cluster", num_replicas = 1 : i64} : () -> () + + %1 = "tf.Reshape"(%arg0, %minus_one) : (tensor<*xi32>, tensor<1xi32>) -> tensor + %2 = "tf.Identity"(%1) {_replication_info = "cluster", _xla_compile_device_type = "TPU"} : (tensor) -> tensor + + %4 = "tf.Reshape"(%arg1, %minus_one) {_replication_info = "cluster", _xla_compile_device_type = "TPU", device = ""} : (tensor, tensor<1xi32>) -> tensor + %5 = "tf.Identity"(%4) {_replication_info = "cluster", _xla_compile_device_type = "TPU"} : (tensor) -> tensor + + func.return %2, %5 : tensor, tensor +} + +// ----- + +// CHECK-LABEL: func @two_clusters +func.func @two_clusters(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor, tensor) { + // CHECK: %{{[a-z0-9_]*}} = "tf.Const"(){{.*}}value = dense<1> + // CHECK-NEXT: %{{[a-z0-9_]*}} = "tf.Const"(){{.*}}value = dense<2> + // CHECK-NEXT: %{{[a-z0-9_]*}} = "tf_device.cluster" + %one = "tf.Const"() {_replication_info = "cluster1", + _xla_compile_device_type = "TPU", + value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %two = "tf.Const"() {_replication_info = "cluster2", + _xla_compile_device_type = "TPU", + value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + "tf.TPUReplicateMetadata"() {_replication_info = "cluster1", num_replicas = 1 : i64} : () -> () + "tf.TPUReplicateMetadata"() {_replication_info = "cluster2", num_replicas = 1 : i64} : () -> () + + %1 = "tf.Reshape"(%arg0, %one) {_replication_info = "cluster2", _xla_compile_device_type = "TPU", device = ""} : (tensor<*xi32>, tensor<1xi32>) -> tensor + %2 = "tf.Identity"(%1) {_replication_info = "cluster2", _xla_compile_device_type = "TPU"} : (tensor) -> tensor + + %3 = "tf.Reshape"(%arg1, %two) {_replication_info = "cluster1", _xla_compile_device_type = "TPU", device = ""} : (tensor, tensor<1xi32>) -> tensor + %4 = "tf.Identity"(%3) {_replication_info = "cluster1", _xla_compile_device_type = "TPU"} : (tensor) -> tensor + + func.return %2, %4 : tensor, tensor +} + +// ----- + +// Check that there is one replicate argument for each TPUReplicatedInput +// even when there are multiple uses of the TPUReplicatedInput. + +// CHECK-LABEL: func @one_arg_per_TRI +func.func @one_arg_per_TRI(%arg0: tensor, %arg1: tensor) -> () { + // CHECK: tf_device.replicate + // CHECK-SAME: [%arg0, %arg1] as + // CHECK-NOT: [%arg0, %arg1] as + %ri = "tf.TPUReplicatedInput"(%arg0, %arg1) : (tensor, tensor) -> tensor + %0 = "tf.opA"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + %1 = "tf.opB"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "replicate", is_stateless = true} : (tensor) -> tensor + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "replicate", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + func.return +} + +// ----- + +// Check that for non-TPU device types we don't cluster. + +// CHECK-LABEL: func @cpu_device +// CHECK-NOT: tf_device.cluster +func.func @cpu_device() { + "tf.opA"() { _xla_compile_device_type = "CPU"} : () -> () + func.return +} + +// ----- + +// CHECK-LABEL: func @gpu_device +// CHECK-NOT: tf_device.cluster +func.func @gpu_device() { + "tf.opA"() { _xla_compile_device_type = "GPU"} : () -> () + func.return +} + +// ----- + +// CHECK-LABEL: func @gather_nd +func.func @gather_nd(%arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<3xf32>) { + // CHECK: ResourceGatherNd + // CHECK: tf_device.cluster + // CHECK: Add + // CHECK: ResourceGatherNd + %0 = "tf.Const"() {value = dense<32> : tensor} : () -> tensor + %1 = "tf.ResourceGatherNd"(%arg0, %0) { + Tindices = i32 + } : (tensor<*x!tf_type.resource>>, tensor) -> tensor<1x80xf32> + %2 = "tf.Add"(%1, %1) { + _xla_compile_device_type = "TPU", _replication_info = "cluster", + device = "/task:0/device:TPU:0", dtype = f32 + } : (tensor<1x80xf32>, tensor<1x80xf32>) -> tensor<1x80xf32> + %3 = "tf.ResourceGatherNd"(%arg0, %0) { + Tindices = i32 + } : (tensor<*x!tf_type.resource>>, tensor) -> tensor<1x80xf32> + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", device = "/device:TPU:0", num_replicas = 1, topology = "topology"} : () -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_composite_resource_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_composite_resource_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8022e4a16633bbe691927a753045073f39addd7a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_composite_resource_ops.mlir @@ -0,0 +1,118 @@ +// RUN: tf-opt %s -tf-tpu-colocate-composite-resource-ops | FileCheck %s + +// Tests ReadVariable op using composite device resource is wrapped inside +// tf_device.Cluster. + +// CHECK-LABEL: func @testReadVariableOpColocated +// CHECK-SAME: (%[[ARG0:.*]]: tensor<*x!tf_type.resource>>) +func.func @testReadVariableOpColocated(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: tf_device.replicate + // CHECK-SAME: (%[[ARG0]] as %[[RI_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>>) + tf_device.replicate(%arg0 as %arg1: tensor<*x!tf_type.resource>>) { + _mirrored_variable_indices = [0], + devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]}, + n = 2 : i32} { + // CHECK: %[[RESOURCE_OUT:.*]] = "tf_device.launch"() + // CHECK-SAME: TPU_REPLICATED_CORE_0 + // CHECK-NEXT: %[[READ_OUT:.*]] = "tf.ReadVariableOp"(%[[RI_0]]) + // CHECK-NEXT: tf_device.return %[[READ_OUT]] + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %1 = "tf.A"() : () -> (tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%arg1, %1) {device_var_reads_indices = [0], device_var_updates_indices = [-1]} : (tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + "tf_device.launch"() ({ + // CHECK: "tf.B"(%[[RESOURCE_OUT]]) + "tf.B"(%0) : (tensor<4xf32>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + } + func.return +} + +// CHECK-LABEL: func @testReadVariableOpAfterIdentityColocated +// CHECK-SAME: (%[[ARG0:.*]]: tensor<*x!tf_type.resource>>) +func.func @testReadVariableOpAfterIdentityColocated(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: tf_device.replicate + // CHECK-SAME: (%[[ARG0]] as %[[RI_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>>) + tf_device.replicate(%arg0 as %arg1: tensor<*x!tf_type.resource>>) { + _mirrored_variable_indices = [0], + devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]}, + n = 2 : i32} { + // CHECK: %[[IDENTITY_OUT:.*]] = "tf.Identity"(%[[RI_0]]) + // CHECK: %[[RESOURCE_OUT:.*]] = "tf_device.launch"() + // CHECK-SAME: TPU_REPLICATED_CORE_0 + // CHECK-NEXT: %[[READ_OUT:.*]] = "tf.ReadVariableOp"(%[[IDENTITY_OUT]]) + // CHECK-NEXT: tf_device.return %[[READ_OUT]] + %0 = "tf.Identity"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<*x!tf_type.resource>> + %1 = "tf.ReadVariableOp"(%0) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + %2 = "tf.A"() : () -> (tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%arg1, %2) {device_var_reads_indices = [0], device_var_updates_indices = [-1]} : (tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + "tf_device.launch"() ({ + // CHECK: "tf.B"(%[[RESOURCE_OUT]]) + "tf.B"(%1) : (tensor<4xf32>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + } + func.return +} + +// Tests AssignVariable op using composite device resource is wrapped inside +// tf_device.Cluster. + +// CHECK-LABEL: func @testAssignVariableOpColocated +// CHECK-SAME: (%[[ARG0:.*]]: tensor<*x!tf_type.resource>>) +func.func @testAssignVariableOpColocated(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: tf_device.replicate + // CHECK-SAME: (%[[ARG0]] as %[[RI_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>>) + tf_device.replicate(%arg0 as %arg1: tensor<*x!tf_type.resource>>) { + _mirrored_variable_indices = [0], + devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]}, + n = 2 : i32} { + // CHECK: %[[VAL_OUT:.*]] = "tf.A"() : () -> tensor<4xf32> + // CHECK: "tf_device.launch"() + // CHECK-SAME: TPU_REPLICATED_CORE_0 + // CHECK-NEXT: "tf.AssignVariableOp"(%[[RI_0]], %[[VAL_OUT]]) + // CHECK: tf_device.return + %1 = "tf.A"() : () -> (tensor<4xf32>) + "tf.AssignVariableOp"(%arg1, %1) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + %2 = "tf.B"() : () -> (tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%arg1, %2) {device_var_reads_indices = [0], device_var_updates_indices = [-1]} : (tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_CORE_0"} : () -> () + tf_device.return + } + func.return +} + +// Tests tf_device.replicate op not running on TPU devices ignored. + +// CHECK-LABEL: func @testNonTPUDeviceReplicationIgnored +// CHECK-SAME: (%[[ARG0:.*]]: tensor<*x!tf_type.resource>>) +func.func @testNonTPUDeviceReplicationIgnored(%arg0: tensor<*x!tf_type.resource>>) { + // CHECK: tf_device.replicate + // CHECK-SAME: (%[[ARG0]] as %[[RI_0:[a-z0-9]*]]: tensor<*x!tf_type.resource>>) + tf_device.replicate(%arg0 as %arg1: tensor<*x!tf_type.resource>>) { + _mirrored_variable_indices = [0], + devices = {TPU_REPLICATED_HOST_0 = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:CPU:1"]}, + n = 2 : i32} { + // CHECK: %[[VAL_OUT:.*]] = "tf.A"() : () -> tensor<4xf32> + // CHECK-NEXT: "tf.AssignVariableOp"(%[[RI_0]], %[[VAL_OUT]]) + %1 = "tf.A"() : () -> (tensor<4xf32>) + "tf.AssignVariableOp"(%arg1, %1) : (tensor<*x!tf_type.resource>>, tensor<4xf32>) -> () + %2 = "tf.B"() : () -> (tensor<2x!tf_type.string>) + "tf_device.launch"() ({ + "tf.TPUExecuteAndUpdateVariables"(%arg1, %2) {device_var_reads_indices = [0], device_var_updates_indices = [-1]} : (tensor<*x!tf_type.resource>>, tensor<2x!tf_type.string>) -> () + tf_device.return + }) {device = "TPU_REPLICATED_HOST_0"} : () -> () + tf_device.return + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_splits.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_splits.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d538212e2d44fe767e3cfcc57e1196578353d196 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_colocate_splits.mlir @@ -0,0 +1,44 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-colocate-splits | FileCheck %s + +// CHECK-LABEL: func @colocate_split_with_pred +func.func @colocate_split_with_pred() { + // CHECK: Split + // CHECK-SAME: _class = ["loc:@class"] + tf_executor.graph { + %c, %control0 = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %a, %control1 = tf_executor.island wraps "tf.A"() {_class = ["loc:@class"]} : () -> (tensor<2xf32>) + %s:2, %control2 = tf_executor.island wraps "tf.Split"(%c, %a) {num_split = 2 : i32} : (tensor, tensor<2xf32>) -> (tensor<1xf32>, tensor<1xf32>) + tf_executor.fetch + } + func.return +} + +// ----- + +// CHECK-LABEL: func @colocate_split_with_pred_results +func.func @colocate_split_with_pred_results() { + // CHECK: Split + // CHECK-SAME: _class = ["loc:@class"] + tf_executor.graph { + %c, %control0 = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %a:2, %control1 = tf_executor.island wraps "tf.A"() {_class = ["loc:@class"]} : () -> (tensor<2xf32>, tensor<2xf32>) + %s:2, %control2 = tf_executor.island wraps "tf.Split"(%c, %a#1) {num_split = 2 : i32} : (tensor, tensor<2xf32>) -> (tensor<1xf32>, tensor<1xf32>) + tf_executor.fetch + } + func.return +} + +// ----- + +// CHECK-LABEL: func @no_colocate_split_has_device +func.func @no_colocate_split_has_device() { + // CHECK: Split + // CHECK-NOT: _class = ["loc:@class"] + tf_executor.graph { + %c, %control0 = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %a, %control1 = tf_executor.island wraps "tf.A"() {_class = ["loc:@class"]} : () -> tensor<2xf32> + %s:2, %control2 = tf_executor.island wraps "tf.Split"(%c, %a) {num_split = 2 : i32, device = "device"} : (tensor, tensor<2xf32>) -> (tensor<1xf32>, tensor<1xf32>) + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_device_propagation.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_device_propagation.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e34acdc3ad136e697c072bc5c3690f90b4af63e7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_device_propagation.mlir @@ -0,0 +1,383 @@ +// RUN: tf-opt %s -tf-tpu-device-propagation | FileCheck %s + +// Tests function passthrough values. + +// CHECK-LABEL: func @testArgToRet +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testArgToRet(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + tf_executor.fetch %arg0 : tensor + } + func.return %0 : tensor +} + +// Tests supported ops. + +// CHECK-LABEL: func @testIdentityOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testIdentityOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%arg0) : (tensor) -> tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @testIdentityNOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, {{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testIdentityNOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg1: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + // CHECK: tf.IdentityN + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:3 = tf_executor.island wraps "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + tf_executor.fetch %1#0, %1#1 : tensor, tensor + } + func.return %0#0, %0#1 : tensor, tensor +} + +// CHECK-LABEL: func @testShapeOp +// CHECK-SAME: ({{%.+}}: tensor<*xi64> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testShapeOp(%arg0: tensor<*xi64> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf.Shape + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Shape"(%arg0) : (tensor<*xi64>) -> tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @testEnterOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testEnterOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf_executor.Enter + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.Enter %arg0 frame "frame" : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @testExitOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testExitOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf_executor.Exit + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.Exit %arg0 : tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// CHECK-LABEL: func @testMergeOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, {{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testMergeOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg1: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + // CHECK: tf_executor.Merge + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:3 = tf_executor.Merge %arg0, %arg1 : tensor + tf_executor.fetch %1#0, %1#1 : tensor, tensor + } + func.return %0#0, %0#1 : tensor, tensor +} + +// CHECK-LABEL: func @testSwitchOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, {{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testSwitchOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg1: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) { + tf_executor.graph { + // CHECK: tf_executor.Switch + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %0:3 = tf_executor.Switch %arg0, %arg1 : tensor {T = "tfdtype$DT_INT64"} + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %2:2 = tf_executor.island wraps "tf.Identity"(%0#1) : (tensor) -> tensor + %3 = tf_executor.ControlTrigger %1#1, %2#1 + tf_executor.fetch %3 : !tf_executor.control + } + func.return +} + +// Tests unsupported op does not have TPU device propagated. + +// CHECK-LABEL: func @testUnsupportedOp +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> tensor +func.func @testUnsupportedOp(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = tf_executor.graph { + // CHECK: tf.UnsupportedOp + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.UnsupportedOp"(%arg0) : (tensor) -> tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// Tests empty devices are overwritten. + +// CHECK-LABEL: func @testEmptyDeviceOverwritten +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testEmptyDeviceOverwritten(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor {tf.device = ""}) { + %0 = tf_executor.graph { + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%arg0) {device = ""} : (tensor) -> tensor + tf_executor.fetch %1#0 : tensor + } + func.return %0 : tensor +} + +// Tests only devices are propagated when all operands are on the same TPU +// device. + +// CHECK-LABEL: func @testOperandsNoDevice +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, {{%.+}}: tensor) +// CHECK-SAME: -> (tensor, tensor) +func.func @testOperandsNoDevice(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg1: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + // CHECK: tf.IdentityN + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:3 = tf_executor.island wraps "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + tf_executor.fetch %1#0, %1#1 : tensor, tensor + } + func.return %0#0, %0#1 : tensor, tensor +} + +// CHECK-LABEL: func @testOperandsDifferentDevice +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, {{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:1"}) +// CHECK-SAME: -> (tensor, tensor) +func.func @testOperandsDifferentDevice(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg1: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:1"}) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + // CHECK: tf.IdentityN + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:1" + %1:3 = tf_executor.island wraps "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + tf_executor.fetch %1#0, %1#1 : tensor, tensor + } + func.return %0#0, %0#1 : tensor, tensor +} + +// Tests op with operand on different device does not have its device +// overwritten. + +// CHECK-LABEL: func @testDifferentOperandAndOpDevice +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +func.func @testDifferentOperandAndOpDevice(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) { + tf_executor.graph { + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:1" + %0:2 = tf_executor.island wraps "tf.Identity"(%arg0) {device = "/job:localhost/replica:0/task:0/device:TPU:1"} : (tensor) -> tensor + tf_executor.fetch %0#1 : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @testDifferentOperandAndResultDevice +// CHECK-SAME: ({{%.+}}: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) +// CHECK-SAME: -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:1"}) +func.func @testDifferentOperandAndResultDevice(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:1"}) { + %0 = tf_executor.graph { + tf_executor.fetch %arg0 : tensor + } + func.return %0 : tensor +} + +// Tests non TPU devices are not propagated. + +// CHECK-LABEL: func @testNonTPUDevice +func.func @testNonTPUDevice(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) { + tf_executor.graph { + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:CPU:0" + %0:2 = tf_executor.island wraps "tf.Identity"(%arg0) : (tensor) -> tensor + tf_executor.fetch %0#1 : !tf_executor.control + } + func.return +} + +// Tests control dependencies are ignored for propagating devices. + +// CHECK-LABEL: func @testControlDependenciesIgnored +func.func @testControlDependenciesIgnored(%arg0: tensor) { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island(%0#1) wraps "tf.Identity"(%arg0) : (tensor) -> tensor + tf_executor.fetch %1#1 : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @testControlDependenciesMismatchedDevices +func.func @testControlDependenciesMismatchedDevices(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:1", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island(%0#1) wraps "tf.Identity"(%arg0) : (tensor) -> tensor + tf_executor.fetch %1#1 : !tf_executor.control + } + func.return +} + +// Tests LoopCond -> Switch where LoopCond has a different device is ignored. + +// CHECK-LABEL: func @testLoopCondSwitchLinkDifferentDevice +func.func @testLoopCondSwitchLinkDifferentDevice() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense : tensor} : () -> tensor + %1:2 = tf_executor.LoopCond %0#0 : (tensor) -> (tensor, !tf_executor.control) {} + %2:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf_executor.Switch + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %3:3 = tf_executor.Switch %2#0, %1#0 : tensor {T = "tfdtype$DT_INT64"} + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %4:2 = tf_executor.island wraps "tf.Identity"(%3#0) : (tensor) -> tensor + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %5:2 = tf_executor.island wraps "tf.Identity"(%3#1) : (tensor) -> tensor + %6 = tf_executor.ControlTrigger %4#1, %5#1 + tf_executor.fetch %6 : !tf_executor.control + } + func.return +} + +// Tests tf_executor.NextIteration.Source/tf_executor.NextIteration.Sink has a +// device when an intermediate op in its loop has a device. + +// CHECK-LABEL: func @testNextIterationNoDevice +func.func @testNextIterationNoDevice() { + tf_executor.graph { + // CHECK: tf_executor.NextIteration.Source + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %0:3 = tf_executor.NextIteration.Source : tensor {T = "tfdtype$DT_INT64"} + // CHECK: tf.Identity + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + // CHECK: tf.IdentityN + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %2:2 = tf_executor.island wraps "tf.IdentityN"(%1#0) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : (tensor) -> tensor + // CHECK: tf_executor.NextIteration.Sink + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + tf_executor.NextIteration.Sink [%0#1] %2#0 : tensor {T = "tfdtype$DT_INT64"} + tf_executor.fetch %0#2 : !tf_executor.control + } + func.return +} + +// Tests tf_executor.NextIteration with mismatched devices does not propagate +// either device. + +// CHECK-LABEL: func @testNextIterationMismatchedDevices +func.func @testNextIterationMismatchedDevices() { + tf_executor.graph { + // CHECK: tf_executor.NextIteration.Source + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:1" + %0:3 = tf_executor.NextIteration.Source : tensor {device = "/job:localhost/replica:0/task:0/device:TPU:1", T = "tfdtype$DT_INT64"} + // CHECK: "tf.Identity"({{.+}}) : + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + // CHECK: tf_executor.NextIteration.Sink + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + tf_executor.NextIteration.Sink [%0#1] %1#0 : tensor {device = "/job:localhost/replica:0/task:0/device:TPU:0", T = "tfdtype$DT_INT64"} + tf_executor.fetch %0#2 : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @testNextIterationMissingSourceDevice +func.func @testNextIterationMissingSourceDevice() { + tf_executor.graph { + // CHECK: tf_executor.NextIteration.Source + %0:3 = tf_executor.NextIteration.Source : tensor {T = "tfdtype$DT_INT64"} + // CHECK: "tf.Identity"({{.+}}) : + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + // CHECK: tf_executor.NextIteration.Sink + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:0" + tf_executor.NextIteration.Sink [%0#1] %1#0 : tensor {device = "/job:localhost/replica:0/task:0/device:TPU:0", T = "tfdtype$DT_INT64"} + tf_executor.fetch %0#2 : !tf_executor.control + } + func.return +} + +// CHECK-LABEL: func @testNextIterationMissingSinkDevice +func.func @testNextIterationMissingSinkDevice() { + tf_executor.graph { + // CHECK: tf_executor.NextIteration.Source + // CHECK-SAME: device = "/job:localhost/replica:0/task:0/device:TPU:1" + %0:3 = tf_executor.NextIteration.Source : tensor {device = "/job:localhost/replica:0/task:0/device:TPU:1", T = "tfdtype$DT_INT64"} + // CHECK: "tf.Identity"({{.+}}) : + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + // CHECK: tf_executor.NextIteration.Sink + tf_executor.NextIteration.Sink [%0#1] %1#0 : tensor {T = "tfdtype$DT_INT64"} + tf_executor.fetch %0#2 : !tf_executor.control + } + func.return +} + +// Tests unsupported functions are not modified. + +// CHECK-LABEL: func @testMultipleBlockFunc +func.func @testMultipleBlockFunc() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + tf_executor.fetch %1#1 : !tf_executor.control + } + cf.br ^bb1 +^bb1: + func.return +} + +// CHECK-LABEL: func @testMultipleGraphs +func.func @testMultipleGraphs() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1:2 = tf_executor.island wraps "tf.Identity"(%0#0) : (tensor) -> tensor + tf_executor.fetch %1#1 : !tf_executor.control + } + tf_executor.graph { + tf_executor.fetch + } + func.return +} + +// CHECK-LABEL: func @testNoGraph +func.func @testNoGraph() -> tensor { + %0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %1 = "tf.Identity"(%0) : (tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @testMismatchedGraphResults +func.func @testMismatchedGraphResults() { + %0 = tf_executor.graph { + %1:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.Identity + // CHECK-NOT: device = "/job:localhost/replica:0/task:0/device:TPU:0" + %2:2 = tf_executor.island wraps "tf.Identity"(%1#0) : (tensor) -> tensor + tf_executor.fetch %2#0 : tensor + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_host_computation_expansion.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_host_computation_expansion.mlir new file mode 100644 index 0000000000000000000000000000000000000000..52db166410d16d6aa0982a30dc21c477f3cfb1c9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_host_computation_expansion.mlir @@ -0,0 +1,95 @@ +// RUN: tf-opt %s -split-input-file -tf-tpu-host-computation-expansion | FileCheck %s + +// Tests expansion of a outside compiled ops at head/tail of TPU computation. + +// CHECK-LABEL: func @identity_at_head_expanded +func.func @identity_at_head_expanded(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: _xla_outside_compilation = "" + "tf_device.cluster"() ({ + %1 = "tf.Identity"(%arg0) : (tensor) -> (tensor) + "tf.B"(%1) {_xla_outside_compilation = "cluster1"} : (tensor) -> () + "tf.C"() : () -> () + tf_device.return + }) : () -> () + func.return +} + +// CHECK-LABEL: func @cast_at_head_expanded +func.func @cast_at_head_expanded(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Cast" + // CHECK-SAME: _xla_outside_compilation = "" + "tf_device.cluster"() ({ + %1 = "tf.Cast"(%arg0) : (tensor) -> (tensor) + "tf.B"(%1) {_xla_outside_compilation = "cluster1"} : (tensor) -> () + "tf.C"() : () -> () + tf_device.return + }) {} : () -> () + func.return +} + +// CHECK-LABEL: func @check_consecutive_unary_ops_outside_compiled +func.func @check_consecutive_unary_ops_outside_compiled(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Cast" + // CHECK-SAME: _xla_outside_compilation = "" + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: _xla_outside_compilation = "" + // CHECK-NEXT: "tf.B" + "tf_device.cluster"() ({ + %1 = "tf.Cast"(%arg0) : (tensor) -> (tensor) + %2 = "tf.Identity"(%1) : (tensor) -> (tensor) + "tf.B"(%2) {_xla_outside_compilation = "cluster1"} : (tensor) -> () + "tf.C"() : () -> () + tf_device.return + }) {} : () -> () + func.return +} + +// CHECK-LABEL: func @check_only_necesarily_ops_outside_compiled +func.func @check_only_necesarily_ops_outside_compiled(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation = "" + // CHECK-NEXT: "tf.B" + "tf_device.cluster"() ({ + %1 = "tf.Identity"(%arg0) : (tensor) -> (tensor) + "tf.B"(%1) : (tensor) -> () + "tf.C"() : () -> () + tf_device.return + }) {} : () -> () + func.return +} + +// CHECK-LABEL: func @check_only_necesarily_ops_outside_compiled_with_chained_ops +func.func @check_only_necesarily_ops_outside_compiled_with_chained_ops(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Cast" + // CHECK-NOT: _xla_outside_compilation + // CHECK-NEXT: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + // CHECK-NEXT: "tf.B" + "tf_device.cluster"() ({ + %1 = "tf.Cast"(%arg0) : (tensor) -> (tensor) + %2 = "tf.Identity"(%1) : (tensor) -> (tensor) + "tf.B"(%2) : (tensor) -> () + "tf.C"() : () -> () + tf_device.return + }) : () -> () + func.return +} + +// CHECK-LABEL: func @check_op_without_usage_not_outside_compiled +func.func @check_op_without_usage_not_outside_compiled(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + "tf_device.cluster"() ({ + "tf.Identity"(%arg0) : (tensor) -> (tensor) + "tf.C"() : () -> () + tf_device.return + }) : () -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_identity_pruning.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_identity_pruning.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d111114d5fdafe9ce155f283ca0fd2a35221e7aa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_identity_pruning.mlir @@ -0,0 +1,93 @@ +// RUN: tf-opt %s -tf-tpu-identity-pruning | FileCheck %s --dump-input=always + +// Tests Identity op in cluster is pruned away. + +// CHECK-LABEL: func @testIdentity +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @testIdentity(%arg0: tensor) { + // CHECK-NOT: "tf.Identity" + // CHECK: "tf_device.cluster" + // CHECK-NEXT: tf_device.return [[ARG0]] + %0 = "tf_device.cluster"() ({ + %1 = "tf.Identity"(%arg0) : (tensor) -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + func.return +} + +// Tests IdentityN op in cluster is pruned away. + +// CHECK-LABEL: func @testIdentityN +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @testIdentityN(%arg0: tensor, %arg1: tensor) { + // CHECK-NOT: "tf.IdentityN" + // CHECK: "tf_device.cluster" + // CHECK-NEXT: tf_device.return [[ARG0]], [[ARG1]] + %0:2 = "tf_device.cluster"() ({ + %1:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + tf_device.return %1#0, %1#1 : tensor, tensor + }) : () -> (tensor, tensor) + func.return +} + +// Tests transitive Identity ops reachable from the cluster are pruned away. + +// CHECK-LABEL: func @testTransitiveIdentity +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @testTransitiveIdentity(%arg0: tensor) { + // CHECK: "tf_device.cluster" + // CHECK: "tf.PartitionedCall"([[ARG0]]) + // CHECK-SAME: f = @callee0 + %0 = "tf_device.cluster"() ({ + %1 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @callee0} : (tensor) -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + func.return +} + +// CHECK-LABEL: func @callee0 +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @callee0(%arg0: tensor) -> tensor { + // CHECK-NOT: "tf.Identity" + // CHECK: "tf.PartitionedCall"([[ARG0]]) + // CHECK-SAME: f = @callee1 + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor + %1 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @callee1} : (tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @callee1 +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @callee1(%arg0: tensor) -> tensor { + // CHECK-NOT: "tf.Identity" + // CHECK: return [[ARG0]] + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// Tests Identity ops not reachable from the cluster are not pruned away. + +// CHECK-LABEL: func @testIdentityOutsideCluster +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @testIdentityOutsideCluster(%arg0: tensor) { + // CHECK: [[IDENTITY:%.*]] = "tf.Identity"([[ARG0]]) + // CHECK: [[CLUSTER:%.*]] = "tf_device.cluster" + // CHECK-NEXT: tf_device.return [[IDENTITY]] + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor + %1 = "tf_device.cluster"() ({ + tf_device.return %0 : tensor + }) : () -> tensor + // CHECK: "tf.PartitionedCall"([[CLUSTER]]) + // CHECK-SAME: f = @callee2 + %2 = "tf.PartitionedCall"(%1) {config = "", config_proto = "", executor_type = "", f = @callee2} : (tensor) -> tensor + func.return +} + +// CHECK-LABEL: func @callee2 +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @callee2(%arg0: tensor) -> tensor { + // CHECK: [[IDENTITY:%.*]] = "tf.Identity"([[ARG0]]) + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor + // CHECK: return [[IDENTITY]] + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_parallel_execute_sink_resource_write.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_parallel_execute_sink_resource_write.mlir new file mode 100644 index 0000000000000000000000000000000000000000..388f4c8d4c7513d35adaa75bdff0db83434038e8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_parallel_execute_sink_resource_write.mlir @@ -0,0 +1,137 @@ +// RUN: tf-opt %s -tf-tpu-parallel-execute-sink-resource-write | env FILECHECK_OPTS="" FileCheck %s + +// CHECK-LABEL: func @multiple_uses +// CHECK-SAME: ({{.+}}: tensor, [[ARG1:%.+]]: tensor) +func.func @multiple_uses(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: [[PARALLEL_EXECUTE:%.+]]:2 = "tf_device.parallel_execute" + %0:2 = "tf_device.parallel_execute"() ({ + tf_device.return %arg0 : tensor + }, { + tf_device.return %arg0 : tensor + // CHECK: }) : () -> (tensor, tensor) + }) : () -> (tensor, tensor) + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG1]], [[PARALLEL_EXECUTE]]#0) + "tf.AssignVariableOp"(%arg1, %0#0) : (tensor, tensor) -> () + // CHECK-NEXT: return [[PARALLEL_EXECUTE]]#0 + func.return %0#0 : tensor +} + +// CHECK-LABEL: func @not_assign_var +// CHECK-SAME: ({{.+}}: tensor, [[ARG1:%.+]]: tensor) +func.func @not_assign_var(%arg0: tensor, %arg1: tensor) { + // CHECK: [[PARALLEL_EXECUTE:%.+]]:2 = "tf_device.parallel_execute" + %0:2 = "tf_device.parallel_execute"() ({ + tf_device.return %arg0 : tensor + }, { + tf_device.return %arg0 : tensor + // CHECK: }) : () -> (tensor, tensor) + }) : () -> (tensor, tensor) + // CHECK-NEXT: "tf.AssignAddVariableOp"([[ARG1]], [[PARALLEL_EXECUTE]]#0) + "tf.AssignAddVariableOp"(%arg1, %0#0) : (tensor, tensor) -> () + func.return +} + +// CHECK-LABEL: func @resource_handle_output +// CHECK-SAME: ([[ARG0:%.+]]: tensor, {{.+}}: tensor) +func.func @resource_handle_output(%arg0: tensor, %arg1: tensor) { + // CHECK: [[PARALLEL_EXECUTE:%.+]]:2 = "tf_device.parallel_execute" + %0:2 = "tf_device.parallel_execute"() ({ + tf_device.return %arg1 : tensor + }, { + tf_device.return %arg1 : tensor + // CHECK: }) : () -> (tensor, tensor) + }) : () -> (tensor, tensor) + // CHECK-NEXT: "tf.AssignVariableOp"([[PARALLEL_EXECUTE]]#0, [[ARG0]]) + "tf.AssignVariableOp"(%0#0, %arg0) : (tensor, tensor) -> () + func.return +} + +// CHECK-LABEL: func @resource_handle_and_value_output +func.func @resource_handle_and_value_output(%arg0: tensor, %arg1: tensor) { + // CHECK: [[PARALLEL_EXECUTE:%.+]]:2 = "tf_device.parallel_execute" + %0:2 = "tf_device.parallel_execute"() ({ + tf_device.return %arg0, %arg1 : tensor, tensor + }, { + tf_device.return + }) : () -> (tensor, tensor) + // CHECK: "tf.AssignVariableOp"([[PARALLEL_EXECUTE]]#1, [[PARALLEL_EXECUTE]]#0) + "tf.AssignVariableOp"(%0#1, %0#0) : (tensor, tensor) -> () + func.return +} + +// CHECK-LABEL: func @resource_handle_after_parallel_execute +func.func @resource_handle_after_parallel_execute(%arg0: tensor) { + // CHECK: [[PARALLEL_EXECUTE:%.+]]:2 = "tf_device.parallel_execute" + %0:2 = "tf_device.parallel_execute"() ({ + tf_device.return %arg0 : tensor + }, { + tf_device.return %arg0 : tensor + // CHECK: }) : () -> (tensor, tensor) + }) : () -> (tensor, tensor) + // CHECK-NEXT: [[VAR:%.+]] = "tf.VarHandleOp" + %1 = "tf.VarHandleOp"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor>> + // CHECK-NEXT: "tf.AssignVariableOp"([[VAR]], [[PARALLEL_EXECUTE]]#0) + "tf.AssignVariableOp"(%1, %0#0) : (tensor>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @replace_single_output +// CHECK-SAME: ([[ARG0:%.+]]: tensor, [[ARG1:%.+]]: tensor, [[ARG2:%.+]]: tensor, [[ARG3:%.+]]: tensor) +func.func @replace_single_output(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) { + // CHECK: {{%.+}}:2 = "tf_device.parallel_execute" + %0:3 = "tf_device.parallel_execute"() ({ + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG3]], [[ARG1]]) + // CHECK-NEXT: tf_device.return [[ARG0]], [[ARG2]] : tensor, tensor + tf_device.return %arg0, %arg1, %arg2 : tensor, tensor, tensor + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: tf_device.return + tf_device.return + // CHECK-NEXT: }) : () -> (tensor, tensor) + }) : () -> (tensor, tensor, tensor) + "tf.AssignVariableOp"(%arg3, %0#1) : (tensor, tensor) -> () + // CHECK-NEXT: return + func.return +} + +// CHECK-LABEL: func @replace_multiple_outputs +// CHECK-SAME: ([[ARG0:%.+]]: tensor, [[ARG1:%.+]]: tensor, [[ARG2:%.+]]: tensor, [[ARG3:%.+]]: tensor, [[ARG4:%.+]]: tensor, [[ARG5:%.+]]: tensor, [[ARG6:%.+]]: tensor) +func.func @replace_multiple_outputs(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, %arg6: tensor) { + // CHECK: {{%.+}}:3 = "tf_device.parallel_execute" + %0:5 = "tf_device.parallel_execute"() ({ + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG5]], [[ARG1]]) + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG6]], [[ARG3]]) + // CHECK-NEXT: tf_device.return [[ARG0]], [[ARG2]], [[ARG4]] : tensor, tensor, tensor + tf_device.return %arg0, %arg1, %arg2, %arg3, %arg4 : tensor, tensor, tensor, tensor, tensor + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: tf_device.return + tf_device.return + // CHECK-NEXT: }) : () -> (tensor, tensor, tensor) + }) : () -> (tensor, tensor, tensor, tensor, tensor) + "tf.AssignVariableOp"(%arg5, %0#1) : (tensor, tensor) -> () + "tf.AssignVariableOp"(%arg6, %0#3) : (tensor, tensor) -> () + // CHECK-NEXT: return + func.return +} + +// CHECK-LABEL: func @replace_multiple_outputs_regions +// CHECK-SAME: ([[ARG0:%.+]]: tensor, [[ARG1:%.+]]: tensor, [[ARG2:%.+]]: tensor, [[ARG3:%.+]]: tensor, [[ARG4:%.+]]: tensor, [[ARG5:%.+]]: tensor, [[ARG6:%.+]]: tensor, [[ARG7:%.+]]: tensor) +func.func @replace_multiple_outputs_regions(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, %arg6: tensor, %arg7: tensor) { + // CHECK: {{%.+}}:4 = "tf_device.parallel_execute" + %0:6 = "tf_device.parallel_execute"() ({ + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG6]], [[ARG1]]) + // CHECK-NEXT: tf_device.return [[ARG0]], [[ARG2]] : tensor, tensor + tf_device.return %arg0, %arg1, %arg2 : tensor, tensor, tensor + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: "tf.AssignVariableOp"([[ARG7]], [[ARG4]]) + // CHECK-NEXT: tf_device.return [[ARG3]], [[ARG5]] : tensor, tensor + tf_device.return %arg3, %arg4, %arg5 : tensor, tensor, tensor + // CHECK-NEXT: }) : () -> (tensor, tensor, tensor, tensor) + }) : () -> (tensor, tensor, tensor, tensor, tensor, tensor) + "tf.AssignVariableOp"(%arg6, %0#1) : (tensor, tensor) -> () + "tf.AssignVariableOp"(%arg7, %0#4) : (tensor, tensor) -> () + // CHECK-NEXT: return + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_partitioned_op_conversion.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_partitioned_op_conversion.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4db7803628dce76f8c60309d7e93342670759519 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_partitioned_op_conversion.mlir @@ -0,0 +1,106 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-partitioned-op-conversion | FileCheck %s + +// CHECK-LABEL:func @replicated +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @replicated(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + // CHECK: [[PI_0:%.*]] = "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + // CHECK: [[PI_1:%.*]] = "tf.TPUPartitionedInputV2"([[ARG2]], [[ARG3]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + // CHECK: [[RI:%.*]] = "tf.TPUReplicatedInput"([[PI_0]], [[PI_1]]) + %pi_0 = "tf.TPUPartitionedInput"(%arg0, %arg1) {_XlaSharding = "", partition_dim = -1 : i64} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInput"(%arg2, %arg3) {_XlaSharding = "", partition_dim = -1 : i64} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[RI]] + func.return %ri : tensor>> +} + +// ----- + +// CHECK-LABEL:func @partitioned_2d +// CHECK-SAME: ([[ARG0:%.*]]: tensor<10x3xf32>, [[ARG1:%.*]]: tensor<10x3xf32>) +func.func @partitioned_2d(%arg0: tensor<10x3xf32>, %arg1: tensor<10x3xf32>) -> tensor<20x3xf32> { + // CHECK: [[PI_0:%.*]] = "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [2, 1] + // CHECK: [[RI:%.*]] = "tf.TPUReplicatedInput"([[PI_0]]) + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 1 : i64} : () -> () + %pi_0 = "tf.TPUPartitionedInput"(%arg0, %arg1) {_XlaSharding = "123", partition_dim = 0 : i64} : (tensor<10x3xf32>, tensor<10x3xf32>) -> tensor<20x3xf32> + %ri = "tf.TPUReplicatedInput"(%pi_0) : (tensor<20x3xf32>) -> tensor<20x3xf32> + // CHECK: return [[RI]] + func.return %ri : tensor<20x3xf32> +} + +// ----- + +// CHECK-LABEL:func @partitioned_2d_resource +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @partitioned_2d_resource(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + // CHECK: [[PI_0:%.*]] = "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [2, 1] + // CHECK: [[PI_1:%.*]] = "tf.TPUPartitionedInputV2"([[ARG2]], [[ARG3]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [2, 1] + // CHECK: [[RI:%.*]] = "tf.TPUReplicatedInput"([[PI_0]], [[PI_1]]) + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %pi_0 = "tf.TPUPartitionedInput"(%arg0, %arg1) {_XlaSharding = "123", partition_dim = 0 : i64} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInput"(%arg2, %arg3) {_XlaSharding = "123", partition_dim = 0 : i64} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[RI]] + func.return %ri : tensor>> +} + +// ----- + +// CHECK-LABEL:func @partitioned_3d +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @partitioned_3d(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + // CHECK: [[PI_0:%.*]] = "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [1, 2, 1] + // CHECK: [[PI_1:%.*]] = "tf.TPUPartitionedInputV2"([[ARG2]], [[ARG3]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [1, 2, 1] + // CHECK: [[RI:%.*]] = "tf.TPUReplicatedInput"([[PI_0]], [[PI_1]]) + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %pi_0 = "tf.TPUPartitionedInput"(%arg0, %arg1) {_XlaSharding = "123", partition_dim = 1 : i64} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInput"(%arg2, %arg3) {_XlaSharding = "123", partition_dim = 1 : i64} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[RI]] + func.return %ri : tensor>> +} + +// ----- + +// CHECK-LABEL:func @partitioned_output_3d +// CHECK-SAME: ([[ARG:%.*]]: tensor>>) +func.func @partitioned_output_3d(%arg: tensor>>) -> tensor>> { + // CHECK: [[PO:%.*]] = "tf.TPUPartitionedOutputV2"([[ARG]]) + // CHECK-SAME: _XlaSharding = "123" + // CHECK-SAME: partition_dims = [1, 2, 1] + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %po:2 = "tf.TPUPartitionedOutput"(%arg) {_XlaSharding = "123", partition_dim = 1 : i64} : (tensor>>) -> (tensor>>, tensor>>) + // CHECK: return [[PO:%.*0]] + func.return %po#0 : tensor>> +} + +// ----- + +func.func @out_of_range_dim(%arg: tensor>>) -> tensor>> { + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + // expected-error @+1 {{cannot partition 'tensor<16x16x16xf32>' (rank = 3) along dimension 3.}} + %po:2 = "tf.TPUPartitionedOutput"(%arg) {_XlaSharding = "123", partition_dim = 3 : i64} : (tensor>>) -> (tensor>>, tensor>>) + func.return %po#0 : tensor>> +} + +// ----- + +func.func @unranked(%arg: tensor>>) -> tensor>> { + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + // expected-error @+1 {{cannot convert op with unranked or non-tensor input type 'tensor<*xf32>'.}} + %po:2 = "tf.TPUPartitionedOutput"(%arg) {_XlaSharding = "123", partition_dim = 3 : i64} : (tensor>>) -> (tensor>>, tensor>>) + func.return %po#0 : tensor>> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_reorder_replicate_and_partitioned_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_reorder_replicate_and_partitioned_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..98fff9380c93e7517f3fa93fe1c3306e0f8f8585 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_reorder_replicate_and_partitioned_inputs.mlir @@ -0,0 +1,149 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-reorder-replicate-partitioned-inputs | FileCheck %s + +// CHECK-LABEL:func @simple +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @simple(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + // CHECK: [[RI_0:%.*]] = "tf.TPUReplicatedInput"([[ARG0]], [[ARG2]]) + // CHECK: [[RI_1:%.*]] = "tf.TPUReplicatedInput"([[ARG1]], [[ARG3]]) + // CHECK: [[PI:%.*]] = "tf.TPUPartitionedInputV2"([[RI_0]], [[RI_1]]) + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[PI]] + func.return %ri : tensor>> +} + +// CHECK-LABEL:func @simple_packed +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>) +func.func @simple_packed(%arg0: tensor>>) -> tensor>> { + // CHECK: "tf.TPUReplicateMetadata"() + // CHECK: [[RI_0:%.*]] = "tf.TPUReplicatedInput"([[ARG0]]) + // CHECK-SAME: is_packed = true + // CHECK: [[RI_1:%.*]] = "tf.TPUReplicatedInput"([[ARG0]]) + // CHECK-SAME: is_packed = true + // CHECK: [[PI:%.*]] = "tf.TPUPartitionedInputV2"([[RI_0]], [[RI_1]]) + // CHECK-SAME: is_packed = false + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %2 = "tf.TPUReplicatedInput"(%0, %1) {is_packed = false} : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[PI]] + func.return %2 : tensor>> +} + +// CHECK-LABEL:func @multi_arg_packed +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>) +func.func @multi_arg_packed(%arg0: tensor>>, %arg1: tensor>>) -> tensor>> { + // CHECK: "tf.TPUReplicateMetadata"() + // CHECK: [[RI_0:%.*]] = "tf.TPUReplicatedInput"([[ARG0]], [[ARG1]]) + // CHECK-SAME: is_packed = false + // CHECK: [[RI_1:%.*]] = "tf.TPUReplicatedInput"([[ARG0]], [[ARG1]]) + // CHECK-SAME: is_packed = false + // CHECK: [[PI:%.*]] = "tf.TPUPartitionedInputV2"([[RI_0]], [[RI_1]]) + // CHECK-SAME: is_packed = false + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.TPUPartitionedInputV2"(%arg1) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %2 = "tf.TPUReplicatedInput"(%0, %1) {is_packed = false} : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[PI]] + func.return %2 : tensor>> +} + +// CHECK-LABEL:func @missing_xla_sharding +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @missing_xla_sharding(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + // CHECK: [[RI_0:%.*]] = "tf.TPUReplicatedInput"([[ARG0]], [[ARG2]]) + // CHECK: [[RI_1:%.*]] = "tf.TPUReplicatedInput"([[ARG1]], [[ARG3]]) + // CHECK: [[PI:%.*]] = "tf.TPUPartitionedInputV2"([[RI_0]], [[RI_1]]) + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {device = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3) {device = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[PI]] + func.return %ri : tensor>> +} + +// Test IR is not modified when none of the operands of tf.TPUReplicaedInput is +// a tf.TPUPartitionedInputV2 op. + +// CHECK-LABEL:func @no_change_to_dag +// CHECK-SAME: ([[ARG0:%.*]]: tensor>>, [[ARG1:%.*]]: tensor>>, [[ARG2:%.*]]: tensor>>, [[ARG3:%.*]]: tensor>>) +func.func @no_change_to_dag(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> (tensor>>, tensor>>, tensor>>) { + // CHECK: [[PI_0:%.*]] = "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {device = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // CHECK: [[PI_1:%.*]] = "tf.TPUPartitionedInputV2"([[ARG2]], [[ARG3]]) + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3) {device = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // CHECK: [[RI:%.*]] = "tf.TPUReplicatedInput"([[ARG0]], [[ARG1]]) + %ri = "tf.TPUReplicatedInput"(%arg0, %arg1) : (tensor>>, tensor>>) -> tensor>> + // CHECK: return [[RI]], [[PI_0]], [[PI_1]] + func.return %ri, %pi_0, %pi_1 : tensor>>, tensor>>, tensor>> +} + +// ----- + +func.func @missing_metadata(%arg0: tensor>>) -> tensor>> { + // expected-error@+1 {{num cores per replica unavailable, metadata missing?}} + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %2 = "tf.TPUReplicatedInput"(%0, %1) {is_packed = false} : (tensor>>, tensor>>) -> tensor>> + func.return %2 : tensor>> +} + +// ----- + +func.func @inconsistent_packing(%arg0: tensor>>) -> tensor>> { + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 2 : i64} : () -> () + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + // expected-error@+1 {{packing should match across ops}} + %1 = "tf.TPUPartitionedInputV2"(%arg0, %arg0) {_XlaSharding = "", partition_dims = [], is_packed = false} : (tensor>>, tensor>>) -> tensor>> + %2 = "tf.TPUReplicatedInput"(%0, %1) {is_packed = false} : (tensor>>, tensor>>) -> tensor>> + func.return %2 : tensor>> +} + +// ----- + +func.func @xla_sharding_mismatch(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3) {_XlaSharding = "123", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // expected-error@+1 {{expects all inputs from 'tf.TPUPartitionedInputV2' ops to have identical XLA sharding}} + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + func.return %ri : tensor>> +} + +// ----- + +func.func @partition_dim_mismatch(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // expected-error@+1 {{expects partition_dims = [] but found [1, 2]}} + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3) {_XlaSharding = "", partition_dims = [1, 2]} : (tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + func.return %ri : tensor>> +} + +// ----- + +func.func @num_partitioned_inputs_mismatch(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>, %arg4: tensor>>) -> tensor>> { + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // expected-error@+1 {{expects 2 operands but found 3}} + %pi_1 = "tf.TPUPartitionedInputV2"(%arg2, %arg3, %arg4) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (tensor>>, tensor>>) -> tensor>> + func.return %ri : tensor>> +} + +// ----- + +func.func @mixed_inputs_to_replicated_op(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>, %arg3: tensor>>) -> tensor>> { + %pi_0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + // expected-error@+1 {{'tf.TPUReplicatedInput' op expects all inputs from 'tf.TPUPartitionedInputV2' ops}} + %ri = "tf.TPUReplicatedInput"(%pi_0, %arg2) {index = 1} : (tensor>>, tensor>>) -> tensor>> + func.return %ri : tensor>> +} + +// ----- + +func.func @num_partitioned_inputs_mismatch_num_cores_per_replica(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>) -> tensor>> { + "tf.TPUReplicateMetadata"() {num_cores_per_replica = 2 : i64, num_replicas = 1 : i64} : () -> () + // expected-error@+1 {{expects 2 operands but found 3}} + %pi = "tf.TPUPartitionedInputV2"(%arg0, %arg1, %arg2) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>, tensor>>) -> tensor>> + %ri = "tf.TPUReplicatedInput"(%pi) : (tensor>>) -> tensor>> + func.return %ri : tensor>> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_resource_partitioning.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_resource_partitioning.mlir new file mode 100644 index 0000000000000000000000000000000000000000..decabe56be96b5625cc92f0b86753d2fdd8a58a0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_resource_partitioning.mlir @@ -0,0 +1,268 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-resource-partition | FileCheck %s + +func.func private @computation(%arg0: tensor) -> tensor + +// CHECK-LABEL: func @read_write_resource +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) +func.func @read_write_resource(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK-DAG: [[READ0:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK-DAG: [[READ1:%.+]] = "tf.ReadVariableOp"([[ARG1]]) + // CHECK: [[INPUT:%.+]] = "tf.TPUPartitionedInputV2"([[READ0]], [[READ1]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + // CHECK: [[COMPUTATION:%.+]] = "tf_device.cluster_func"([[INPUT]]) + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + // CHECK: [[OUTPUT:%.+]]:2 = "tf.TPUPartitionedOutputV2"([[COMPUTATION]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + // CHECK-DAG: "tf.AssignVariableOp"([[ARG0]], [[OUTPUT]]#0) + // CHECK-DAG: "tf.AssignVariableOp"([[ARG1]], [[OUTPUT]]#1) + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @read_write_packed_resource +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>) +func.func @read_write_packed_resource(%arg0: tensor>>) { + // CHECK-DAG: [[READ0:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK: [[INPUT:%.+]] = "tf.TPUPartitionedInputV2"([[READ0]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: is_packed = true + // CHECK-SAME: partition_dims = [] + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + // CHECK: [[COMPUTATION:%.+]] = "tf_device.cluster_func"([[INPUT]]) + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 2 : i64} : (tensor) -> tensor + // CHECK: [[OUTPUT:%.+]]:2 = "tf.TPUPartitionedOutputV2"([[COMPUTATION]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + // CHECK-DAG: "tf.AssignVariableOp"([[ARG0]], [[OUTPUT]]#0) + // CHECK-DAG: "tf.AssignVariableOp"([[ARG0]], [[OUTPUT]]#1) + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @read_only_resource +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) +func.func @read_only_resource(%arg0: tensor>>, %arg1: tensor>>) -> tensor { + // CHECK-DAG: [[READ0:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK-DAG: [[READ1:%.+]] = "tf.ReadVariableOp"([[ARG1]]) + // CHECK: [[INPUT:%.+]] = "tf.TPUPartitionedInputV2"([[READ0]], [[READ1]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + // CHECK: "tf_device.cluster_func"([[INPUT]]) + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + // CHECK-NOT: tf.AssignVariableOp + func.return %2 : tensor +} + +func.func private @computation_two_args(%arg0: tensor, %arg1: tensor) + +// CHECK-LABEL: func @partitioned_variable_multiple_users +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) +func.func @partitioned_variable_multiple_users(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK-DAG: [[READ0:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK-DAG: [[READ1:%.+]] = "tf.ReadVariableOp"([[ARG1]]) + // CHECK: [[INPUT0:%.+]] = "tf.TPUPartitionedInputV2"([[READ0]], [[READ1]]) + // CHECK-DAG: [[READ2:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK-DAG: [[READ3:%.+]] = "tf.ReadVariableOp"([[ARG1]]) + // CHECK: [[INPUT1:%.+]] = "tf.TPUPartitionedInputV2"([[READ2]], [[READ3]]) + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + // CHECK: "tf_device.cluster_func"([[INPUT0]], [[INPUT1]]) + "tf_device.cluster_func"(%1, %2) {func = @computation_two_args, use_spmd_for_xla_partitioning = true} : (tensor, tensor) -> () + func.return +} + +// Tests unsupported cases and IR are not modified. + +// CHECK-LABEL: func @no_spmd +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) +func.func @no_spmd(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK: "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.cluster_func"(%1) {func = @computation} : (tensor) -> tensor + // CHECK: "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + %3 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %4 = "tf.ReadVariableOp"(%3) : (tensor>>) -> tensor + %5 = "tf_device.cluster_func"(%4) {func = @computation, use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return +} + +// CHECK-LABEL: func @read_write_unpartitioned_resource +func.func @read_write_unpartitioned_resource(%arg0: tensor>>) { + // CHECK-NOT: tf.TPUPartitionedInputV2 + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %1 = "tf_device.cluster_func"(%0) {func = @computation} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + "tf.AssignVariableOp"(%arg0, %1) : (tensor>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @read_only_unpartitioned_resource +func.func @read_only_unpartitioned_resource(%arg0: tensor>>) { + // CHECK-NOT: tf.TPUPartitionedInputV2 + %0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %1 = "tf_device.cluster_func"(%0) {func = @computation} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + // CHECK-NOT: tf.AssignVariableOp + func.return +} + +// CHECK-LABEL: func @resource_read_multiple_users +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) -> tensor +func.func @resource_read_multiple_users(%arg0: tensor>>, %arg1: tensor>>) -> tensor { + // CHECK: "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.cluster_func"(%1) {func = @computation} : (tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @non_resource_read_input_write_output +func.func @non_resource_read_input_write_output(%arg0: tensor) -> tensor { + // CHECK-NOT: tf.TPUPartitionedInputV2 + %0 = "tf_device.cluster_func"(%arg0) {func = @computation} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + func.return %0 : tensor +} + +// CHECK-LABEL: func @resource_missing_subtype +// CHECK-SAME: ([[ARG0:%.+]]: tensor, [[ARG1:%.+]]: tensor) +func.func @resource_missing_subtype(%arg0: tensor, %arg1: tensor) { + // CHECK: "tf.TPUPartitionedInputV2"([[ARG0]], [[ARG1]]) + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor, tensor) -> tensor + %1 = "tf.ReadVariableOp"(%0) : (tensor) -> tensor + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + "tf.AssignVariableOp"(%0, %2) : (tensor, tensor) -> () + func.return +} + +// ----- + +func.func private @computation(%arg0: tensor) -> tensor + +func.func @missing_num_cores_per_replica(%arg0: tensor>>) { + // expected-error@+1 {{op num cores per replica unavailable}} + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "", partition_dims = [], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// ----- + +func.func private @computation(%arg0: tensor) -> tensor + +func.func @mismatch_num_cores_per_replica(%arg0: tensor>>) { + // expected-error@+1 {{expects 2 operands but found 3}} + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg0, %arg0) {_XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 2 : i64} : (tensor) -> tensor + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// ----- + +// Check outside compiled that uses a TPUPartitionedInputV2. + +func.func private @computation(%arg0: tensor) -> tensor + +// CHECK-LABEL: func @with_host_process +// CHECK-SAME: ([[ARG0:%.+]]: tensor>>, [[ARG1:%.+]]: tensor>>) +func.func @with_host_process(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK-DAG: [[READ0:%.+]] = "tf.ReadVariableOp"([[ARG0]]) + // CHECK-DAG: [[READ1:%.+]] = "tf.ReadVariableOp"([[ARG1]]) + // CHECK: [[INPUT:%.+]] = "tf.TPUPartitionedInputV2"([[READ0]], [[READ1]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + // CHECK: [[COMPUTATION:%.+]] = "tf_device.parallel_execute"() + // CHECK: "tf.OpA"([[READ0]]) + %2 = "tf_device.parallel_execute"() ({ + "tf_device.launch"() ({ + "tf.OpA"(%1) : (tensor) -> () + tf_device.return + }) {device = "TPU_REPLICATED_HOST_0"} : () -> () + tf_device.return + }, { + %3 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + tf_device.return %3 : tensor + }) : () -> tensor + // CHECK: [[OUTPUT:%.+]]:2 = "tf.TPUPartitionedOutputV2"([[COMPUTATION]]) + // CHECK-SAME: _XlaSharding = "" + // CHECK-SAME: partition_dims = [] + // CHECK-DAG: "tf.AssignVariableOp"([[ARG0]], [[OUTPUT]]#0) + // CHECK-DAG: "tf.AssignVariableOp"([[ARG1]], [[OUTPUT]]#1) + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// ----- + +// Check for an error that reports the unsupported case of outside compiled +// code that uses a TPUPartitionedInputV2 without REPLICATED sharding. + +// The TPUParitionedInput has the following OpSharding: +// Proto debug string: +// input 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" + +func.func private @computation(%arg0: tensor) -> tensor + +func.func @non_replicated_sharding(%arg0: tensor>>, %arg1: tensor>>) { + // expected-error@+1 {{support}} + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "\08\03\1A\02\01\02\22\02\00\01", partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.parallel_execute"() ({ + "tf_device.launch"() ({ + "tf.OpA"(%1) : (tensor) -> () + tf_device.return + }) {device = "TPU_REPLICATED_HOST_0"} : () -> () + tf_device.return + }, { + %3 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + tf_device.return %3 : tensor + }) : () -> tensor + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} + +// ----- + +func.func private @computation(%arg0: tensor) -> tensor + +func.func @packed_replicated(%arg0: tensor>> {tf.device = "COMPOSITE"}) { + // expected-error@+1 {{support}} + %0 = "tf.TPUPartitionedInputV2"(%arg0, %arg0) {_XlaSharding = "", partition_dims = [], is_packed = false} : (tensor>>, tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor + %2 = "tf_device.parallel_execute"() ({ + "tf_device.launch"() ({ + "tf.OpA"(%1) : (tensor) -> () + tf_device.return + }) {device = "TPU_REPLICATED_HOST_0"} : () -> () + tf_device.return + }, { + %3 = "tf_device.cluster_func"(%1) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + tf_device.return %3 : tensor + }) : () -> tensor + "tf.AssignVariableOp"(%0, %2) : (tensor>>, tensor) -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_rewrite.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_rewrite.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0cfa8464587707f5f9285f92d4689f1e3c47956a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_rewrite.mlir @@ -0,0 +1,3007 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics --tf-tpu-rewrite=tpu-compile-metadata-debug | env FILECHECK_OPTS="" FileCheck %s + +// Tests module with missing `tf.versions` attribute. + +// expected-error@+1 {{requires attribute 'tf.versions'}} +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_tf_versions() { + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests collecting compilation and execution devices results in an error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_devices() { + // expected-error@+1 {{error in fetching TPU compilation/execution devices: no TPU_SYSTEM devices found}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `num_cores_per_replicas` +// attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_num_cores_per_replica() { + // expected-error@+1 {{requires attribute 'num_cores_per_replica'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad `num_cores_per_replicas` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_num_cores_per_replica() { + // expected-error@+1 {{requires attribute 'num_cores_per_replica'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = "", step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `step_marker_location` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_num_cores_per_replica() { + // expected-error@+1 {{requires attribute 'step_marker_location'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad `step_marker_location` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_step_marker_location() { + // expected-error@+1 {{requires attribute 'step_marker_location'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = 1, topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with unparsable `step_marker_location` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @unparsable_step_marker_location() { + // expected-error@+1 {{bad 'step_marker_location' attribute with value 'test'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "test", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `topology` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_topology() { + // expected-error@+1 {{requires attribute 'topology'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad `topology` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_topology() { + // expected-error@+1 {{requires attribute 'topology'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = 1 : i32, device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with `topology` attribute resulting in device assignment error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @invalid_topology() { + // expected-error@+1 {{error in fetching TPU compilation/execution devices}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "test", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `device_assignment` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_device_assignment() { + // expected-error@+1 {{requires attribute 'device_assignment'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad `device_assignment` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_device_assignment() { + // expected-error@+1 {{requires attribute 'device_assignment'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = "", input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad element in `device_assignment` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_element_device_assignment() { + // expected-error@+1 {{bad 'device_assignment' attribute at index 0, not an int}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [""], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// The following topology is used in subsequent test cases: +// Proto debug string: +// mesh_shape: 1 +// mesh_shape: 1 +// mesh_shape: 1 +// mesh_shape: 2 +// num_tasks: 1 +// num_tpu_devices_per_task: 2 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// Serialized string: +// "\0A\04\01\01\01\02\10\01\18\02\22\06\00\00\00\00\00\00\00\01" + +// ----- + +// Tests `tf_device.cluster_func` with `device_assignment` attribute resulting in device assignment error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @invalid_device_assignment() { + // expected-error@+1 {{error in fetching TPU compilation/execution devices}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\03\01\01\02\10\01\18\02\22\06\00\00\00\00\00\01", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `input_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_input_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{requires attribute 'input_sharding_configuration'}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", device_assignment = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// The following op sharding is used in subsequent test cases: +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\00" + +// ----- + +// Tests `tf_device.cluster_func` with bad `input_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_input_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{requires attribute 'input_sharding_configuration'}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = "", output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with mismatched `input_sharding_configuration` attribute size. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @mismatched_size_input_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'input_sharding_configuration' attribute, expected array attribute of size 1, got size 0}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with unsupported operand type. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @unsupported_operand_type(%arg0: tensor) { + // expected-error@+1 {{failed to determine operand type at index 0: Converting i3 to DataType}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad element in `input_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_element_input_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'input_sharding_configuration' attribute at index 0, not a string}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [1], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with unparsable element in `input_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @unparsable_element_input_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'input_sharding_configuration' attribute at index 0 with value 'test': failed to parse to xla::OpSharding}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["test"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with missing `output_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_output_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{requires attribute 'output_sharding_configuration'}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with bad `output_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_output_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{requires attribute 'output_sharding_configuration'}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = "", use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with mismatched `output_sharding_configuration` attribute size. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @mismatched_size_output_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'output_sharding_configuration' attribute, expected array attribute of size 1, got size 0}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + + +// Tests `tf_device.cluster_func` with bad element in `output_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @bad_element_output_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'output_sharding_configuration' attribute at index 0, not a string}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = [1], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with unparsable element in `output_sharding_configuration` attribute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @unparsable_element_output_sharding_configuration(%arg0: tensor) { + // expected-error@+1 {{bad 'output_sharding_configuration' attribute at index 0 with value 'test': failed to parse to xla::OpSharding}} + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["test"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + func.return + } + func.func @empty_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests `tf_device.cluster_func` with empty `step_marker_location` attribute +// defaults to `STEP_MARK_AT_ENTRY`. +// +// The expected TPUCompileMetadataProto is: +// num_replicas: 1 +// num_cores_per_replica: 1 + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @default_step_marker_location + func.func @default_step_marker_location() { + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + // CHECK: metadata + // CHECK-SAME: num_replicas: 1 + // CHECK-SAME: num_cores_per_replica: 1 + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests argument with unranked shape. Empty shape should be populated in the +// metadata for associated argument. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @unranked_shape_arg + func.func @unranked_shape_arg(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: metadata + // CHECK-SAME: shape {\0A unknown_rank: true + + func.return %0: tensor<*xi32> + } + func.func @_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + func.return %arg0 : tensor<*xi32> + } +} + +// ----- + +// Tests argument with partial shape. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @partial_shape_arg + func.func @partial_shape_arg(%arg0: tensor) -> tensor { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: metadata + // CHECK-SAME: args + // CHECK-SAME: shape {\0A dim {\0A size: -1\0A }\0A dim {\0A size: -1\0A }\0A dim {\0A size: 3\0A }\0A } + func.return %0: tensor + } + func.func @_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +// Tests argument with static shape. + +// The expected TensorShapeProto is: +// shape { +// dim { +// size: 1 +// } +// dim { +// size: 2 +// } +// dim { +// size: 3 +// } +// } + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @static_shape_arg + func.func @static_shape_arg(%arg0: tensor<1x2x3xi32>) -> tensor<1x2x3xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<1x2x3xi32>) -> tensor<1x2x3xi32> + // CHECK: metadata + // CHECK-SAME: args + // CHECK-SAME: shape + // CHECK-SAME: dim + // CHECK-SAME: size: 1 + // CHECK-SAME: dim + // CHECK-SAME: size: 2 + // CHECK-SAME: dim + // CHECK-SAME: size: 3 + + func.return %0: tensor<1x2x3xi32> + } + func.func @_func(%arg0: tensor<1x2x3xi32>) -> tensor<1x2x3xi32> { + func.return %arg0 : tensor<1x2x3xi32> + } +} + +// ----- + +// Tests argument that is a resource variable. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @resource_arg + func.func @resource_arg(%arg0: tensor<*x!tf_type.resource>) -> tensor<*x!tf_type.resource> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<*x!tf_type.resource>) -> tensor<*x!tf_type.resource> + // CHECK: metadata + // CHECK: dtype: DT_RESOURCE + // CHECK-SAME: kind: VARIABLE + + func.return %0: tensor<*x!tf_type.resource> + } + func.func @_func(%arg0: tensor<*x!tf_type.resource>) -> tensor<*x!tf_type.resource> { + func.return %arg0 : tensor<*x!tf_type.resource> + } +} + +// ----- + +// Tests argument that is a parameter. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @parameter_arg + func.func @parameter_arg(%arg0: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<*xf32>) -> tensor<*xf32> + // CHECK: metadata + // CHECK: dtype: DT_FLOAT + // CHECK-SAME: kind: PARAMETER + + func.return %0: tensor<*xf32> + } + func.func @_func(%arg0: tensor<*xf32>) -> tensor<*xf32> { + func.return %arg0 : tensor<*xf32> + } +} + +// ----- + +// Tests metadata is populated correctly based on cluster_func op and attributes. +// +// The expected TPUCompileMetadataProto is: +// args { +// dtype: DT_INT32 +// shape { +// dim { +// size: 8 +// } +// . } +// kind: PARAMETER +// sharding { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// } +// } +// retvals { +// sharding { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// } +// } +// num_replicas: 1 +// num_cores_per_replica: 1 +// step_marker_location: STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @metadata + func.func @metadata(%arg0: tensor<8xi32>) -> tensor<8xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> tensor<8xi32> + // CHECK: metadata + // CHECK-SAME: args + // CHECK-SAME: dtype: DT_INT32 + // CHECK-SAME: shape + // CHECK-SAME: dim + // CHECK-SAME: size: 8 + // CHECK-SAME: kind: PARAMETER + // CHECK-SAME: sharding + // CHECK-SAME: type: MAXIMAL + // CHECK-SAME: tile_assignment_dimensions: 1 + // CHECK-SAME: tile_assignment_devices: 0 + // CHECK-SAME: retvals + // CHECK-SAME: sharding + // CHECK-SAME: type: MAXIMAL + // CHECK-SAME: tile_assignment_dimensions: 1 + // CHECK-SAME: tile_assignment_devices: 0 + // CHECK-SAME: num_replicas: 1 + // CHECK-SAME: num_cores_per_replica: 1 + // CHECK-SAME: step_marker_location: STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP + + func.return %0: tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + +// ----- + +// Tests metadata is populated correctly for use_spmd_for_xla_partitioning == +// true. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @metadata_use_spmd + func.func @metadata_use_spmd(%arg0: tensor<8xi32>) -> tensor<8xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = true} : (tensor<8xi32>) -> tensor<8xi32> + // CHECK: metadata + // CHECK-SAME: use_spmd_for_xla_partitioning: true + func.return %0: tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + +// ----- + +// Tests metadata is populated correctly for is_same_data_across_replicas. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @metadata_same_data_across_replicas + func.func @metadata_same_data_across_replicas(%arg0: tensor<8xi32>) -> tensor<8xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> tensor<8xi32> + // CHECK: metadata + // CHECK-SAME: is_same_data_across_replicas: true + // CHECK-SAME: mhlo.is_same_data_across_replicas = true + func.return %0: tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32> {mhlo.is_same_data_across_replicas = true}) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + +// ----- + +// Tests shape ops are only generated for operands with non static shapes. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @static_and_dynamic_shapes + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_1:[a-z0-9]*]]: tensor<8xi32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<8xi32>) + func.func @static_and_dynamic_shapes(%arg0: tensor<*xi32>, %arg1: tensor<8xi32>, %arg2: tensor<*xi32>, %arg3: tensor<8xi32>) -> tensor<8xi32> { + // CHECK-NOT: "tf.Shape"(%[[ARG_1]]) + // CHECK-NOT: "tf.Shape"(%[[ARG_3]]) + // CHECK: %[[ARG_0_SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_0]]) + // CHECK: %[[ARG_2_SHAPE:[0-9]*]] = "tf.Shape"(%[[ARG_2]]) + %0 = "tf_device.cluster_func"(%arg0, %arg1, %arg2, %arg3) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<*xi32>, tensor<8xi32>, tensor<*xi32>, tensor<8xi32>) -> tensor<8xi32> + // CHECK: "tf._TPUCompileMlir"(%[[ARG_0_SHAPE]], %[[ARG_2_SHAPE]]) + + func.return %0: tensor<8xi32> + } + func.func @_func(%arg0: tensor<*xi32>, %arg1: tensor<8xi32>, %arg2: tensor<*xi32>, %arg3: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg1 : tensor<8xi32> + } +} + +// ----- + +// Tests simple case of `tf_device.cluster_func` on TPU with single input and +// single output. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @single_tpu_cluster_func + func.func @single_tpu_cluster_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-SAME: device = "/job:worker/replica:0/task:0/device:TPU:0" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests simple case of `tf_device.cluster_func` on TPU with replication. Under +// data parallelism replicated host devices are also added to the +// tf_device.replicate + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + // CHECK-LABEL: func @replicated_tpu_cluster_func + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor) + func.func @replicated_tpu_cluster_func(%arg0: tensor) -> tensor { + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + %0 = "tf.A"(%arg0) : (tensor) -> tensor + + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + // CHECK-SAME: ([%[[A_OUTPUT]], %[[ARG_0]]] as %[[RI_0:[a-z0-9]*]]: tensor) + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"], TPU_REPLICATED_HOST_0 = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:CPU:0"]} + // CHECK-SAME: n = 2 + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[RI_0]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[RI_0]], %[[COMPILE_OUTPUT]]#1) + %2 = "tf_device.cluster_func"(%ri_0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + + // CHECK: tf_device.return %[[EXECUTE_OUTPUT]] + tf_device.return %2 : tensor + } + + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[REPLICATE]]#1) + %2 = "tf.C"(%1#1) : (tensor) -> tensor + + // CHECK: return %[[C_OUTPUT]] + func.return %2 : tensor + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests that cluster_func without _xla_compile_device_type = "TPU", _replication_info attribute is ignored. + +module attributes {tf.versions = {producer = 888 : i32}} { + // CHECK-LABEL: func @single_gpu_cluster_func + func.func @single_gpu_cluster_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + + %1 = "tf_device.cluster_func"(%0) {device = "gpu0", func = @gpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: tf_device.cluster_func + // CHECK-SAME: func = @gpu0_func + // CHECK-SAME: device = "gpu0" + // CHECK-SAME: num_cores_per_replica = 1 + // CHECK-SAME: step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP" + // CHECK-NOT: metadata + + func.return %1 : tensor + } + + func.func @gpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests of `tf_device.cluster_func` on TPU with nested function calls. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @with_nested_func + func.func @with_nested_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-SAME: func private @nested_func + // CHECK-SAME: tf.D + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + %1 = func.call @nested_func(%0) : (tensor) -> tensor + func.return %1 : tensor + } + + func.func @nested_func(%arg0: tensor) -> tensor { + %0 = "tf.D"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests of `tf_device.cluster_func` on TPU with referenced function that's not +// via a standard call op. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @with_referenced_func + func.func @with_referenced_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-SAME: func private @referenced_func + // CHECK-SAME: tf.D + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) {body = @referenced_func} : (tensor) -> tensor + func.return %0 : tensor + } + + func.func @referenced_func(%arg0: tensor) -> tensor { + %0 = "tf.D"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests rewriting `tf_device.cluster_func` on TPU with a chain of referenced +// functions. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @with_referenced_func_chain + func.func @with_referenced_func_chain(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-SAME: @referenced_func1 + // CHECK-SAME: tf.D + // CHECK-SAME: @referenced_func2 + // CHECK-SAME: tf.E + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) {body = @referenced_func1} : (tensor) -> tensor + func.return %0 : tensor + } + + func.func @referenced_func1(%arg0: tensor) -> tensor { + %0 = "tf.D"(%arg0) : (tensor) -> tensor + %1 = func.call @referenced_func2(%0) : (tensor) -> tensor + func.return %1 : tensor + } + + func.func @referenced_func2(%arg0: tensor) -> tensor { + %0 = "tf.E"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests rewriting `tf_device.cluster_func` on TPU with multiple calls to same +// function. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @with_multiple_call_same_referenced_func + func.func @with_multiple_call_same_referenced_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-COUNT-2: call @referenced_func + // CHECK-COUNT-1: func private @referenced_func + // CHECK-SAME: tf.D + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) {body = @referenced_func1} : (tensor) -> tensor + %1 = func.call @referenced_func(%0) : (tensor) -> tensor + %2 = func.call @referenced_func(%1) : (tensor) -> tensor + func.return %2 : tensor + } + + func.func @referenced_func(%arg0: tensor) -> tensor { + %1 = "tf.D"(%arg0) : (tensor) -> tensor + func.return %1 : tensor + } +} + +// ----- + +// Tests multiple `tf_device.cluster_func` on TPU with different computation. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @multiple_cluster_different_func + func.func @multiple_cluster_different_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func0, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-NOT: func = @tpu0_func0 + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE0_OUTPUT]]#0) + // CHECK: %[[EXECUTE0_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE0_OUTPUT]]#1) + + %2 = "tf_device.cluster_func"(%1) {_xla_compile_device_type = "TPU", _replication_info = "cluster1", func = @tpu0_func1, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[EXECUTE0_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[EXECUTE0_OUTPUT]]) + // CHECK: %[[COMPILE1_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[EXECUTE0_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.D + // CHECK-NOT: func = @tpu0_func1 + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE1_OUTPUT]]#0) + // CHECK: %[[EXECUTE1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[EXECUTE0_OUTPUT]], %[[COMPILE1_OUTPUT]]#1) + + %3 = "tf.C"(%2) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE1_OUTPUT]]) + + func.return %3 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func0(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } + + func.func @tpu0_func1(%arg0: tensor) -> tensor { + %0 = "tf.D"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests multiple `tf_device.cluster_func` on TPU with same computation. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @multiple_cluster_same_func + func.func @multiple_cluster_same_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE0_OUTPUT]]#0) + // CHECK: %[[EXECUTE0_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE0_OUTPUT]]#1) + + %2 = "tf_device.cluster_func"(%1) {_xla_compile_device_type = "TPU", _replication_info = "cluster1", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[EXECUTE0_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[EXECUTE0_OUTPUT]]) + // CHECK: %[[COMPILE1_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[EXECUTE0_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-NOT: func = @tpu0_func + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE1_OUTPUT]]#0) + // CHECK: %[[EXECUTE1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[EXECUTE0_OUTPUT]], %[[COMPILE1_OUTPUT]]#1) + + %3 = "tf.C"(%2) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE1_OUTPUT]]) + + func.return %3 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests Functions referenced by TPU function via SymbolRefAttr nested in +// ArrayAttr and DictionaryAttr. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @single_tpu_cluster_func + func.func @single_tpu_cluster_func(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + + %1 = "tf_device.cluster_func"(%0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + // CHECK: %[[A_SHAPE_OUTPUT:[0-9]*]] = "tf.Shape"(%[[A_OUTPUT]]) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"(%[[A_SHAPE_OUTPUT]]) + // CHECK-SAME: metadata + // CHECK-SAME: mlir_module + // CHECK-SAME: func @main + // CHECK-SAME: tf.B + // CHECK-SAME: func private @referenced_func3 + // CHECK-SAME: tf.I + // CHECK-SAME: func private @referenced_func2 + // CHECK-SAME: tf.H + // CHECK-SAME: func private @referenced_func1 + // CHECK-SAME: tf.G + // CHECK-SAME: func private @referenced_func0 + // CHECK-SAME: tf.F + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute"(%[[A_OUTPUT]], %[[COMPILE_OUTPUT]]#1) + + %2 = "tf.C"(%1) : (tensor) -> tensor + // CHECK: %[[C_OUTPUT:[0-9]*]] = "tf.C"(%[[EXECUTE_OUTPUT]]) + + func.return %2 : tensor + // CHECK: return %[[C_OUTPUT]] + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + %1 = "tf.D"(%0) {array_attr_funcs = [@referenced_func0, @referenced_func1]} : (tensor) -> tensor + %2 = "tf.E"(%1) {dictionary_attr_funcs = {fn1 = @referenced_func2, fn2 = @referenced_func3}} : (tensor) -> tensor + func.return %0 : tensor + } + + func.func @referenced_func0(%arg0: tensor) -> tensor { + %1 = "tf.F"(%arg0) : (tensor) -> tensor + func.return %1 : tensor + } + + func.func @referenced_func1(%arg0: tensor) -> tensor { + %1 = "tf.G"(%arg0) : (tensor) -> tensor + func.return %1 : tensor + } + + func.func @referenced_func2(%arg0: tensor) -> tensor { + %1 = "tf.H"(%arg0) : (tensor) -> tensor + func.return %1 : tensor + } + + func.func @referenced_func3(%arg0: tensor) -> tensor { + %1 = "tf.I"(%arg0) : (tensor) -> tensor + func.return %1 : tensor + } +} + +// ----- + +// Test `tf_device.cluster_func` on TPU with pre-split replicate sharded +// input/output using `tf.TPUPartitionedInputV2` and `tf.TPUPartitionedOutputV2`. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK: %[[READ_VAR_0:[0-9]*]] = "tf.ReadVariableOp"(%arg0) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: %[[READ_VAR_1:[0-9]*]] = "tf.ReadVariableOp"(%arg1) + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + // CHECK-NOT: tf.TPUPartitionedInputV2 + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {N = 2 : i64, partition_dims = []} : (tensor, tensor) -> tensor + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:3 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK: "tf_device.launch" + // CHECK: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: [[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:2 = "tf_device.parallel_execute" + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_VAR_0]], %[[COMPILE_OUTPUT]]#1) + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:TPU:1"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_VAR_1]], %[[COMPILE_OUTPUT]]#2) + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + // CHECK-NOT: tf.TPUPartitionedOutputV2 + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + // CHECK: "tf.AssignVariableOp"(%arg0, %[[PARALLEL_EXECUTE_OUTPUT]]#0) + // CHECK: "tf.AssignVariableOp"(%arg1, %[[PARALLEL_EXECUTE_OUTPUT]]#1) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} + +// ----- + +// Test `tf_device.cluster_func` on TPU with pre-split tile sharded input/ +// output using `tf.TPUPartitionedInputV2` and `tf.TPUPartitionedOutputV2`. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + // CHECK: %[[READ_VAR_0:[0-9]*]] = "tf.ReadVariableOp"(%arg0) + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor<3x2xf32> + // CHECK: %[[READ_VAR_1:[0-9]*]] = "tf.ReadVariableOp"(%arg1) + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor<3x2xf32> + // CHECK-NOT: tf.TPUPartitionedInputV2 + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {_XlaSharding = "\08\03\1A\02\01\02\22\02\00\01", partition_dims = [1, 2]} : (tensor<3x2xf32>, tensor<3x2xf32>) -> tensor<3x4xf32> + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:3 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE_OUTPUT]]#0) + // CHECK: [[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:2 = "tf_device.parallel_execute" + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_VAR_0]], %[[COMPILE_OUTPUT]]#1) + // CHECK: "tf_device.launch"() <{device = "/job:worker/replica:0/task:0/device:TPU:1"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[READ_VAR_1]], %[[COMPILE_OUTPUT]]#2) + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01"], output_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01"], use_spmd_for_xla_partitioning = true} : (tensor<3x4xf32>) -> tensor<3x4xf32> + // CHECK-NOT: tf.TPUPartitionedOutputV2 + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {_XlaSharding = "\08\03\1A\02\01\02\22\02\00\01", partition_dims = [1, 2]} : (tensor<3x4xf32>) -> (tensor<3x2xf32>, tensor<3x2xf32>) + // CHECK: "tf.AssignVariableOp"(%arg0, %[[PARALLEL_EXECUTE_OUTPUT]]#0) + // CHECK: "tf.AssignVariableOp"(%arg1, %[[PARALLEL_EXECUTE_OUTPUT]]#1) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor<3x2xf32>) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor<3x2xf32>) -> () + func.return + } + func.func @computation(%arg0: tensor<3x4xf32>) -> tensor<3x4xf32> { + func.return %arg0: tensor<3x4xf32> + } +} + +// ----- + +// Test that unsupported input sharding type of TPUPartitionedInputV2Op inputs of +// ClusterFuncOp result in error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {N = 2 : i64, partition_dims = []} : (tensor, tensor) -> tensor + // expected-error@+1 {{unsupported input sharding type MAXIMAL for 0-th input}} + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], + input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} + +// ----- + +// Test that unsupported output sharding type of TPUPartitionedOutputV2Op outputs +// of ClusterFuncOp result in error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {N = 2 : i64, partition_dims = []} : (tensor, tensor) -> tensor + // expected-error@+1 {{unsupported output sharding type MAXIMAL for 0-th output}} + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], + input_sharding_configuration = [""], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} + +// ----- + +// Test that multiple uses of ClusterFuncOp output along with +// TPUPartitionedOutputV2Op results in error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {N = 2 : i64, partition_dims = []} : (tensor, tensor) -> tensor + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + // expected-error@+1 {{'tf.TPUPartitionedOutputV2' op must be a unique user of TPU Cluster}} + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + "tf._SomeOp"(%computation) : (tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} +// ----- + +// Test that ParallelExecute that is not a direct parent of ClusterFunc results +// in an error. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0 : tensor) { + "tf_device.parallel_execute"() ({ + "tf.IfRegion"(%arg0) ({ + // expected-error@+1 {{The ParallelExecute ancestor of a ClusterFunc must be its direct parent.}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) {is_stateless = false, device = "/device:CPU:0"} : (tensor) -> () + tf_device.return + }) : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Tests that TPUCompilationResult operations are properly rewritten. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + // CHECK-LABEL: func @tpu_compilation_result + func.func @tpu_compilation_result(%arg0: tensor) -> (tensor, tensor, tensor) { + + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: %[[COMPILE_RESULT_0:.*]] = "tf.Identity"(%[[COMPILE_OUTPUT]]#0) + // CHECK: %[[COMPILE_RESULT_1:.*]] = "tf.Identity"(%[[COMPILE_RESULT_0]]) + // CHECK: "tf_device.launch" + // CHECK-NEXT: "tf.TPUCompileSucceededAssert" + // CHECK: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: "tf.TPUExecute" + %1 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + + %compile_result = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster0"} : () -> tensor + %compile_result2 = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster0"} : () -> tensor + + // CHECK-NOT: "tf.TPUCompilationResult" + + // CHECK: return %[[EXECUTE_OUTPUT]], %[[COMPILE_OUTPUT]]#0, %[[COMPILE_OUTPUT]]#0 + func.return %1, %compile_result, %compile_result2 : tensor, tensor, tensor + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests simple case of `tf_device.cluster_func` on TPU with replication and +// parallel_execute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + // CHECK-LABEL: func @replicated_parallel_tpu_cluster_func + func.func @replicated_parallel_tpu_cluster_func(%arg0: tensor) -> tensor { + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK: "tf._TPUCompileMlir" + // CHECK: "tf.TPUCompileSucceededAssert" + // CHECK: "tf_device.parallel_execute" + // CHECK-NOT:"tf._XlaCompileMlirPlaceholderProgramKey" + // CHECK: "tf.D"(%[[COMPILE_OUTPUT]]#1 + // CHECK: "tf.TPUExecute" + // CHECK-NOT:"tf._XlaCompileMlirPlaceholderProgramKey" + // CHECK: "tf.E"(%[[COMPILE_OUTPUT]]#1 + %3 = "tf_device.parallel_execute"() ({ + %program = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + "tf.D"(%program) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }, { + %4 = "tf_device.cluster_func"(%ri_0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor) -> tensor + tf_device.return %4 : tensor + }, { + %program = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + "tf.E"(%program) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }) : () -> (tensor) + tf_device.return %3 : tensor + } + %2 = "tf.C"(%1#1) : (tensor) -> tensor + func.return %2 : tensor + } + + func.func @tpu0_func(%arg0: tensor) -> tensor { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests case of `tf_device.cluster_func` on TPU with replication, model +// parallelism and parallel_execute. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2", "/job:worker/replica:0/task:0/device:TPU:3", "/job:worker/replica:0/task:0/device:TPU:4", "/job:worker/replica:0/task:0/device:TPU:5", "/job:worker/replica:0/task:0/device:TPU:6", "/job:worker/replica:0/task:0/device:TPU:7"]} { + // CHECK-LABEL: func @replicated_model_parallel_execute_tpu_cluster_func + func.func @replicated_model_parallel_execute_tpu_cluster_func(%arg0: tensor) -> tensor { + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[REPLICATE:[0-9]*]]:4 = tf_device.replicate + %1:4 = tf_device.replicate([%0, %arg0, %arg0, %arg0] as %ri_0: tensor) {n = 4 : i32} { + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:3 = "tf_device.launch" + // CHECK: "tf._TPUCompileMlir" + // CHECK: "tf.TPUCompileSucceededAssert" + // CHECK: "tf_device.parallel_execute" + // CHECK-NOT:"tf._XlaCompileMlirPlaceholderProgramKey" + // CHECK: "tf.D"(%[[COMPILE_OUTPUT]]#1 + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK: "tf.TPUExecute" + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK: "tf.TPUExecute" + // CHECK-NOT: "tf.TPUExecute" + %3 = "tf_device.parallel_execute"() ({ + %program = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + "tf.D"(%program) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }, { + %4 = "tf_device.cluster_func"(%ri_0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01*\02\08\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + tf_device.return %4 : tensor + }) : () -> (tensor) + tf_device.return %3 : tensor + } + %2 = "tf.C"(%1#1) : (tensor) -> tensor + func.return %2 : tensor + } + + func.func @tpu0_func(%arg0: tensor {mhlo.sharding = ""}) -> (tensor {mhlo.sharding = ""}) { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests case of `tf_device.cluster_func` on TPU with replication, model +// parallelism and parallel_execute with 2 parallel_execute children after the +// cluster. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2", "/job:worker/replica:0/task:0/device:TPU:3", "/job:worker/replica:0/task:0/device:TPU:4", "/job:worker/replica:0/task:0/device:TPU:5", "/job:worker/replica:0/task:0/device:TPU:6", "/job:worker/replica:0/task:0/device:TPU:7"]} { + // CHECK-LABEL: func @replicated_model_parallel_execute_2proc_tpu_cluster_func + func.func @replicated_model_parallel_execute_2proc_tpu_cluster_func(%arg0: tensor) -> tensor { + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[REPLICATE:[0-9]*]]:4 = tf_device.replicate + %1:4 = tf_device.replicate([%0, %arg0, %arg0, %arg0] as %ri_0: tensor) {n = 4 : i32} { + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:3 = "tf_device.launch" + // CHECK: "tf._TPUCompileMlir" + // CHECK: "tf.TPUCompileSucceededAssert" + // CHECK: "tf_device.parallel_execute" + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK: "tf.TPUExecute" + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK: "tf.TPUExecute" + // CHECK-NOT: "tf.TPUExecute" + // CHECK-NOT:"tf._XlaCompileMlirPlaceholderProgramKey" + // CHECK: "tf.D"(%[[COMPILE_OUTPUT]]#1 + // CHECK: "tf.E"(%[[COMPILE_OUTPUT]]#1 + %3 = "tf_device.parallel_execute"() ({ + %4 = "tf_device.cluster_func"(%ri_0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01*\02\08\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + tf_device.return %4 : tensor + }, { + %program = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + "tf.D"(%program) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }, { + %program = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + "tf.E"(%program) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }) : () -> (tensor) + tf_device.return %3 : tensor + } + %2 = "tf.C"(%1#1) : (tensor) -> tensor + func.return %2 : tensor + } + + func.func @tpu0_func(%arg0: tensor {mhlo.sharding = ""}) -> (tensor {mhlo.sharding = ""}) { + %0 = "tf.B"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } +} + +// ----- + +// Tests devices are set properly for non replicated model parallelism. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @non_replicated_parallel_execute + func.func @non_replicated_parallel_execute(%arg0: tensor<8xi32>) -> tensor<8xi32> { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: "tf.TPUExecute" + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:TPU:1"}> + // CHECK-NEXT: "tf.TPUExecute" + // CHECK-NEXT: tf_device.return + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> tensor<8xi32> + func.return %0 : tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + +// ----- + +// The following topology is used in subsequent test cases: +// Proto debug string: +// mesh_shape: 1 +// mesh_shape: 2 +// mesh_shape: 1 +// mesh_shape: 2 +// num_tasks: 2 +// num_tpu_devices_per_task: 2 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// Serialized string: +// "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01" +// ----- + +// Tests devices are set properly for replicated model parallelism. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @replicated_parallel_execute + func.func @replicated_parallel_execute(%arg0: tensor<8xi32>, %arg1: tensor<8xi32>) -> (tensor<8xi32>, tensor<8xi32>) { + // CHECK: tf_device.replicate + // CHECK-SAME: devices = {TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"], TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"], TPU_REPLICATED_HOST_0 = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:1/device:CPU:0"], TPU_REPLICATED_HOST_1 = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:1/device:CPU:0"]} + %0:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<8xi32>) {n = 2 : i32} { + // CHECK-NEXT: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: "tf.TPUExecute" + // CHECK-NEXT: tf_device.return + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: "tf.TPUExecute" + // CHECK-NEXT: tf_device.return + %1 = "tf_device.cluster_func"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> tensor<8xi32> + tf_device.return %1 : tensor<8xi32> + } + func.return %0#0, %0#1 : tensor<8xi32>, tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + + +// ----- + +// Tests that inputs are inputs with maximal and replicate sharding are set +// properly for replicated model parallelism. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_input_with_sharding_configurations + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<8xi32>, %[[ARG_1:[a-z0-9]*]]: tensor<8xi32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi1>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi1>, %[[ARG_4:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_5:[a-z0-9]*]]: tensor<*xi32>) + func.func @parallel_execute_with_input_with_sharding_configurations(%arg0: tensor<8xi32>, %arg1: tensor<8xi32>, %arg2: tensor<*xi1>, %arg3: tensor<*xi1>, %arg4: tensor<*xi32>, %arg5: tensor<*xi32>) -> (tensor<8xi32>, tensor<8xi32>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<8xi32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi1> + // CHECK-SAME: [%[[ARG_4]], %[[ARG_5]]] as %[[RI_2:[a-z0-9]*]]: tensor<*xi32> + %0:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<8xi32>, [%arg2, %arg3] as %ri2: tensor<*xi1>, [%arg4, %arg5] as %ri3: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch" + // CHECK: "tf._TPUCompileMlir" + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[RI_0]], %[[RI_1]], %[[RI_2]], %[[COMPILE]]#1) + // CHECK-NEXT: tf_device.return %[[EXECUTE_OUTPUT]] + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: "tf.TPUExecute"(%[[RI_1]], %[[RI_2]], %[[COMPILE]]#2) + %1 = "tf_device.cluster_func"(%ri, %ri2, %ri3) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "", ""], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>, tensor<*xi1>, tensor<*xi32>) -> tensor<8xi32> + tf_device.return %1 : tensor<8xi32> + } + func.return %0#0, %0#1 : tensor<8xi32>, tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>, %arg1: tensor<*xi1>, %arg2: tensor<*xi32>) -> tensor<8xi32> { + %1 = "tf.A"(%arg0, %arg1, %arg2) : (tensor<8xi32>, tensor<*xi1>, tensor<*xi32>) -> (tensor<8xi32>) + func.return %1 : tensor<8xi32> + } +} + +// ----- + +// Tests devices are set properly for replicated model parallelism with outputs +// to TPU computation placed on logical device 0. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_different_outputs + func.func @parallel_execute_with_different_outputs(%arg0: tensor<8xi32>, %arg1: tensor<8xi32>) -> (tensor<8xi32>, tensor<8xi32>) { + // CHECK: tf_device.replicate + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<8xi32>) {n = 2 : i32} { + // CHECK-NEXT: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_OUTPUT]] + // CHECK: "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: "tf.TPUExecute" + %1 = "tf_device.cluster_func"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> tensor<8xi32> + tf_device.return %1 : tensor<8xi32> + } + func.return %0#0, %0#1 : tensor<8xi32>, tensor<8xi32> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> tensor<8xi32> { + func.return %arg0 : tensor<8xi32> + } +} + +// ----- + +// Tests devices are set properly for replicated model parallelism with +// TPU computation with maximal and replicated outputs. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_replicated_output + func.func @parallel_execute_with_replicated_output(%arg0: tensor<8xi32>, %arg1: tensor<8xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor<8xi32>) {n = 2 : i32} { + // CHECK-NEXT: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir"() + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<8xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<8xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<8xi32>) -> (tensor<*xi32>, tensor<*xi1>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %1, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests inputs are correctly split and fed into TPU computation for tiled input +// sharding. + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_tiled_input + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @parallel_execute_with_tiled_input(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // + // CHECK: %[[CONST_SPLIT_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_DIM]], %[[RI_0]]) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_OUT]]#0, %[[COMPILE]]#1) + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_OUT]]#1, %[[RI_1]], %[[COMPILE]]#2) + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests that outputs are correctly merged and fed from TPU computation for +// tiled output sharding. + +// The following OpSharding is used for TPU computation outputs in below test: +// Proto debug string: +// output 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" +// +// output 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_tiled_output + func.func @parallel_execute_with_tiled_output(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + // + // CHECK: %[[CONST_CONCAT_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#0, %[[PARALLEL_EXECUTE_OUTPUT]]#2 + + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests that outputs are correctly merged and fed from TPU computation for +// tiled output sharding with MAXIMAL sharding for one of the output and OTHER +// for latter output. + +// The following OpSharding is used for TPU computation outputs in below test: +// Proto debug string: +// output 0 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" +// +// output 1 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_tiled_output + func.func @parallel_execute_with_tiled_output(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + // + // CHECK: %[[CONST_CONCAT_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#1, %[[PARALLEL_EXECUTE_OUTPUT]]#2 + + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) { + _xla_compile_device_type = "TPU", _replication_info = "cluster0", + func = @tpu0_func, num_cores_per_replica = 2, + step_marker_location = "", + topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", + device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], + input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\00"], + output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\03\1A\02\01\02\22\02\00\01"], + use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 4 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\01\04\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\01\04\22\04\00\01\02\03" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" +// +// ----- + + +// Tests tile sharding of inputs with number of splits that does not evenly divide +// the input results in an error, when shapes are not fully known. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + func.func @uneven_input_sharding_disallowed(%arg0: tensor, %arg1: tensor, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // expected-error@+1 {{incorrect input sharding configuration received. 1-th dimension of the input must be evenly divisible by 4}} + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\01\04\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\01\04\22\04\00\01\02\03", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// The following OpSharding is used for TPU computation outputs in below test: +// Proto debug string: +// output 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 4 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\01\04\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\01\04\22\04\00\01\02\03" +// +// output 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" +// +// ----- + +// Tests tile sharding of outputs with number of splits that exeed number +// of logical devices is not allowed. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + func.func @uneven_output_sharding_disallowed(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // expected-error@+1 {{incorrect sharding format for outputs. Number of tiled outputs(4) must match the number of logical devices(2)}} + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["", ""], output_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\01\04\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\01\04\22\04\00\01\02\03", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi1>, tensor<*xi32>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %3, %4 : tensor<*xi1>, tensor<*xi32> + } +} + +// ----- + +// The following topology is used in subsequent test cases: +// Proto debug string: +// mesh_shape: 2 +// mesh_shape: 1 +// mesh_shape: 2 +// num_tasks: 2 +// num_tpu_devices_per_task: 2 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_shape { +// element_type: F32 +// dimensions: 2 +// dimensions: 2 +// layout { +// minor_to_major: 1 +// minor_to_major: 0 +// format: DENSE +// } +// is_dynamic_dimension: false +// is_dynamic_dimension: false +// } +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// Tests inputs to TPUComputation that are tiled in multiple dimensions. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_multi_dimension_tiled_input + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @parallel_execute_with_multi_dimension_tiled_input(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[CONST_SPLIT_0_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_0_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_0_DIM]], %[[RI_0]]) + // CHECK: %[[CONST_SPLIT_1_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_1_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_1_DIM]], %[[SPLIT_0_OUT]]#0) + // CHECK: %[[CONST_SPLIT_2_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_2_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_2_DIM]], %[[SPLIT_0_OUT]]#1) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#0, %[[COMPILE]]#1) + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#1, %[[RI_1]], %[[COMPILE]]#2) + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#0, %[[COMPILE]]#3) + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#1, %[[COMPILE]]#4) + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + + +// ----- + +// The following topology is used in subsequent test cases: +// Proto debug string: +// mesh_shape: 2 +// mesh_shape: 1 +// mesh_shape: 2 +// num_tasks: 2 +// num_tpu_devices_per_task: 2 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_shape { +// element_type: F32 +// dimensions: 2 +// dimensions: 2 +// layout { +// minor_to_major: 1 +// minor_to_major: 0 +// format: DENSE +// } +// is_dynamic_dimension: false +// is_dynamic_dimension: false +// } +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// Tests inputs to TPUComputation that are tiled in multiple dimensions. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_multi_dimension_tiled_input + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @parallel_execute_with_multi_dimension_tiled_input(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[CONST_SPLIT_0_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_0_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_0_DIM]], %[[RI_0]]) + // CHECK: %[[CONST_SPLIT_1_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_1_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_1_DIM]], %[[SPLIT_0_OUT]]#0) + // CHECK: %[[CONST_SPLIT_2_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_2_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_2_DIM]], %[[SPLIT_0_OUT]]#1) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#0, %[[COMPILE]]#1) + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#1, %[[RI_1]], %[[COMPILE]]#2) + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#0, %[[COMPILE]]#3) + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#1, %[[COMPILE]]#4) + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests inputs to TPUComputation that are tiled in multiple dimensions with +// replicate_on_last_tile_dim set. + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// replicate_on_last_tile_dim: true +// Serialized string: +// "\08\03\1A\03\02\01\02\22\04\00\01\02\030\01" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @multi_dimension_tiled_input_replicate_last_dim + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @multi_dimension_tiled_input_replicate_last_dim(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[CONST_SPLIT_0_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_0_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_0_DIM]], %[[RI_0]]) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#0, %[[COMPILE]]#1) + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#0, %[[RI_1]], %[[COMPILE]]#2) + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#1, %[[COMPILE]]#3) + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#1, %[[COMPILE]]#4) + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\03\1A\03\02\01\02\22\04\00\01\02\030\01", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests that tiled output with multiple dimension sharding works properly. + +// The following OpSharding is used for TPU computation outputs in below test: +// output 0 +// Proto debug string: +// type: OTHER +// tile_shape { +// element_type: F32 +// dimensions: 2 +// dimensions: 2 +// layout { +// minor_to_major: 1 +// minor_to_major: 0 +// format: DENSE +// } +// is_dynamic_dimension: false +// is_dynamic_dimension: false +// } +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03" +// +// output 1 +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @multiple_dimension_output_sharding + func.func @multiple_dimension_output_sharding(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + // CHECK: %[[CONST_CONCAT_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#0, %[[PARALLEL_EXECUTE_OUTPUT]]#2 + // CHECK: %[[CONST_CONCAT2_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT2_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT2_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#3, %[[PARALLEL_EXECUTE_OUTPUT]]#4 + // CHECK: %[[CONST_CONCAT3_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT3_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT3_DIM]], %[[CONCAT_OUTPUT]], %[[CONCAT2_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\00\01\02\03", "\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests that tiled output with multiple dimension sharding works properly with +// replicate_on_last_tile_dim set. + +// The following OpSharding is used for TPU computation outputs in below test: +// output 0 +// Proto debug string: +// type: OTHER +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// replicate_on_last_tile_dim: true +// Serialized string: +// "\08\03\1A\03\02\01\02\22\04\00\01\02\030\01" +// +// output 1 +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\00" + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @multi_dimension_tiled_output_replicate_last_dim + func.func @multi_dimension_tiled_output_replicate_last_dim(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + // CHECK: %[[CONST_CONCAT_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#0, %[[PARALLEL_EXECUTE_OUTPUT]]#3 + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\03\1A\03\02\01\02\22\04\00\01\02\030\01", "\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests inputs device assignment order is well preserved for tiled input sharding. + +// The following OpSharding is used for TPU computation inputs in below test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_shape { +// element_type: F32 +// dimensions: 2 +// dimensions: 2 +// layout { +// minor_to_major: 1 +// minor_to_major: 0 +// format: DENSE +// } +// is_dynamic_dimension: false +// is_dynamic_dimension: false +// } +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 3 +// tile_assignment_devices: 2 +// tile_assignment_devices: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\03\02\01\00" +// +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" +// +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @tiled_input_sharding_with_device_assignment_order + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @tiled_input_sharding_with_device_assignment_order(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[CONST_SPLIT_0_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_0_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_0_DIM]], %[[RI_0]]) + // CHECK: %[[CONST_SPLIT_1_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_1_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_1_DIM]], %[[SPLIT_0_OUT]]#0) + // CHECK: %[[CONST_SPLIT_2_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_2_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_2_DIM]], %[[SPLIT_0_OUT]]#1) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#1, %[[COMPILE]]#1) + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_2_OUT]]#0, %[[RI_1]], %[[COMPILE]]#2) + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#1, %[[COMPILE]]#3) + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_1_OUT]]#0, %[[COMPILE]]#4) + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\03\02\01\00", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests device assignment is well preserved for tile sharded outputs. + +// The following OpSharding is used for TPU computation outputs in below test: +// output 0 +// Proto debug string: +// type: OTHER +// tile_shape { +// element_type: F32 +// dimensions: 2 +// dimensions: 2 +// layout { +// minor_to_major: 1 +// minor_to_major: 0 +// format: DENSE +// } +// is_dynamic_dimension: false +// is_dynamic_dimension: false +// } +// tile_assignment_dimensions: 2 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 3 +// tile_assignment_devices: 2 +// tile_assignment_devices: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\03\02\01\00" +// +// output 1 +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @device_order_preserved_for_tiled_output + func.func @device_order_preserved_for_tiled_output(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"( + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + // CHECK: %[[CONST_CONCAT_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#4, %[[PARALLEL_EXECUTE_OUTPUT]]#3 + // CHECK: %[[CONST_CONCAT2_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT2_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT2_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#2, %[[PARALLEL_EXECUTE_OUTPUT]]#0 + // CHECK: %[[CONST_CONCAT3_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT3_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT3_DIM]], %[[CONCAT_OUTPUT]], %[[CONCAT2_OUTPUT]] + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\02\02\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\02\02\22\04\03\02\01\00", "\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_compilation_and_replication_attributes() { + // expected-error@+1 {{'tf_device.cluster_func' op is expected to have either both or none of '_replication_info' and '_xla_compile_device_type' attributes}} + "tf_device.cluster_func"() {_replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @empty_replication_attribute() { + // expected-error@+1 {{'tf_device.cluster_func' op has an empty '_replication_info' attribute}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @empty_compilation_attribute() { + // expected-error@+1 {{'tf_device.cluster_func' op has invalid '_xla_compile_device_type' value 'XPU'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "XPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @invalid_compilation_attribute() { + // expected-error@+1 {{'tf_device.cluster_func' op has invalid '_xla_compile_device_type' value 'XPU'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "XPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} + +// ----- + +// Test `tf.TPUPartitionedInputV2` has outputs not in `tf_device.cluster_func` + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + // expected-error@+1 {{Output of TPUPartitionedInputV2 must be in tpu computation.}} + %partitioned_input = "tf.TPUPartitionedInputV2"(%arg0, %arg1) {N = 2 : i64, partition_dims = []} : (tensor>>, tensor>>) -> tensor>> + + %read = "tf.ReadVariableOp"(%partitioned_input) : (tensor>>) -> tensor + + %computation = "tf_device.cluster_func"(%read) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%computation) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} + +// ----- + +// Test `tf.TPUPartitionedOutputV2` has inputs not in `tf_device.cluster_func` + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %partitioned_input = "tf.TPUPartitionedInputV2"(%read0, %read1) {N = 2 : i64, partition_dims = []} : (tensor, tensor) -> tensor + %computation = "tf_device.cluster_func"(%partitioned_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @computation, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = [""], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + %add_result = "tf.Add"(%computation, %computation) : (tensor, tensor) -> tensor + // expected-error@+1 {{Input of TPUPartitionedOutputV2 must be in tpu computation.}} + %partitioned_output:2 = "tf.TPUPartitionedOutputV2"(%add_result) {N = 2 : i64, partition_dims = []} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + func.return + } + func.func @computation(%arg0: tensor) -> tensor { + func.return %arg0: tensor + } +} + +// ----- + +// The following xla.OpSharding is used: +// Proto debug string: +// type : TUPLE +// tuple_shardings: { +// type: OTHER +// tile_assignment_dimensions: [ 2, 1 ] +// tile_assignment_devices : [ 0, 1 ] +// } +// tuple_shardings: { +// type: OTHER +// tile_assignment_dimensions: [ 2, 1 ] +// tile_assignment_devices : [ 0, 1 ] +// } +// Serialized string: +// "\08\02*\0A\08\03\1A\02\02\01\22\02\00\01*\0A\08\03\1A\02\02\01\22\02\00\01" + +// Test that an attempt to map an invalid cluster output index to core program +// index is caught. The output has sharding type TUPLE, which causes the +// cluster output index to be invalid for core 0. + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster_to_core_index_check() -> tensor<128xf32> { + // expected-error@+1 {{Attempted to map cluster_func output index 0 to program assigned to core 0}} + %0 = "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], input_sharding_configuration = [], output_sharding_configuration = ["\08\02*\0A\08\03\1A\02\02\01\22\02\00\01*\0A\08\03\1A\02\02\01\22\02\00\01"], use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> tensor<128xf32> + func.return %0 : tensor<128xf32> + } + func.func @_func() -> tensor<128xf32> { + %0 = "tf.Const"() {value = dense<0.0> : tensor<128xf32>} : () -> tensor<128xf32> + func.return %0 : tensor<128xf32> + } +} + +// ----- +// CHECK-LABEL: func @return_from_host_and_tpu +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1199 : i32}} { + func.func @return_from_host_and_tpu() -> (tensor, tensor) attributes {tf._construction_context = "kEagerRuntime", tf.signature.is_stateful} { + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:2 = "tf_device.parallel_execute" + // CHECK: %[[LAUNCH_0_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK: %[[B_OUTPUT:[0-9]*]] = "tf.B" + // CHECK: tf_device.return %[[B_OUTPUT:[0-9]*]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:TPU:0"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: return %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]#1, %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]#0 + %0:2 = "tf_device.parallel_execute"() ({ + %1 = "tf_device.launch"() ({ + %2 = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + %3 = "tf._XlaRecvAtHost"(%2) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_0_args"} : (tensor<3x!tf_type.string>) -> tensor + %4 = "tf.B"(%3) : (tensor) -> tensor + tf_device.return %4 : tensor + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor + tf_device.return %1 : tensor + }, { + %1 = "tf_device.cluster_func"() {step_marker_location = "STEP_MARK_AT_ENTRY", _replication_info = "cluster", _xla_compile_device_type = "TPU", allow_soft_placement = true, computation_shape = [], device = "", device_assignment = [], func = @_func, input_sharding_configuration = [], num_cores_per_replica = 1 : i64, output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> tensor + tf_device.return %1 : tensor + }) : () -> (tensor, tensor) + return %0#1, %0#0 : tensor, tensor + } + func.func private @_func() -> (tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %0 = "tf.A"() : () -> tensor + "tf._XlaHostComputeMlir"(%0) {host_mlir_module = "", recv_key = "host_compute_channel_0_retvals", send_key = "host_compute_channel_0_args"} : (tensor) -> () + %1 = "tf.C"() : () -> tensor + return %1 : tensor + } +} + +// ----- + +// The following xla.OpSharding is used: +// Proto debug string: +// type : OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// last_tile_dims: REPLICATED +// Serialized string: +// "\08\03\1A\06\01\01\01\01\01\01\22\01\00B\01\00" + +// Test that an input sharding with last_tile_dims REPLICATED won't generate SplitOp. +//CHECK-NOT: tf.Split +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"]} { + func.func @cluster_to_single_core(%arg0: tensor<128xf32>) -> tensor<128xf32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster1", func = @_func, num_replica = 1, num_cores_per_replica = 1, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", device_assignment = [], input_sharding_configuration = ["\08\03\1A\06\01\01\01\01\01\01\22\01\00B\01\00"], output_sharding_configuration = [""], use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor<128xf32>) -> tensor<128xf32> + func.return %0 : tensor<128xf32> + } + func.func @_func(%arg0: tensor<128xf32>) -> tensor<128xf32> { + func.return %arg0 : tensor<128xf32> + } +} + +// ----- + +// CHECK-LABEL: func @annotate_dynamic_shape_tensor +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:COMPOSITE:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1437 : i32}} { + func.func @annotate_dynamic_shape_tensor(%arg0: tensor<512xi64> {tf._user_specified_name = "190", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) -> (tensor<512xi32>) { + %0 = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster_test_fn", device = ""} : () -> tensor + %cst = "tf.Const"() {value = dense<512> : tensor} : () -> tensor + %2:4 = "tf_device.launch"() ({ + %4 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<512xi64>) -> tensor<512xi32> + %5 = "tf.TPUCopyWithDynamicShape"(%4, %cst) {operandSegmentSizes = array} : (tensor<512xi32>, tensor) -> tensor<512xi32> + tf_device.return %5 : tensor<512xi32> + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> (tensor<512xi32>, tensor<1024xi32>, tensor<1024xi32>, tensor<1024xf32>) + // CHECK: %[[COMPILE_OUTPUT:[0-9]*]]:4 = "tf_device.launch" + // CHECK: "tf._TPUCompileMlir"() + // CHECK: is_bounded_dynamic_dim: true + %3 = "tf_device.cluster_func"(%2#0) {_dynamic_arg_index = [0 : i32], _has_manual_control_dependencies = true, _replication_info = "cluster_test_fn", _xla_compile_device_type = "TPU", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], func = @_func, host_compute_core = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], num_cores_per_replica = 1 : i64, output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor<512xi32>) -> tensor<512xi32> + return %3: tensor<512xi32> + } +func.func private @_func(%arg0: tensor> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor<512xi32>) { + %0 = "tf.A"(%arg0) {} : (tensor>) -> tensor<512xi32> + return %0 : tensor<512xi32> + } +} + +// ----- + +// The following xla.OpSharding is used: +// Proto debug string: +// type : OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 4 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// tile_assignment_devices: 2 +// tile_assignment_devices: 3 +// last_tile_dims: REPLICATED +// Serialized string: +// "\08\03\1A\03\01\01\04\22\04\00\01\02\03B\01\00" + +// Test that SplitOp is not generated when an input sharding has +// last_tile_dims REPLICATED and more tile_assignment_dimensions +// than tensor dimenstions, even when the SPMD sharding is enabled and +// num_cores_per_replica is more than 1. +// Test that ConcatV2 Op is not generated when a output sharding has +// last_tile_dims REPLICATED and more tile_assignment_dimensions +// than tensor dimenstions, even when the SPMD sharding is enabled and +// num_cores_per_replica is more than 1. +//CHECK-NOT: tf.Split +// CHECK-NOT: tf.ConcatV2 +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2", "/job:worker/replica:0/task:0/device:TPU:3"]} { + func.func @cluster_to_single_core(%arg0: tensor<4x128xf32>) -> tensor<4x128xf32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster1", func = @_func, num_replica = 1, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "\0A\04\02\02\01\01\10\01\18\04\22\10\00\00\00\00\01\00\00\00\00\01\00\00\01\01\00\00", device_assignment = [0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\03\01\01\04\22\04\00\01\02\03B\01\00"], output_sharding_configuration = ["\08\03\1A\03\01\01\04\22\04\00\01\02\03B\01\00"], use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<4x128xf32>) -> tensor<4x128xf32> + func.return %0 : tensor<4x128xf32> + } + func.func @_func(%arg0: tensor<4x128xf32>) -> tensor<4x128xf32> { + func.return %arg0 : tensor<4x128xf32> + } +} + +// ----- + +// Tests _ici_weight_distribution_mlir_bridge_marker attribute can be +// produced in created Split op +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @propagate_ici_weight_attr_to_split_op + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x10xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<*xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<*xi32>) + func.func @propagate_ici_weight_attr_to_split_op(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<*xi32> + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:5 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // CHECK: %[[CONST_SPLIT_0_DIM:.*]] = "tf.Const"() + // CHECK: %[[SPLIT_0_OUT:[a-z0-9]+]]:2 = "tf.Split" + // CHECK: _ici_weight_distribution_mlir_bridge_marker = true + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:5 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#0, %[[COMPILE]]#1) + // CHECK: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#0, %[[RI_1]], %[[COMPILE]]#2) + // CHECK: tf_device.return %[[EXECUTE_1_OUTPUT]] + // CHECK: %[[LAUNCH_2_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_2_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#1, %[[COMPILE]]#3) + // CHECK: tf_device.return %[[EXECUTE_2_OUTPUT]] + // CHECK: %[[LAUNCH_3_OUTPUT:[0-9]*]] = "tf_device.launch" + // CHECK-NEXT: %[[EXECUTE_3_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_0_OUT]]#1, %[[COMPILE]]#4) + // CHECK: tf_device.return %[[EXECUTE_3_OUTPUT]] + %1 = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %identity = "tf.Identity"(%ri_1) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor<128x10xf32>) -> tensor<128x10xf32> + tf_device.return %identity : tensor<128x10xf32> + }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor<128x10xf32> + %2, %3 = "tf_device.cluster_func"(%1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 4, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1], input_sharding_configuration = ["\08\03\1A\03\02\01\02\22\04\00\01\02\030\01", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %2, %3 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} + +// ----- + +// Tests that outputs are correctly merged and fed from TPU computation for +// tiled output sharding with padding for concat ops. + +// The following OpSharding is used for TPU computation outputs in below test: +// Proto debug string: +// output 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" +// +// output 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_tiled_output + func.func @parallel_execute_with_tiled_output(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<128x10xi32>, %arg3: tensor<128x10xi32>) -> (tensor<128x5xi32>, tensor<10x5xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x10xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<128x10xi32> + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<128x10xi32>) {n = 2 : i32} { + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute" + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + // + // CHECK: %[[CONST_CONCAT3_DIM:.*]] = "tf.Const"() + // CHECK: %[[CONCAT3_OUTPUT:[0-9]*]] = "tf.Concat"(%[[CONST_CONCAT3_DIM]], %[[PARALLEL_EXECUTE_OUTPUT]]#0, %[[PARALLEL_EXECUTE_OUTPUT]]#2) + // CHECK: %[[CONST_SLICE_BEGIN:.*]] = "tf.Const"() + // dense<0> + // tensor<2xi64>}> : () -> tensor<2xi64> + // CHECK: %[[CONST_SLICE_SIZE:.*]] = "tf.Const"() + // dense<[128, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + // CHECK: "tf.Slice"(%[[CONCAT3_OUTPUT]], %[[CONST_SLICE_BEGIN]], %[[CONST_SLICE_SIZE]]) + // : (tensor<128x6xi32>, tensor<2xi64>, tensor<2xi64>) -> tensor<128x5xi32> + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\00"], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<128x10xi32>) -> (tensor<128x5xi32>, tensor<10x5xi1>) + tf_device.return %1, %2 : tensor<128x5xi32>, tensor<10x5xi1> + } + func.return %0#0, %1#0 : tensor<128x5xi32>, tensor<10x5xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<128x10xi32>, tensor<128x10xi32>) -> (tensor<128x10xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<10x5xi1>) -> tensor<10x5xi1> + func.return %4, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } +} + +// ----- + +// Tests inputs are correctly split and fed into TPU computation for tiled input +// sharding with padding. + +// The following OpSharding is used for TPU computation inputs in the below +// test: +// Proto debug string: +// input 0 +// type: OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\03\1A\02\01\02\22\02\00\01" +// +// input 1 +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + // CHECK-LABEL: func @parallel_execute_with_tiled_input + // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: tensor<128x9xf32>, %[[ARG_1:[a-z0-9]*]]: tensor<128x9xf32>, %[[ARG_2:[a-z0-9]*]]: tensor<128x10xi32>, %[[ARG_3:[a-z0-9]*]]: tensor<128x10xi32>) + func.func @parallel_execute_with_tiled_input(%arg0: tensor<128x9xf32>, %arg1: tensor<128x9xf32>, %arg2: tensor<128x10xi32>, %arg3: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + // CHECK: tf_device.replicate + // CHECK-SAME: [%[[ARG_0]], %[[ARG_1]]] as %[[RI_0:[a-z0-9]*]]: tensor<128x9xf32> + // CHECK-SAME: [%[[ARG_2]], %[[ARG_3]]] as %[[RI_1:[a-z0-9]*]]: tensor<128x10xi32> + // CHECK-SAME: devices = + // CHECK-SAME: TPU_REPLICATED_CORE_0 = ["/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1"] + // CHECK-SAME: TPU_REPLICATED_CORE_1 = ["/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU:0"] + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x9xf32>, [%arg2, %arg3] as %ri_2: tensor<128x10xi32>) {n = 2 : i32} { + // CHECK: %[[DEVICE_LAUNCH_OUT:[a-z0-9]+]] = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> + // CHECK: %[[COMPILE:[a-z0-9]+]]:3 = "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf._TPUCompileMlir" + // CHECK: "tf_device.launch"() <{device = "/job:localhost/replica:0/task:0/device:CPU:0"}> + // CHECK-NEXT: "tf.TPUCompileSucceededAssert"(%[[COMPILE]]#0) + // + // CHECK: %[[PAD_SHAPE:[a-z0-9]+]] = "tf.Const"() + // CHECK: [0, 0], [0, 1] + // CHECK: : tensor<2x2xi64>}> : () -> tensor<2x2xi64> + // CHECK: %[[PAD_OUT:[a-z0-9]+]] = "tf.Pad"(%[[DEVICE_LAUNCH_OUT]], %[[PAD_SHAPE]]) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor<128x9xf32>, tensor<2x2xi64>) -> tensor<128x10xf32> + // CHECK: %[[CONST_SPLIT_DIM:.*]] = "tf.Const"() <{value = dense<1> : tensor}> {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor + // CHECK: %[[SPLIT_OUT:[a-z0-9]+]]:2 = "tf.Split"(%[[CONST_SPLIT_DIM]], %[[PAD_OUT]]) {_ici_weight_distribution_mlir_bridge_marker = true, num_split = 2 : i32} : (tensor, tensor<128x10xf32>) -> (tensor<128x5xf32>, tensor<128x5xf32>) + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]]:3 = "tf_device.parallel_execute" + // CHECK-NEXT: %[[LAUNCH_0_OUTPUT:[0-9]*]]:2 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> + // + // CHECK-NEXT: %[[EXECUTE_0_OUTPUT:[0-9]*]]:2 = "tf.TPUExecute"(%[[SPLIT_OUT]]#0, %[[COMPILE]]#1) + // CHECK-NEXT: tf_device.return %[[EXECUTE_0_OUTPUT]] + // CHECK: %[[LAUNCH_1_OUTPUT:[0-9]*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> + // CHECK-NEXT: %[[EXECUTE_1_OUTPUT:[0-9]*]] = "tf.TPUExecute"(%[[SPLIT_OUT]]#1, %[[RI_1]], %[[COMPILE]]#2) + // CHECK-NEXT: tf_device.return %[[EXECUTE_1_OUTPUT]] + %1 = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %identity = "tf.Identity"(%ri_1) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor<128x9xf32>) -> tensor<128x9xf32> + tf_device.return %identity : tensor<128x9xf32> + }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor<128x9xf32> + %2, %3 = "tf_device.cluster_func"(%1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02\22\02\00\01", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x9xf32>, tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + tf_device.return %2, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.return %0#0, %1#0 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.func @tpu0_func(%arg0: tensor<128x9xf32>, %arg1: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x9xf32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<128x10xi32>, tensor<128x10xi32>) -> (tensor<128x10xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<10x5xi1>) -> tensor<10x5xi1> + func.return %4, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } +} + +// ----- + +// CHECK: "tf.Split" +// : (tensor<128x1024xf32>) -> (tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>) +module attributes {tf.devices = {"/job:tpu_host_worker/replica:0/task:0/device:CPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:1", "/job:tpu_host_worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:1/device:CPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:1", "/job:tpu_host_worker/replica:0/task:1/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:2/device:CPU:0", "/job:tpu_host_worker/replica:0/task:2/device:TPU:0", "/job:tpu_host_worker/replica:0/task:2/device:TPU:1", "/job:tpu_host_worker/replica:0/task:2/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:3/device:CPU:0", "/job:tpu_host_worker/replica:0/task:3/device:TPU:0", "/job:tpu_host_worker/replica:0/task:3/device:TPU:1", "/job:tpu_host_worker/replica:0/task:3/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1857 : i32}} { + func.func @main(%arg0: tensor {tf._user_specified_name = "steps", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg1: tensor<*x!tf_type.resource>> {tf._user_specified_name = "899", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg2: tensor<*x!tf_type.resource>> {tf._user_specified_name = "901", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg3: tensor<*x!tf_type.resource>> {tf._user_specified_name = "903", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg4: tensor<*x!tf_type.resource>> {tf._user_specified_name = "905", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg5: tensor<*x!tf_type.resource>> {tf._user_specified_name = "907", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg6: tensor<*x!tf_type.resource>> {tf._user_specified_name = "909", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg7: tensor<*x!tf_type.resource>> {tf._user_specified_name = "911", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg8: tensor<*x!tf_type.resource>> {tf._user_specified_name = "913", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg9: tensor<*x!tf_type.resource>> {tf._user_specified_name = "915", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg10: tensor<*x!tf_type.resource>> {tf._user_specified_name = "917", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg11: tensor<*x!tf_type.resource>> {tf._user_specified_name = "919", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg12: tensor<*x!tf_type.resource>> {tf._user_specified_name = "921", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg13: tensor<*x!tf_type.resource>> {tf._user_specified_name = "923", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg14: tensor<*x!tf_type.resource>> {tf._user_specified_name = "925", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg15: tensor<*x!tf_type.resource>> {tf._user_specified_name = "927", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg16: tensor<*x!tf_type.resource>> {tf._user_specified_name = "929", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg17: tensor<*x!tf_type.resource>> {tf._user_specified_name = "931", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg18: tensor<*x!tf_type.resource>> {tf._user_specified_name = "933", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg19: tensor<*x!tf_type.resource>> {tf._user_specified_name = "935", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg20: tensor<*x!tf_type.resource>> {tf._user_specified_name = "937", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg21: tensor<*x!tf_type.resource>> {tf._user_specified_name = "939", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}) -> tensor attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "steps,unknown,unknown_0,unknown_1,unknown_2,unknown_3,unknown_4,unknown_5,unknown_6,unknown_7,unknown_8,unknown_9,unknown_10,unknown_11,unknown_12,unknown_13,unknown_14,unknown_15,unknown_16,unknown_17,unknown_18,unknown_19", outputs = "statefulpartitionedcall_RetVal"}} { + %0 = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster__train_helper", device = ""} : () -> tensor + %1 = "tf.ReadVariableOp"(%arg3) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %2 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor<1024xf32> + %3:2 = tf_device.replicate {n = 2 : i32} { + %6 = "tf_device.cluster_func"(%1, %2) <{func = @_func}> {_dynamic_arg_index = [], _has_manual_control_dependencies = true, _replication_info = "cluster__train_helper", _xla_compile_device_type = "TPU", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0], host_compute_core = [], input_sharding_configuration = ["\08\03\1A\01\04\22\04\00\01\02\03", ""], num_cores_per_replica = 4 : i64, output_sharding_configuration = [""], padding_map = [], step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\02\01\10\04\18\02\22 \00\00\00\00\00\01\00\00\01\00\00\00\01\01\00\00\00\00\01\00\00\01\01\00\01\00\01\00\01\01\01\00*\02\08\01", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<128x1024xf32>, tensor<1024xf32>) -> tensor<*xf32> + tf_device.return %6 : tensor<*xf32> + } + %4 = "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor + %5 = "tf.Identity"(%4) {device = ""} : (tensor) -> tensor + return %5 : tensor + } + func.func private @_func(%arg0: tensor<128x1024xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\03\1A\01\04\22\04\00\01\02\03"}, %arg1: tensor<1024xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = ""}) -> (tensor<*xf32> {mhlo.sharding = ""}) { + %0 = "tf.XlaSharding"(%arg0) <{_XlaSharding = "\08\03\1A\01\04\22\04\00\01\02\03", sharding = "\08\03\1A\01\04\22\04\00\01\02\03"}> {unspecified_dims = []} : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + %1 = "tf.MatMul"(%0, %arg1) : (tensor<128x1024xf32>, tensor<1024xf32>) -> tensor<*xf32> + return %1 : tensor<*xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_sharding_identification.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_sharding_identification.mlir new file mode 100644 index 0000000000000000000000000000000000000000..39449d9e84634f1fa8710154e6dd312fc7e46e0c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_sharding_identification.mlir @@ -0,0 +1,1055 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-sharding-identification | FileCheck %s + +// Tests invalid cluster_func op with no `num_cores_per_replica` attribute. +func.func @no_num_cores_per_replica_attr() { + // expected-error @+1 {{'tf_device.cluster_func' op requires attribute 'num_cores_per_replica'}} + "tf_device.cluster_func"() {func = @empty_func, step_marker_location = ""} : () -> () + func.return +} + +func.func @empty_func() { + func.return +} + +// ----- + +// Tests empty cluster func. Empty input/output sharding configuration +// attributes must be added. +// CHECK-LABEL: func @check_sharding_attrs_exists_for_empty_cluster_func +func.func @check_sharding_attrs_exists_for_empty_cluster_func() { + "tf_device.cluster_func"() {func = @empty_func, step_marker_location = "", num_cores_per_replica = 1 : i64} : () -> () + // CHECK: input_sharding_configuration = [] + // CHECK: output_sharding_configuration = [] + func.return +} + +// CHECK-LABEL: func @empty_func() { +func.func @empty_func() { + func.return +} + +// ----- + +// Tests with a block argument inputs/outputs with no xla sharding op attached +// gets default maximal(0) sharding configuration. +// CHECK-LABEL: func @check_default_sharding_for_block_arg_inputs_outputs +func.func @check_default_sharding_for_block_arg_inputs_outputs(%arg0: tensor<*xi32>) { + "tf_device.cluster_func"(%arg0) {func = @func_without_sharding, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\08\01\1A\01\01\22\01\00"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\08\01\1A\01\01\22\01\00"] + func.return +} + +// CHECK-LABEL: func @func_without_sharding +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +func.func @func_without_sharding(%arg0: tensor<*xi32>) -> tensor<*xi32> { + func.return %arg0 : tensor<*xi32> +} + +// ----- + +// Tests with a inputs/outputs with no xla sharding op attached gets +// default maximal(0) sharding configuration. +// CHECK-LABEL: func @check_default_sharding_for_inputs_outputs +func.func @check_default_sharding_for_inputs_outputs(%arg0: tensor<*xi32>) { + "tf_device.cluster_func"(%arg0) {func = @func_without_sharding, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\08\01\1A\01\01\22\01\00"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\08\01\1A\01\01\22\01\00"] + func.return +} + +// CHECK-LABEL: func @func_without_sharding +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +func.func @func_without_sharding(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.A"(%arg0) : (tensor<*xi32>) -> tensor<*xi32> + func.return %0 : tensor<*xi32> +} + +// ----- + +// Tests with a input arg connected to XlaSharding op. +// CHECK-LABEL: func @check_sharding_for_input_correctly_identified +func.func @check_sharding_for_input_correctly_identified(%arg0: tensor<*xi32>) { + "tf_device.cluster_func"(%arg0) {func = @inputs_with_sharding_func, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\08\01\1A\01\01\22\01\00"] + func.return +} + +// CHECK-LABEL: func @inputs_with_sharding_func +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +func.func @inputs_with_sharding_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %1 : tensor<*xi32> +} + +// ----- + +// Tests with sharding is correctly parsed for multiple inputs/outputs. +// CHECK-LABEL: func @check_sharding_for_multiple_inputs_outputs +func.func @check_sharding_for_multiple_inputs_outputs(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_sharding, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03", "\04\05\06"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0A\0B\0C", "\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}, %{{[a-z0-9]+}}: tensor<*xi1> {mhlo.sharding = "\04\05\06"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\0A\0B\0C"}, tensor<*xi1> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.XlaSharding"(%arg1) { _XlaSharding = "\04\05\06", sharding = "\04\05\06" } : (tensor<*xi1>) -> tensor<*xi1> + %2, %3 = "tf.A"(%0, %1) : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.XlaSharding"(%2) { _XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C" } : (tensor<*xi32>) -> tensor<*xi32> + %5 = "tf.XlaSharding"(%3) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %5 : tensor<*xi32> , tensor<*xi1> +} + +// ----- + +func.func @check_sharding_for_input_inconsistent(%arg0: tensor<4x8xi32>) { + // expected-error @+1 {{'tf_device.cluster_func' op _XlaShardingV2 is not equivalent to _XlaSharding: {devices=[2,1]<=[2]} vs {devices=[2,1]1,0}}} + "tf_device.cluster_func"(%arg0) {func = @func_with_sharding, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<4x8xi32>) -> (tensor<4x8xi32>) + func.return +} + +func.func @func_with_sharding(%arg0: tensor<4x8xi32>) -> (tensor<4x8xi32>) { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "{devices=[2,1]1,0}", sharding = "{devices=[2,1]1,0}", _XlaShardingV2 = "{devices=[2,1]<=[2]}" } : (tensor<4x8xi32>) -> tensor<4x8xi32> + %1 = "tf.A"(%0) : (tensor<4x8xi32>) -> (tensor<4x8xi32>) + func.return %1 : tensor<4x8xi32> +} + +// ----- + +// CHECK-LABEL: func @check_sharding_for_input_consistent +func.func @check_sharding_for_input_consistent(%arg0: tensor<4x8xi32>) { + // CHECK: %[[A0:.*]] = "tf_device.cluster_func"(%arg0) + // CHECK-SAME: input_sharding_configuration = ["{devices=[2,1]<=[2]}" + "tf_device.cluster_func"(%arg0) {func = @func_with_sharding, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<4x8xi32>) -> (tensor<4x8xi32>) + func.return +} + +func.func @func_with_sharding(%arg0: tensor<4x8xi32>) -> (tensor<4x8xi32>) { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "{devices=[2,1]0,1}", sharding = "{devices=[2,1]0,1}", _XlaShardingV2 = "{devices=[2,1]<=[2]}" } : (tensor<4x8xi32>) -> tensor<4x8xi32> + %1 = "tf.A"(%0) : (tensor<4x8xi32>) -> (tensor<4x8xi32>) + func.return %1 : tensor<4x8xi32> +} + +// ----- + +// Tests with input sharding following an identity op. +// CHECK-LABEL: func @check_sharding_after_identity +func.func @check_sharding_after_identity(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_sharding_after_identity, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03", "\04\05\06"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0A\0B\0C", "\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding_after_identity +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}, %{{[a-z0-9]+}}: tensor<*xi1> {mhlo.sharding = "\04\05\06"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\0A\0B\0C"}, tensor<*xi1> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding_after_identity(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) { + %0 = "tf.Identity"(%arg0) : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.XlaSharding"(%0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi32>) -> tensor<*xi32> + %2 = "tf.XlaSharding"(%arg1) { _XlaSharding = "\04\05\06", sharding = "\04\05\06" } : (tensor<*xi1>) -> tensor<*xi1> + %3, %4 = "tf.A"(%1, %2) : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + %5 = "tf.XlaSharding"(%3) { _XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C" } : (tensor<*xi32>) -> tensor<*xi32> + %6 = "tf.XlaSharding"(%4) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %5, %6 : tensor<*xi32> , tensor<*xi1> +} + +// ----- + +// Tests with input sharding following a ReadVariable op. +// CHECK-LABEL: func @check_sharding_after_read_variable +func.func @check_sharding_after_read_variable(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_sharding_after_read_variable, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03", "\04\05\06"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0A\0B\0C", "\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding_after_read_variable +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*x!tf_type.resource>> {mhlo.sharding = "\01\02\03"}, %{{[a-z0-9]+}}: tensor<*x!tf_type.resource>> {mhlo.sharding = "\04\05\06"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\0A\0B\0C"}, tensor<*xi1> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding_after_read_variable(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) -> (tensor<*xi32>, tensor<*xi1>) { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %1 = "tf.XlaSharding"(%0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<32xf32>) -> tensor<32xf32> + %2 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %3 = "tf.Identity"(%2) : (tensor<32xf32>) -> tensor<32xf32> + %4 = "tf.XlaSharding"(%3) { _XlaSharding = "\04\05\06", sharding = "\04\05\06" } : (tensor<32xf32>) -> tensor<32xf32> + %5, %6 = "tf.A"(%1, %3) : (tensor<32xf32>, tensor<32xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %7 = "tf.XlaSharding"(%5) { _XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C" } : (tensor<*xi32>) -> tensor<*xi32> + %8 = "tf.XlaSharding"(%6) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %7, %8 : tensor<*xi32> , tensor<*xi1> +} + +// ----- + +// Tests sharding propagation in while region body. +// CHECK-LABEL: func @check_sharding_for_read_variable_inside_while_body +func.func @check_sharding_for_read_variable_inside_while_body(%arg0 : tensor, %arg1: tensor<*x!tf_type.resource>>) { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %1:1 = "tf_device.cluster_func"(%arg0, %0) {func = @func_with_sharding_inside_while_body, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", num_cores_per_replica = 2 : i64, use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor, tensor<128x1024xf32>) -> (tensor<128x1024xf32>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["", "\0A\0B\0C"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding_inside_while_body +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor {mhlo.sharding = ""}, %{{[a-z0-9]+}}: tensor<128x1024xf32> {mhlo.sharding = "\0A\0B\0C"}) +// CHECK-SAME: -> (tensor<128x1024xf32> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding_inside_while_body(%arg0: tensor, %arg1: tensor<128x1024xf32>) -> (tensor<128x1024xf32>) { + %cst = "tf.Const"() <{value = dense<0> : tensor}> {device = ""} : () -> tensor + %0:2 = "tf.WhileRegion"(%cst, %arg1) <{is_stateless = false, parallel_iterations = 1 : i64}> ({ + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.Less"(%arg2, %arg0) : (tensor, tensor) -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.XlaSharding"(%arg3) <{_XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C"}> {unspecified_dims = []} : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + %2 = "tf.Square"(%1) : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + "tf.Yield"(%arg2, %2) : (tensor, tensor<128x1024xf32>) -> () + }) {_num_original_outputs = 1 : i64, _read_only_resource_inputs = [1], _xla_propagate_compile_time_consts = true} : (tensor, tensor<128x1024xf32>) -> (tensor, tensor<128x1024xf32>) + %1 = "tf.XlaSharding"(%0#1) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + func.return %1 : tensor<128x1024xf32> +} + +// ----- + +// Tests sharding propagation in while region condition. +// CHECK-LABEL: func @check_sharding_for_read_variable_inside_while_cond +func.func @check_sharding_for_read_variable_inside_while_cond(%arg0 : tensor, %arg1: tensor<*x!tf_type.resource>>) { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %1:1 = "tf_device.cluster_func"(%arg0, %0) {func = @func_with_sharding_inside_while_cond, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", num_cores_per_replica = 2 : i64, use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor, tensor<128x1024xf32>) -> (tensor<128x1024xf32>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["", "\0A\0B\0C"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding_inside_while_cond +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor {mhlo.sharding = ""}, %{{[a-z0-9]+}}: tensor<128x1024xf32> {mhlo.sharding = "\0A\0B\0C"}) +// CHECK-SAME: -> (tensor<128x1024xf32> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding_inside_while_cond(%arg0: tensor, %arg1: tensor<128x1024xf32>) -> (tensor<128x1024xf32>) { + %cst = "tf.Const"() <{value = dense<0> : tensor}> {device = ""} : () -> tensor + %0:2 = "tf.WhileRegion"(%cst, %arg1) <{is_stateless = false, parallel_iterations = 1 : i64}> ({ + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.XlaSharding"(%arg3) <{_XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C"}> {unspecified_dims = []} : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + %2 = "tf.Less"(%arg2, %arg0) : (tensor, tensor) -> tensor + "tf.Yield"(%2) : (tensor) -> () + }, { + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.Square"(%arg3) : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + "tf.Yield"(%arg2, %1) : (tensor, tensor<128x1024xf32>) -> () + }) {_num_original_outputs = 1 : i64, _read_only_resource_inputs = [1], _xla_propagate_compile_time_consts = true} : (tensor, tensor<128x1024xf32>) -> (tensor, tensor<128x1024xf32>) + %1 = "tf.XlaSharding"(%0#1) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + func.return %1 : tensor<128x1024xf32> +} + +// ----- + +// Tests output sharding propagation in while region body. +// CHECK-LABEL: func @check_output_sharding_for_while_region_op +func.func @check_output_sharding_for_while_region_op(%arg0 : tensor, %arg1: tensor<*x!tf_type.resource>>) { + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %1:1 = "tf_device.cluster_func"(%arg0, %0) {func = @func_with_sharded_while_region_op_output, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", num_cores_per_replica = 2 : i64, use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor, tensor<128x1024xf32>) -> (tensor<128x1024xf32>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["", "\0D\0E\0F"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharded_while_region_op_output +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor {mhlo.sharding = ""}, %{{[a-z0-9]+}}: tensor<128x1024xf32> {mhlo.sharding = "\0D\0E\0F"}) +// CHECK-SAME: -> (tensor<128x1024xf32> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharded_while_region_op_output(%arg0: tensor, %arg1: tensor<128x1024xf32>) -> (tensor<128x1024xf32>) { + %cst = "tf.Const"() <{value = dense<0> : tensor}> {device = ""} : () -> tensor + %0:2 = "tf.WhileRegion"(%cst, %arg1) <{is_stateless = false, parallel_iterations = 1 : i64}> ({ + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.Less"(%arg2, %arg0) : (tensor, tensor) -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tensor, %arg3: tensor<128x1024xf32>): + %1 = "tf.Square"(%arg3) : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + %2 = "tf.XlaSharding"(%1) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<128x1024xf32>) -> tensor<128x1024xf32> + "tf.Yield"(%arg2, %2) : (tensor, tensor<128x1024xf32>) -> () + }) {_num_original_outputs = 1 : i64, _read_only_resource_inputs = [1], _xla_propagate_compile_time_consts = true} : (tensor, tensor<128x1024xf32>) -> (tensor, tensor<128x1024xf32>) + func.return %0#1 : tensor<128x1024xf32> +} + +// ----- + +// Tests with input sharding following an identity op and cast op. +// CHECK-LABEL: func @check_sharding_after_cast_op +func.func @check_sharding_after_cast_op(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_sharding_after_cast, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03", "\04\05\06"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0A\0B\0C", "\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_sharding_after_cast +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}, %{{[a-z0-9]+}}: tensor<*xi1> {mhlo.sharding = "\04\05\06"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\0A\0B\0C"}, tensor<*xi1> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_sharding_after_cast(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) { + %0 = "tf.Identity"(%arg0) : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.Cast"(%0) : (tensor<*xi32>) -> tensor<*xi1> + %2 = "tf.XlaSharding"(%1) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi1>) -> tensor<*xi1> + %3 = "tf.XlaSharding"(%arg1) { _XlaSharding = "\04\05\06", sharding = "\04\05\06" } : (tensor<*xi1>) -> tensor<*xi1> + %4, %5 = "tf.A"(%2, %3) : (tensor<*xi1>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + %6 = "tf.XlaSharding"(%4) { _XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C" } : (tensor<*xi32>) -> tensor<*xi32> + %7 = "tf.XlaSharding"(%5) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %6, %7 : tensor<*xi32> , tensor<*xi1> +} + +// ----- + +// Tests that input sharding inside a functional op is parsed correctly. +// CHECK-LABEL: func @check_sharding_inside_functional_op +func.func @check_sharding_inside_functional_op(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) { + "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_device_training_loop, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03", "\04\05\06"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\0A\0B\0C", "\0D\0E\0F"] + func.return +} + +// CHECK-LABEL: func @func_with_device_training_loop +// CHECK-SAME: (%{{[a-z0-9]+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}, %{{[a-z0-9]+}}: tensor<*xi1> {mhlo.sharding = "\04\05\06"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\0A\0B\0C"}, tensor<*xi1> {mhlo.sharding = "\0D\0E\0F"}) +func.func @func_with_device_training_loop(%arg0: tensor<*xi32>, %arg1: tensor<*xi1>) -> (tensor<*xi32>, tensor<*xi1>) { + %1:2 = "tf.StatefulPartitionedCall"(%arg0){f= @func_body, config="", config_proto="", executor_type=""} + : (tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + %2 = "tf.PartitionedCall"(%arg1) {config = "", config_proto = "", executor_type = "", f = @pcall_func_body} : (tensor<*xi1>) -> (tensor) + %3, %4 = "tf.A"(%1#0, %2) : (tensor<*xi32>, tensor) -> (tensor<*xi32>, tensor<*xi1>) + + %5 = "tf.XlaSharding"(%3) { _XlaSharding = "\0A\0B\0C", sharding = "\0A\0B\0C" } : (tensor<*xi32>) -> tensor<*xi32> + %6 = "tf.XlaSharding"(%4) { _XlaSharding = "\0D\0E\0F", sharding = "\0D\0E\0F" } : (tensor<*xi1>) -> tensor<*xi1> + + func.return %5, %6 : tensor<*xi32> , tensor<*xi1> +} + +// CHECK-LABEL: func @func_body +func.func @func_body(%arg0: tensor<*xi32>)-> (tensor<*xi32>, tensor<*xi1>) { + %1 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi32>) -> tensor<*xi32> + %2, %3 = "tf.C"(%1) : (tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + func.return %2, %3 : tensor<*xi32> , tensor<*xi1> +} + +// CHECK-LABEL: func @pcall_func_body +func.func @pcall_func_body(%arg0: tensor<*xi1>) -> tensor { + %1 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\04\05\06", sharding = "\04\05\06" } : (tensor<*xi1>) -> tensor<*xi1> + %2 = "tf.D"(%1) : (tensor<*xi1>) -> (tensor) + func.return %2 : tensor +} + +// ----- + +// Tests that output sharding inside a functional op is parsed correctly. + +// CHECK-LABEL: func @check_sharding_inside_functional_op +func.func @check_sharding_inside_functional_op(%arg0: tensor<*xi32>) { + "tf_device.cluster_func"(%arg0) {func = @cluster_func, step_marker_location = "", num_cores_per_replica = 1 : i64} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: input_sharding_configuration + // CHECK-SAME: ["\01\02\03"] + // CHECK: output_sharding_configuration + // CHECK-SAME: ["\01\02\03"] + func.return +} + +func.func @cluster_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.PartitionedCall"(%arg0) {f= @func_body, config="", config_proto="", executor_type=""} : (tensor<*xi32>) -> tensor<*xi32> + func.return %0 : tensor<*xi32> +} + +func.func @func_body(%arg0: tensor<*xi32>)-> tensor<*xi32> { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03", sharding = "\01\02\03" } : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.Identity"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %1 : tensor<*xi32> +} + +// ----- + +// Tests partitioned data inputs/outputs are set correctly (via XLA SPMD) is +// enabled. Non replicated inputs/outputs should have shardings set to be +// replicate sharding (""). + +// CHECK-LABEL: func @partitioned_input_output +func.func @partitioned_input_output(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\01\02\03", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\01\02\03", ""] + // CHECK-SAME: output_sharding_configuration = ["", "\04\05\06"] + %1:2 = "tf_device.cluster_func"(%0, %arg1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + %2 = "tf.TPUPartitionedOutputV2"(%1#1) {_XlaSharding = "\04\05\06", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + func.return %1#0, %2 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xi32> {mhlo.sharding = "\01\02\03"}, {{.+}}: tensor<*xi32> {mhlo.sharding = ""}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = ""}, tensor<*xi32> {mhlo.sharding = "\04\05\06"}) +func.func @cluster_func(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + func.return %arg0, %arg1 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// Tests XlaShardingV2 is preferred over XlaSharding if set. + +// CHECK-LABEL: func @partitioned_input_output +func.func @partitioned_input_output(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "{devices=[2,1]1,0}", _XlaShardingV2 = "{devices=[2,1]<=[2]}", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["{devices=[2,1]<=[2]}", ""] + // CHECK-SAME: output_sharding_configuration = ["", "{devices=[2,1]<=[2]}"] + %1:2 = "tf_device.cluster_func"(%0, %arg1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + %2 = "tf.TPUPartitionedOutputV2"(%1#1) {_XlaSharding = "{devices=[2,1]1,0}", _XlaShardingV2 = "{devices=[2,1]<=[2]}", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + func.return %1#0, %2 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xi32> {mhlo.sharding = "{devices=[2,1]<=[2]}"}, {{.+}}: tensor<*xi32> {mhlo.sharding = ""}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = ""}, tensor<*xi32> {mhlo.sharding = "{devices=[2,1]<=[2]}"}) +func.func @cluster_func(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + func.return %arg0, %arg1 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// Tests partitioned variables (via XLA SPMD) propagates shardings correctly. + +// CHECK-LABEL: func @partitioned_variable +func.func @partitioned_variable(%arg0: tensor>>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\01\02\03", partition_dims = []} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<*xf32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\01\02\03"] + // CHECK-SAME: output_sharding_configuration = [] + "tf_device.cluster_func"(%1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xf32>) -> () + func.return +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xf32> {mhlo.sharding = "\01\02\03"}) +func.func @cluster_func(%arg0: tensor<*xf32>) { + func.return +} + +// ----- + +// Tests that device variable sharding for an implicitly capture device variable +// defaults to REPLICATE. + +// CHECK-LABEL: func @replicated_device_variable +func.func @replicated_device_variable(%arg0: tensor<*x!tf_type.resource>>) { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<*xf32> + tf_device.replicate() + {n = 2 : i32} { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = [""] + "tf_device.cluster_func"(%0) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xf32>) -> () + tf_device.return + } + func.return +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xf32> {mhlo.sharding = ""}) +func.func @cluster_func(%arg0: tensor<*xf32>) { + func.return +} + +// ----- + +// Tests TPIv2 with a "partially tiled" XLA annotation where: +// type: OTHER +// tile_assignment_dimensions: [4, 1, 1, 1, 2] +// tile_assignment_devices: [0, 1, 2, 3, 4, 5, 6, 7] +// replicate_on_last_tile_dim: true +// Serialized string: +// "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01" + +// CHECK-LABEL: func @partial_tile_partitioned_variable +func.func @partial_tile_partitioned_variable(%arg0: tensor>>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01", partition_dims = [4, 1, 1, 1, 2], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<4x4x4x4xf32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01"] + // CHECK-SAME: output_sharding_configuration = [] + // CHECK-SAME: use_spmd_for_xla_partitioning = true + "tf_device.cluster_func"(%1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 8 : i64} : (tensor<4x4x4x4xf32>) -> () + func.return +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<4x4x4x4xf32> {mhlo.sharding = "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01"}) +func.func @cluster_func(%arg0: tensor<4x4x4x4xf32>) { + func.return +} + +// ----- + +// Tests TPIv2 with a "subgroup tiled" XLA annotation where: +// type: OTHER +// tile_assignment_dimensions: [4, 1, 1, 1, 2] +// tile_assignment_devices: [0, 1, 2, 3, 4, 5, 6, 7] +// last_tile_dims: [REPLICATED] +// Serialized string: +// "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\07B\01\00" + +// CHECK-LABEL: func @subgroup_tile_partitioned_variable +func.func @subgroup_tile_partitioned_variable(%arg0: tensor>>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\07B\01\00", partition_dims = [4, 1, 1, 1, 2], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<4x4x4x4xf32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\07B\01\00"] + // CHECK-SAME: output_sharding_configuration = [] + // CHECK-SAME: use_spmd_for_xla_partitioning = true + "tf_device.cluster_func"(%1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 8 : i64} : (tensor<4x4x4x4xf32>) -> () + func.return +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<4x4x4x4xf32> {mhlo.sharding = "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\07B\01\00"}) +func.func @cluster_func(%arg0: tensor<4x4x4x4xf32>) { + func.return +} + +// ----- + +// Tests TPIv2 with a "partially tiled" XLA annotation where: +// type: OTHER +// tile_assignment_dimensions: [4, 1, 1, 1, 2] +// tile_assignment_devices: [0, 1, 2, 3, 4, 5, 6, 7] +// replicate_on_last_tile_dim: true +// Serialized string: +// "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01" + +// This sharding has an extra dimension than the TPIv2's rank, causing an error. + +func.func @partitioned_input_rank_mismatch(%arg0: tensor>>) { + // expected-error @+1 {{rank}} + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\08\03\1A\05\04\01\01\01\02\22\08\00\01\02\03\04\05\06\070\01", partition_dims = [4, 1, 1, 2], is_packed = true} : (tensor>>) -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<4x4x4xf32> + "tf_device.cluster_func"(%1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 8 : i64} : (tensor<4x4x4xf32>) -> () + func.return +} + +func.func @cluster_func(%arg0: tensor<4x4x4xf32>) { + func.return +} + +// ----- + +// Tests partitioned inputs/outputs with no sharding (via XLA SPMD) defaults to +// replicate sharding (""). + +// CHECK-LABEL: func @partitioned_input_output +func.func @partitioned_input_output(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = [""] + // CHECK-SAME: output_sharding_configuration = [""] + %1 = "tf_device.cluster_func"(%0) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xi32>) -> tensor<*xi32> + %2 = "tf.TPUPartitionedOutputV2"(%1) {partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + func.return %2 : tensor<*xi32> +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xi32> {mhlo.sharding = ""}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = ""}) +func.func @cluster_func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + func.return %arg0 : tensor<*xi32> +} + +// ----- + +// Tests output sharding of unpartitioned resource write takes on same sharding +// as unpartitioned resource. + +// CHECK-LABEL: func @partitioned_input_output +func.func @partitioned_input_output(%arg0: tensor>>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\01\02\03", partition_dims = []} : (tensor>>) -> tensor>> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = [] + // CHECK-SAME: output_sharding_configuration = ["\01\02\03"] + %1 = "tf_device.cluster_func"() {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : () -> tensor + "tf.AssignVariableOp"(%0, %1) : (tensor>>, tensor) -> () + func.return +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: -> (tensor {mhlo.sharding = "\01\02\03"}) +func.func @cluster_func() -> tensor { + %0 = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + func.return %0 : tensor +} + +// ----- + +// Tests if any of inputs have MAXIMAL sharding, then revert to MPMD. + +// CHECK-LABEL: func @partitioned_input_maximal_sharding_revert_mpmd +func.func @partitioned_input_maximal_sharding_revert_mpmd(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\08\01\1A\01\01\22\01\00", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"] + // CHECK-SAME: output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\04\05\06"] + // CHECK-SAME: use_spmd_for_xla_partitioning = false + %1:2 = "tf_device.cluster_func"(%0, %arg1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + %2 = "tf.TPUPartitionedOutputV2"(%1#1) {_XlaSharding = "\04\05\06", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + func.return %1#0, %2 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, {{.+}}: tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor<*xi32> {mhlo.sharding = "\04\05\06"}) +func.func @cluster_func(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + func.return %arg0, %arg1 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// Tests if any of outputs have MAXIMAL sharding, then revert to MPMD. + +// CHECK-LABEL: func @partitioned_output_maximal_sharding_revert_mpmd +func.func @partitioned_output_maximal_sharding_revert_mpmd(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\04\05\06", "\08\01\1A\01\01\22\01\00"] + // CHECK-SAME: output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"] + // CHECK-SAME: use_spmd_for_xla_partitioning = false + %0 = "tf.TPUPartitionedInputV2"(%arg0) {_XlaSharding = "\04\05\06", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + %1:2 = "tf_device.cluster_func"(%0, %arg1) {func = @cluster_func, use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64} : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + %2 = "tf.TPUPartitionedOutputV2"(%1#1) {_XlaSharding = "\08\01\1A\01\01\22\01\00", partition_dims = []} : (tensor<*xi32>) -> tensor<*xi32> + func.return %1#0, %2 : tensor<*xi32>, tensor<*xi32> +} + +// CHECK-LABEL: func @cluster_func +// CHECK-SAME: ({{.+}}: tensor<*xi32> {mhlo.sharding = "\04\05\06"}, {{.+}}: tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +// CHECK-SAME: -> (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) +func.func @cluster_func(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + func.return %arg0, %arg1 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_downwards_with_alias +func.func @check_propagation_downwards_with_alias(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\01\02\03", "\04\05\06"] + // CHECK-SAME: output_sharding_configuration = ["\04\05\06", "\01\02\03"] + "tf_device.cluster_func"(%arg0, %arg1) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + func.return +} + +// CHECK-LABEL: func @func +// CHECK-SAME: %arg0: tensor<*xi32> {mhlo.sharding = "\01\02\03" +// CHECK-SAME: %arg1: tensor<*xi32> {mhlo.sharding = "\04\05\06" +// CHECK-SAME: ->{{.*}}mhlo.sharding = "\04\05\06"{{.*}}mhlo.sharding = "\01\02\03" +func.func @func(%arg0: tensor<*xi32> {tf.aliasing_output = 1 : i64}, + %arg1: tensor<*xi32> {tf.aliasing_output = 0 : i64}) -> (tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.XlaSharding"(%arg1) { _XlaSharding = "\04\05\06"} : (tensor<*xi32>) -> tensor<*xi32> + // flip order + %2 = "tf.A"(%1) : (tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.B"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %2, %3 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: func @check_symmetric_alias_propagation +func.func @check_symmetric_alias_propagation(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\01\02\03", "\04\05\06"] + // CHECK-SAME: output_sharding_configuration = ["\01\02\03", "\04\05\06"] + "tf_device.cluster_func"(%arg0, %arg1) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + func.return +} + +// CHECK-LABEL: func @func +// CHECK-SAME: %arg0: tensor<*xi32> {mhlo.sharding = "\01\02\03" +// CHECK-SAME: %arg1: tensor<*xi32> {mhlo.sharding = "\04\05\06" +// CHECK-SAME: ->{{.*}}mhlo.sharding = "\01\02\03"{{.*}}mhlo.sharding = "\04\05\06" +func.func @func(%arg0: tensor<*xi32> {tf.aliasing_output = 0 : i64}, + %arg1: tensor<*xi32> {tf.aliasing_output = 1 : i64}) -> (tensor<*xi32>, tensor<*xi32>) { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + %2 = "tf.B"(%arg1) : (tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "\04\05\06"} : (tensor<*xi32>) -> tensor<*xi32> + func.return %2, %3 : tensor<*xi32>, tensor<*xi32> +} + +// ----- + +// Partial tiled inputs using XlaSharding ops identified as REPLICATED should keep the sharding configuration. +// The following xla.OpSharding is used: +// Proto debug string: +// type : OTHER +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 1 +// tile_assignment_dimensions: 2 +// tile_assignment_devices: 0 +// tile_assignment_devices: 1 +// last_tile_dims: REPLICATED +// Serialized string: +// "\08\03\1A\03\01\01\02\22\02\00\01B\01\00" + +// CHECK-LABEL: func @check_partial_tile_mpmd_fallback +func.func @check_partial_tile_mpmd_fallback(%arg0: tensor<2x7xi64>) -> tensor<2x7xi32> { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\08\03\1A\03\01\01\02\22\02\00\01B\01\00"] + // CHECK-SAME: output_sharding_configuration = [""] + // CHECK-SAME: use_spmd_for_xla_partitioning = true + %0 = "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 2 : i64 + } : (tensor<2x7xi64>) -> (tensor<2x7xi32>) + %1 = "tf.Identity"(%0) : (tensor<2x7xi32>) -> tensor<2x7xi32> + func.return %1 : tensor<2x7xi32> +} + +// CHECK-LABEL: func @func +// CHECK-SAME: %arg0: tensor<2x7xi64> {mhlo.sharding = "\08\03\1A\03\01\01\02\22\02\00\01B\01\00" +func.func @func(%arg0: tensor<2x7xi64>) -> (tensor<2x7xi32>) { + %0 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<2x7xi64>) -> tensor<2x7xi32> + %1 = "tf.XlaSharding"(%0) {_XlaSharding = "\08\03\1A\03\01\01\02\22\02\00\01B\01\00", sharding = "\08\03\1A\03\01\01\02\22\02\00\01B\01\00", unspecified_dims = []} : (tensor<2x7xi32>) -> tensor<2x7xi32> + func.return %0 : tensor<2x7xi32> +} + +// ----- + +// CHECK-LABEL: func @check_arg_sharding_errors +func.func @check_arg_sharding_errors(%arg0: tensor<1x2x3xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"] + // CHECK-SAME: use_spmd_for_xla_partitioning = false + "tf_device.cluster_func"(%arg0) {func = @func, num_cores_per_replica = 1 : i64} : (tensor<1x2x3xi32>) -> tensor<1x2x3xi32> + func.return +} + +func.func @func(%arg0: tensor<1x2x3xi32>) -> tensor<1x2x3xi32> { + // Use a four dimension sharding (devices=[1,1,1,1]0) + // Since the input tensor only has three dimensions, we expect this to fail. + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\08\03\1A\04\01\01\01\01\22\01\00" } : (tensor<1x2x3xi32>) -> tensor<1x2x3xi32> + %1 = "tf.A"(%0) : (tensor<1x2x3xi32>) -> (tensor<1x2x3xi32>) + func.return %1: tensor<1x2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @check_retval_sharding_errors +func.func @check_retval_sharding_errors(%arg0: tensor<1x2x3xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"] + // CHECK-SAME: use_spmd_for_xla_partitioning = false + "tf_device.cluster_func"(%arg0) {func = @func, num_cores_per_replica = 1 : i64} : (tensor<1x2x3xi32>) -> tensor<1x2x3xi32> + func.return +} + +func.func @func(%arg0: tensor<1x2x3xi32>) -> tensor<1x2x3xi32> { + %0 = "tf.A"(%arg0) : (tensor<1x2x3xi32>) -> (tensor<1x2x3xi32>) + // Use a four dimension sharding (devices=[1,1,1,1]0) + // Since the output tensor only has three dimensions, we expect this to fail. + %1 = "tf.XlaSharding"(%0) { _XlaSharding = "\08\03\1A\04\01\01\01\01\22\01\00" } : (tensor<1x2x3xi32>) -> tensor<1x2x3xi32> + func.return %1: tensor<1x2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_upwards_when_spmd_for_xla_is_true +func.func @check_propagation_upwards_when_spmd_for_xla_is_true(%arg0: tensor<*xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["\01\02\03"] + // CHECK-SAME: output_sharding_configuration = [""] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64 + } : (tensor<*xi32>) -> tensor<*xi32> + func.return +} + +func.func @func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %1 : tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_downwards_when_spmd_for_xla_is_true +func.func @check_propagation_downwards_when_spmd_for_xla_is_true(%arg0: tensor<*xi32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = [""] + // CHECK-SAME: output_sharding_configuration = ["\01\02\03"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64 + } : (tensor<*xi32>) -> tensor<*xi32> + func.return +} + +func.func @func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.A"(%arg0) : (tensor<*xi32>) -> (tensor<*xi32>) + %1 = "tf.XlaSharding"(%0) { _XlaSharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + func.return %1 : tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_downwards_through_ops +func.func @check_propagation_downwards_through_ops(%arg0: tensor<4xf32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: output_sharding_configuration = ["\01\02\03"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<4xf32>) -> tensor<4xf32> + func.return +} + +// CHECK-LABEL: func @func +// CHECK-SAME: ->{{.*}}mhlo.sharding = "\01\02\03" +func.func @func(%arg0: tensor<4xf32>) -> tensor<4xf32> { + %cst = "tf.Const"() {value = dense<23.0> : tensor<4xf32>} : () -> tensor<4xf32> + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03"} : (tensor<4xf32>) -> tensor<4xf32> + %1 = "tf.AddV2"(%0, %cst) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + %2 = "tf.AddV2"(%cst, %1) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + %3 = "tf.Mul"(%2, %cst) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + %4 = "tf.Mul"(%cst, %3) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + %5 = "tf.Sub"(%4, %cst) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + %6 = "tf.Sub"(%cst, %5) : (tensor<4xf32>, tensor<4xf32>) -> (tensor<4xf32>) + func.return %6 : tensor<4xf32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_through_same_operands_and_result_type_trait_ops +func.func @check_propagation_through_same_operands_and_result_type_trait_ops(%arg0: tensor<4xf32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: output_sharding_configuration = ["\01\02\03"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<4xf32>) -> tensor<4xf32> + func.return +} + +// CHECK-LABEL: func @func +// CHECK-SAME: ->{{.*}}mhlo.sharding = "\01\02\03" +func.func @func(%arg0: tensor<4xf32>) -> tensor<4xf32> { + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "\01\02\03"} : (tensor<4xf32>) -> tensor<4xf32> + %1 = "tf.Abs"(%0) : (tensor<4xf32>) -> (tensor<4xf32>) + func.return %1 : tensor<4xf32> +} + +// ----- + +// CHECK-LABEL: func @check_propagation_for_output_sharding_from_tf_matmul +// CHECK: tf_device.cluster_func +// CHECK-SAME: input_sharding_configuration = ["", ""] +// CHECK-SAME: output_sharding_configuration = ["\08\03\1A\02\02\01\22\02\00\01"] +func.func @check_propagation_for_output_sharding_from_tf_matmul(%arg0: tensor<2x4xf32>, %arg1: tensor<4x2xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) { + %0 = "tf_device.cluster_func"(%arg0, %arg1) {func = @_func, use_spmd_for_xla_partitioning = true, use_tpu = true, num_cores_per_replica = 2 : i64} : (tensor<2x4xf32>, tensor<4x2xf32>) -> tensor<2x2xf32> + %1:2 = "tf.TPUPartitionedOutputV2"(%0) {device = "", partition_dims = [2, 1]} : (tensor<2x2xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) + return %1#0, %1#1 : tensor<1x2xf32>, tensor<1x2xf32> +} +func.func @_func(%arg0: tensor<2x4xf32>, %arg1: tensor<4x2xf32>) -> tensor<2x2xf32> { + %0 = "tf.MatMul"(%arg0, %arg1) {_XlaSharding = "\08\03\1A\02\02\01\22\02\00\01"} : (tensor<2x4xf32>, tensor<4x2xf32>) -> tensor<2x2xf32> + return %0 : tensor<2x2xf32> +} + +// ----- +// CHECK-LABEL: func @check_propagation_for_output_sharding_from_tf_matmul_following_by_identity_op +// CHECK: tf_device.cluster_func +// CHECK-SAME: input_sharding_configuration = ["", ""] +// CHECK-SAME: output_sharding_configuration = ["\08\03\1A\02\02\01\22\02\00\01"] +func.func @check_propagation_for_output_sharding_from_tf_matmul_following_by_identity_op(%arg0: tensor<2x4xf32>, %arg1: tensor<4x2xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) { + %0 = "tf_device.cluster_func"(%arg0, %arg1) {func = @_func, use_spmd_for_xla_partitioning = true, use_tpu = true, num_cores_per_replica = 2 : i64} : (tensor<2x4xf32>, tensor<4x2xf32>) -> tensor<2x2xf32> + %1:2 = "tf.TPUPartitionedOutputV2"(%0) {device = "", partition_dims = [2, 1]} : (tensor<2x2xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) + return %1#0, %1#1 : tensor<1x2xf32>, tensor<1x2xf32> +} +func.func @_func(%arg0: tensor<2x4xf32>, %arg1: tensor<4x2xf32>) -> tensor<2x2xf32> { + %0 = "tf.MatMul"(%arg0, %arg1) {_XlaSharding = "\08\03\1A\02\02\01\22\02\00\01"} : (tensor<2x4xf32>, tensor<4x2xf32>) -> tensor<2x2xf32> + %1 = "tf.Identity"(%0) : (tensor<2x2xf32>) -> tensor<2x2xf32> + return %1 : tensor<2x2xf32> +} + +// ----- +// The following op sharding is used in the following test case: +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\00" +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// Serialized string: +// "\08\01\1A\01\01\22\01\01" + +// CHECK-LABEL: func @check_propagation_for_tpu_replicate_core +// CHECK: tf_device.cluster_func +// CHECK-SAME: input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\01"] +// CHECK-SAME: output_sharding_configuration = ["\08\01\1A\01\01\22\01\01"] +func.func @check_propagation_for_tpu_replicate_core(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>) -> (tensor, tensor, tensor, tensor) { + %cst = "tf.Const"() {value = dense<5.000000e+00> : tensor} : () -> tensor + %0 = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster_tpu_function", device = ""} : () -> tensor + %1:4 = tf_device.replicate([%cst, %cst, %cst, %cst] as %arg2: tensor, %arg0 as %arg3: tensor<*x!tf_type.resource>>, %arg1 as %arg4: tensor<*x!tf_type.resource>>) {_mirrored_variable_indices = [1, 2], n = 4 : i32} { + %10 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor + %11 = "tf.ReadVariableOp"(%arg3) : (tensor<*x!tf_type.resource>>) -> tensor + %12 = "tf_device.cluster_func"(%11, %arg2, %10) {func = @_func, host_compute_core = [], num_cores_per_replica = 2 : i64, use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor, tensor, tensor) -> tensor + tf_device.return %12 : tensor + } + return %1#0, %1#1, %1#2, %1#3 : tensor, tensor, tensor, tensor +} +func.func private @_func(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + %0 = "tf.Mul"(%arg0, %arg1) {device = "/device:TPU_REPLICATED_CORE:0"} : (tensor, tensor) -> tensor + %1 = "tf.Mul"(%arg2, %0) {device = "/device:TPU_REPLICATED_CORE:1"} : (tensor, tensor) -> tensor + return %1 : tensor +} + +// ----- +// CHECK-LABEL: func @check_AddV2_sharding_propagation +func.func @check_AddV2_sharding_propagation(%arg0: tensor<4xf32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["sharding_info_1"] + // CHECK-SAME: output_sharding_configuration = ["sharding_info_1"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<4xf32>) -> tensor<4xf32> + func.return +} + +// CHECK-LABEL: func @func +// CHECK: {{.*}}mhlo.sharding = "sharding_info_1"{{.*}}->{{.*}}mhlo.sharding = "sharding_info_1" +func.func @func(%arg0: tensor<4xf32>) -> tensor<4xf32> { + %cst = "tf.Const"() {value = dense<23.0> : tensor<4xf32>} : () -> tensor<4xf32> + %add = "tf.AddV2"(%cst, %arg0) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %0 = "tf.XlaSharding"(%add) { _XlaSharding = "sharding_info_1"} : (tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// ----- +// CHECK-LABEL: func @check_Exp_sharding_propagation +func.func @check_Exp_sharding_propagation(%arg0: tensor<4xf32>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["sharding_info_1"] + // CHECK-SAME: output_sharding_configuration = ["sharding_info_1"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = false, num_cores_per_replica = 1 : i64 + } : (tensor<4xf32>) -> tensor<4xf32> + func.return +} + +// CHECK-LABEL: func @func +// CHECK: {{.*}}mhlo.sharding = "sharding_info_1"{{.*}}->{{.*}}mhlo.sharding = "sharding_info_1" +func.func @func(%arg0: tensor<4xf32>) -> tensor<4xf32> { + %exp = "tf.Exp"(%arg0) : (tensor<4xf32>) -> tensor<4xf32> + %0 = "tf.XlaSharding"(%exp) { _XlaSharding = "sharding_info_1"} : (tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// ----- +// CHECK-LABEL: func @check_Mul_no_input_sharding_propagation +func.func @check_Mul_no_input_sharding_propagation(%arg0: tensor) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = [""] + // CHECK-SAME: output_sharding_configuration = ["sharding_info_2"] + "tf_device.cluster_func"(%arg0) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64 + } : (tensor) -> tensor<4xf32> + func.return +} + +// CHECK-LABEL: func @func +// CHECK: {{.*}}mhlo.sharding = ""{{.*}}->{{.*}}mhlo.sharding = "sharding_info_2" +func.func @func(%arg0: tensor) -> tensor<4xf32> { + %cst = "tf.Const"() {value = dense<23.0> : tensor<4xf32>} : () -> tensor<4xf32> + %mul = "tf.Mul"(%cst, %arg0) : (tensor<4xf32>, tensor) -> tensor<4xf32> + %0 = "tf.XlaSharding"(%mul) { _XlaSharding = "sharding_info_1"} : (tensor<4xf32>) -> tensor<4xf32> + %1 = "tf.XlaSharding"(%cst) { _XlaSharding = "sharding_info_2"} : (tensor<4xf32>) -> tensor<4xf32> + func.return %1 : tensor<4xf32> +} + + +// ----- +// CHECK-LABEL: func @check_AddV2_variant_shape_with_input_sharding_propagation +func.func @check_AddV2_variant_shape_with_input_sharding_propagation(%arg0: tensor, %arg1: tensor<12x384xbf16>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["sharding_info_1", "sharding_info_1"] + // CHECK-SAME: output_sharding_configuration = ["sharding_info_1"] + "tf_device.cluster_func"(%arg0, %arg1) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64 + } : (tensor, tensor<12x384xbf16>) -> tensor + func.return +} + +// CHECK-LABEL: func @func +// CHECK: {{.*}}mhlo.sharding = "sharding_info_1"{{.*}}mhlo.sharding = "sharding_info_1"{{.*}}->{{.*}}mhlo.sharding = "sharding_info_1" +func.func @func(%arg0: tensor, %arg1: tensor<12x384xbf16>) -> tensor { + %add = "tf.AddV2"(%arg0, %arg1) : (tensor, tensor<12x384xbf16>) -> tensor + %0 = "tf.XlaSharding"(%add) { _XlaSharding = "sharding_info_1"} : (tensor) -> tensor + func.return %0 : tensor +} + + + +// ----- +// CHECK-LABEL: func @check_BatchMatMul_variant_shape_without_input_sharding_propagation +func.func @check_BatchMatMul_variant_shape_without_input_sharding_propagation(%arg0: tensor, %arg1: tensor<256x384xbf16>) { + // CHECK: tf_device.cluster_func + // CHECK-SAME: input_sharding_configuration = ["", ""] + // CHECK-SAME: output_sharding_configuration = ["sharding_info_1"] + "tf_device.cluster_func"(%arg0, %arg1) { + func = @func, + use_spmd_for_xla_partitioning = true, num_cores_per_replica = 1 : i64 + } : (tensor, tensor<256x384xbf16>) -> tensor + func.return +} + +// CHECK-LABEL: func @func +// CHECK: {{.*}}mhlo.sharding = ""{{.*}}mhlo.sharding = ""{{.*}}->{{.*}}mhlo.sharding = "sharding_info_1" +func.func @func(%arg0: tensor, %arg1: tensor<256x384xbf16>) -> tensor { + %mul = "tf.BatchMatMul"(%arg0, %arg1) : (tensor, tensor<256x384xbf16>) -> tensor + %0 = "tf.XlaSharding"(%mul) { _XlaSharding = "sharding_info_1"} : (tensor) -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_space_to_depth_pass.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_space_to_depth_pass.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ff64f2833b7ab12ff11fa9bdb48e5ac8b421f2c8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_space_to_depth_pass.mlir @@ -0,0 +1,236 @@ +// RUN: tf-opt %s -split-input-file -tf-tpu-space-to-depth-pass | FileCheck %s + +// Tests for space to depth host and device transform. + +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:1" = {}, "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0" = {}}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 390 : i32}} { + func.func @main(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg1: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg2: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg3: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg4: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg5: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg6: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg7: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) attributes {tf.entry_function = {control_outputs = "while", inputs = "iterator,iterator_1,iterator_2,iterator_3,while_input_6,while_input_7,while_input_8,while_input_9", outputs = ""}} { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense<2> : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %3:10 = "tf.While"(%2, %1, %2, %0, %1, %arg2, %arg4, %arg5, %arg6, %arg7) {_lower_using_switch_merge = true, _num_original_outputs = 10 : i64, _read_only_resource_inputs = [], body = @while_body_2710, cond = @while_cond_2700, device = "", is_stateless = false, parallel_iterations = 10 : i64} : (tensor, tensor, tensor, tensor, tensor, tensor, tensor>>, tensor>>, tensor>>, tensor>>) -> (tensor, tensor, tensor, tensor, tensor, tensor, tensor>>, tensor>>, tensor>>, tensor>>) + func.return + } + // CHECK-LABEL: func @while_body_2710 + func.func @while_body_2710(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg6: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg7: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg8: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg9: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> (tensor, tensor, tensor, tensor, tensor, tensor, tensor>>, tensor>>, tensor>>, tensor>>) attributes {tf.signature.is_stateful} { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // CHECK: %[[INPUT:.*]] = "tf.IteratorGetNext" + %1 = "tf.IteratorGetNext"(%arg5) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor<2x224x224x3xf32> + // CHECK-DAG: %[[SPACETODEPTH0:.*]] = "tf.SpaceToDepth"([[INPUT:.*]]) <{block_size = 2 : i64, data_format = "NHWC"}> : (tensor<2x224x224x3xf32>) -> tensor<2x112x112x12xf32> + %2 = "tf.AddV2"(%arg2, %arg3) {device = ""} : (tensor, tensor) -> tensor + %3 = "tf.ReadVariableOp"(%arg6) : (tensor>>) -> tensor<7x7x3x64xf32> + %4 = "tf.ReadVariableOp"(%arg8) : (tensor>>) -> tensor + %5 = "tf.ReadVariableOp"(%arg7) : (tensor>>) -> tensor + %6 = "tf.ReadVariableOp"(%arg9) : (tensor>>) -> tensor + %7:2 = "tf_device.cluster_func"(%1, %3, %5, %6) {_xla_compile_device_type = "TPU", _replication_info = "while/cluster_while_body_271", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [0, 0, 0, 0], func = @_func, host_compute_core = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], num_cores_per_replica = 1 : i64, output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", use_tpu = true} : (tensor<2x224x224x3xf32>, tensor<7x7x3x64xf32>, tensor, tensor) -> (tensor<7x7x3x64xf32>, tensor) + "tf.AssignVariableOp"(%arg6, %7#0) : (tensor>>, tensor<7x7x3x64xf32>) -> () + "tf.AssignVariableOp"(%arg9, %7#1) : (tensor>>, tensor) -> () + %8 = "tf.Identity"(%arg1) {device = ""} : (tensor) -> tensor + %9 = "tf.Identity"(%2) {device = ""} : (tensor) -> tensor + %10 = "tf.AddV2"(%arg0, %0) {device = ""} : (tensor, tensor) -> tensor + %11 = "tf.Identity"(%10) {device = ""} : (tensor) -> tensor + func.return %11, %8, %9, %arg3, %arg4, %arg5, %arg6, %arg7, %arg8, %arg9 : tensor, tensor, tensor, tensor, tensor, tensor, tensor>>, tensor>>, tensor>>, tensor>> + } + func.func @while_cond_2700(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg6: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg7: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg8: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}, %arg9: tensor>> {tf.device = "/job:localhost/replica:0/task:0/device:TPU:0"}) -> tensor { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %1 = "tf.GreaterEqual"(%arg3, %0) {device = ""} : (tensor, tensor) -> tensor + %2 = "tf.Less"(%arg3, %0) {device = ""} : (tensor, tensor) -> tensor + %3 = "tf.Greater"(%arg2, %arg4) {device = ""} : (tensor, tensor) -> tensor + %4 = "tf.LogicalAnd"(%2, %3) {device = ""} : (tensor, tensor) -> tensor + %5 = "tf.Less"(%arg2, %arg4) {device = ""} : (tensor, tensor) -> tensor + %6 = "tf.LogicalAnd"(%1, %5) {device = ""} : (tensor, tensor) -> tensor + %7 = "tf.LogicalOr"(%6, %4) {device = ""} : (tensor, tensor) -> tensor + %8 = "tf.Less"(%arg0, %arg1) {device = ""} : (tensor, tensor) -> tensor + %9 = "tf.LogicalAnd"(%8, %7) {device = ""} : (tensor, tensor) -> tensor + %10 = "tf.Identity"(%9) {device = ""} : (tensor) -> tensor + func.return %10 : tensor + } + // CHECK-LABEL: func private @_func + // CHECK-SAME: [[FUNCINPUT0:.*]]: tensor<2x112x112x12xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, [[FUNCINPUT1:%.*]]: tensor<7x7x3x64xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, [[FUNCINPUT2:%.*]]: tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, [[VAL_59:%.*]]: tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor<7x7x3x64xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + func.func private @_func(%arg0: tensor<2x224x224x3xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<7x7x3x64xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg3: tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor<7x7x3x64xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense<0> : tensor<1x1xi32>} : () -> tensor<1x1xi32> + %2 = "tf.Const"() {value = dense<[7, 7, 3, 64]> : tensor<4xi32>} : () -> tensor<4xi32> + %3 = "tf.Const"() {value = dense<[[0, 0], [3, 3], [3, 3], [0, 0]]> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %4 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %5 = "tf.Pad"(%arg0, %3) : (tensor<2x224x224x3xf32>, tensor<4x2xi32>) -> tensor<2x230x230x3xf32> + // CHECK: "tf.Conv2D" + // CHECK-SAME: strides = [1, 1, 1, 1] + // CHECK-SAME: (tensor<2x115x115x12xf32>, tensor<4x4x12x64xf32>) -> tensor<2x112x112x64xf32> + %6 = "tf.Conv2D"(%5, %arg1) {data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true} : (tensor<2x230x230x3xf32>, tensor<7x7x3x64xf32>) -> tensor<2x112x112x64xf32> + // CHECK: %[[BACKPROP:.*]] = "tf.Conv2DBackpropFilter" + // CHECK-SAME: strides = [1, 1, 1, 1] + // CHECK-SAME: (tensor<2x115x115x12xf32>, tensor<4xi32>, tensor<2x112x112x64xf32>) -> tensor<4x4x12x64xf32> + %7 = "tf.Conv2DBackpropFilter"(%5, %2, %6) {data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true} : (tensor<2x230x230x3xf32>, tensor<4xi32>, tensor<2x112x112x64xf32>) -> tensor<7x7x3x64xf32> + // CHECK: %[[CONST0:.*]] = "tf.Const"() <{value = dense< + // CHECK-SAME: [4, 4, 2, 2, 3, 64] + // CHECK: %[[RESHAPE0:.*]] = "tf.Reshape"(%[[BACKPROP:.*]], %[[CONST0:.*]]) : (tensor<4x4x12x64xf32>, tensor<6xi64>) -> tensor<4x4x2x2x3x64xf32> + // CHECK: %[[CONST1:.*]] = "tf.Const"() <{value = dense< + // CHECK-SAME: [0, 2, 1, 3, 4, 5] + // CHECK: %[[TRANSPOSE0:.*]] = "tf.Transpose"(%[[RESHAPE0:.*]], %[[CONST1:.*]]) : (tensor<4x4x2x2x3x64xf32>, tensor<6xi32>) -> tensor<4x2x4x2x3x64xf32> + // CHECK: %[[CONST2:.*]] = "tf.Const"() <{value = dense< + // CHECK-SAME: [8, 8, 3, 64] + // CHECK: %[[RESHAPE1:.*]] = "tf.Reshape"(%[[TRANSPOSE1:.*]], %[[CONST2:.*]]) : (tensor<4x2x4x2x3x64xf32>, tensor<4xi64>) -> tensor<8x8x3x64xf32> + // CHECK: %[[CONST3:.*]] = "tf.Const"() <{value = dense< + // CHECK-SAME: [7, 7, 3, 64] + // CHECK: %[[CONST4:.*]] = "tf.Const"() <{value = dense< + // CHECK-SAME: 0 + // CHECK: %[[SLICE0:.*]] = "tf.Slice"(%[[RESHAPE1:.*]], %[[CONST4:.*]], %[[CONST3:.*]]) : (tensor<8x8x3x64xf32>, tensor<4xi64>, tensor<4xi32>) -> tensor<7x7x3x64xf32> + %8 = "tf.CrossReplicaSum"(%7, %1) : (tensor<7x7x3x64xf32>, tensor<1x1xi32>) -> tensor<7x7x3x64xf32> + %9 = "tf.Mul"(%arg2, %8) : (tensor, tensor<7x7x3x64xf32>) -> tensor<7x7x3x64xf32> + %10 = "tf.Sub"(%arg1, %9) : (tensor<7x7x3x64xf32>, tensor<7x7x3x64xf32>) -> tensor<7x7x3x64xf32> + %11 = "tf.AddV2"(%arg3, %0) : (tensor, tensor) -> tensor + func.return %10, %11 : tensor<7x7x3x64xf32>, tensor + } +} + +// ----- + +// Tests for space to depth host and device transform with replicate inputs. + +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:COMPOSITE:0" = {}, "/job:localhost/replica:0/task:0/device:CPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:1" = {}, "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0" = {}}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 458 : i32}} { + func.func @main(%arg0: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg1: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg2: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg3: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg4: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg5: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg6: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg7: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg8: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg9: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg10: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg11: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg12: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}) attributes {tf.entry_function = {control_outputs = "IteratorGetNext,IteratorGetNext_1,CrossReplicaSum,AssignAddVariableOp,CrossReplicaSum_1,AssignAddVariableOp_1,CrossReplicaSum_2,AssignAddVariableOp_2,CrossReplicaSum_3,AssignAddVariableOp_3", inputs = "iterator,iterator_1,iterator_2,iterator_3,iterator_4,iterator_5,resnet50_conv1_conv2d_conv1_kernel_140365606309224_handle_inputs_0,resnet50_fc1000_matmul_fc1000_kernel_140365944145960_handle_inputs_0,resnet50_fc1000_biasadd_fc1000_bias_140365944146240_handle_inputs_0,total_140366323758976_handle_inputs_0,count_140366323759312_handle_inputs_0,total_140366323760264_handle_inputs_0,count_140366323760600_handle_inputs_0", outputs = ""}} { + // CHECK: %[[INPUT00:.*]] = "tf.IteratorGetNext" + // CHECK-DAG: %[[SPACETODEPTH00:.*]] = "tf.SpaceToDepth"([[INPUT00:.*]]#0) <{block_size = 2 : i64, data_format = "NHWC"}> : (tensor<2x224x224x3xf32>) -> tensor<2x112x112x12xf32> + %0:2 = "tf.IteratorGetNext"(%arg2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<2x224x224x3xf32>, tensor<2x1xf32>) + // CHECK: %[[INPUT01:.*]] = "tf.IteratorGetNext" + // CHECK-DAG: %[[SPACETODEPTH01:.*]] = "tf.SpaceToDepth"([[INPUT01:.*]]#0) <{block_size = 2 : i64, data_format = "NHWC"}> : (tensor<2x224x224x3xf32>) -> tensor<2x112x112x12xf32> + %1:2 = "tf.IteratorGetNext"(%arg4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<2x224x224x3xf32>, tensor<2x1xf32>) + tf_device.replicate([%0#0, %1#0] as %arg13: tensor<2x224x224x3xf32>, [%0#1, %1#1] as %arg14: tensor<2x1xf32>, %arg6 as %arg15: tensor<*x!tf_type.resource>>, %arg8 as %arg16: tensor<*x!tf_type.resource>>, %arg7 as %arg17: tensor<*x!tf_type.resource>>, %arg9 as %arg18: tensor<*x!tf_type.resource>>, %arg10 as %arg19: tensor<*x!tf_type.resource>>, %arg11 as %arg20: tensor<*x!tf_type.resource>>, %arg12 as %arg21: tensor<*x!tf_type.resource>>) {_mirrored_variable_indices = [2, 3, 4, 5, 6, 7, 8], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + %2 = "tf.ReadVariableOp"(%arg15) : (tensor<*x!tf_type.resource>>) -> tensor<7x7x3x64xf32> + %3 = "tf.ReadVariableOp"(%arg16) : (tensor<*x!tf_type.resource>>) -> tensor<1001xf32> + %4 = "tf.ReadVariableOp"(%arg17) : (tensor<*x!tf_type.resource>>) -> tensor<64x1001xf32> + %5 = "tf.ReadVariableOp"(%arg18) : (tensor<*x!tf_type.resource>>) -> tensor + %6 = "tf.ReadVariableOp"(%arg19) : (tensor<*x!tf_type.resource>>) -> tensor + %7 = "tf.ReadVariableOp"(%arg20) : (tensor<*x!tf_type.resource>>) -> tensor + %8 = "tf.ReadVariableOp"(%arg21) : (tensor<*x!tf_type.resource>>) -> tensor + %9:4 = "tf_device.cluster_func"(%arg13, %arg14, %2, %4, %3, %5, %6, %7, %8) {_xla_compile_device_type = "TPU", _replication_info = "cluster_eval_step", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], func = @_func, host_compute_core = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], num_cores_per_replica = 1 : i64, output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor<2x224x224x3xf32>, tensor<2x1xf32>, tensor<7x7x3x64xf32>, tensor<64x1001xf32>, tensor<1001xf32>, tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor, tensor) + "tf.AssignVariableOp"(%arg18, %9#0) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg19, %9#1) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg20, %9#2) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg21, %9#3) : (tensor<*x!tf_type.resource>>, tensor) -> () + tf_device.return + } + func.return + } + // CHECK-LABEL: func private @_func + // CHECK-SAME: [[FUNCINPUT00:.*]]: tensor<2x112x112x12xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<2x1xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor<7x7x3x64xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg3: tensor<64x1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg4: tensor<1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg5: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg6: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg7: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg8: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + func.func private @_func(%arg0: tensor<2x224x224x3xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<2x1xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor<7x7x3x64xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg3: tensor<64x1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg4: tensor<1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg5: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg6: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg7: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg8: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %0 = "tf.Const"() {value = dense<2.000000e+00> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %3 = "tf.Const"() {value = dense<[[0, 1]]> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %4 = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %5 = "tf.Const"() {value = dense<2.500000e-01> : tensor} : () -> tensor + %6 = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %7 = "tf.Const"() {value = dense<[-1, 1001]> : tensor<2xi32>} : () -> tensor<2xi32> + %8 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %9 = "tf.Const"() {value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + %10 = "tf.Const"() {value = dense<[[0, 0], [3, 3], [3, 3], [0, 0]]> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %11 = "tf.Pad"(%arg0, %10) : (tensor<2x224x224x3xf32>, tensor<4x2xi32>) -> tensor<2x230x230x3xf32> + %12 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2x1xf32>) -> tensor<2x1xi64> + %13 = "tf.Reshape"(%12, %9) : (tensor<2x1xi64>, tensor<1xi32>) -> tensor<2xi64> + %14 = "tf.Squeeze"(%arg1) {squeeze_dims = [-1]} : (tensor<2x1xf32>) -> tensor<2xf32> + // CHECK: "tf.Conv2D" + // CHECK-SAME: strides = [1, 1, 1, 1] + // CHECK-SAME: (tensor<2x115x115x12xf32>, tensor<4x4x12x64xf32>) -> tensor<2x112x112x64xf32> + %15 = "tf.Conv2D"(%11, %arg2) {data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true} : (tensor<2x230x230x3xf32>, tensor<7x7x3x64xf32>) -> tensor<2x112x112x64xf32> + %16 = "tf.Mean"(%15, %8) {keep_dims = false} : (tensor<2x112x112x64xf32>, tensor<2xi32>) -> tensor<2x64xf32> + %17 = "tf.MatMul"(%16, %arg3) {transpose_a = false, transpose_b = false} : (tensor<2x64xf32>, tensor<64x1001xf32>) -> tensor<2x1001xf32> + %18 = "tf.BiasAdd"(%17, %arg4) {data_format = "NHWC"} : (tensor<2x1001xf32>, tensor<1001xf32>) -> tensor<2x1001xf32> + %19 = "tf.Reshape"(%18, %7) : (tensor<2x1001xf32>, tensor<2xi32>) -> tensor<2x1001xf32> + %loss, %backprop = "tf.SparseSoftmaxCrossEntropyWithLogits"(%19, %13) : (tensor<2x1001xf32>, tensor<2xi64>) -> (tensor<2xf32>, tensor<2x1001xf32>) + %20 = "tf.Sum"(%loss, %6) {keep_dims = false} : (tensor<2xf32>, tensor<1xi32>) -> tensor + %21 = "tf.Mul"(%20, %5) : (tensor, tensor) -> tensor + %22 = "tf.Sum"(%21, %4) {keep_dims = false} : (tensor, tensor<0xi32>) -> tensor + %23 = "tf.CrossReplicaSum"(%22, %3) : (tensor, tensor<1x2xi32>) -> tensor + %24 = "tf.Softmax"(%18) : (tensor<2x1001xf32>) -> tensor<2x1001xf32> + %25 = "tf.ArgMax"(%24, %2) : (tensor<2x1001xf32>, tensor) -> tensor<2xi64> + %26 = "tf.Cast"(%25) {Truncate = false} : (tensor<2xi64>) -> tensor<2xf32> + %27 = "tf.Equal"(%14, %26) {incompatible_shape_error = true} : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + %28 = "tf.Cast"(%27) {Truncate = false} : (tensor<2xi1>) -> tensor<2xf32> + %29 = "tf.Sum"(%28, %6) {keep_dims = false} : (tensor<2xf32>, tensor<1xi32>) -> tensor + %30 = "tf.CrossReplicaSum"(%29, %3) : (tensor, tensor<1x2xi32>) -> tensor + %31 = "tf.AddV2"(%arg5, %23) : (tensor, tensor) -> tensor + %32 = "tf.CrossReplicaSum"(%1, %3) : (tensor, tensor<1x2xi32>) -> tensor + %33 = "tf.AddV2"(%arg6, %32) : (tensor, tensor) -> tensor + %34 = "tf.AddV2"(%arg7, %30) : (tensor, tensor) -> tensor + %35 = "tf.CrossReplicaSum"(%0, %3) : (tensor, tensor<1x2xi32>) -> tensor + %36 = "tf.AddV2"(%arg8, %35) : (tensor, tensor) -> tensor + func.return %31, %33, %34, %36 : tensor, tensor, tensor, tensor + } +} + +// ----- + +// Tests for space to depth host and device transform with replicate packed inputs. + +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:COMPOSITE:0" = {}, "/job:localhost/replica:0/task:0/device:CPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:0" = {}, "/job:localhost/replica:0/task:0/device:TPU:1" = {}, "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0" = {}}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 458 : i32}} { + func.func @main(%arg0: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg1: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg2: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg3: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg4: tensor<*x!tf_type.resource> {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg5: tensor {tf._user_specified_name = "iterator", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}, %arg6: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg7: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg8: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg9: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg10: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg11: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}, %arg12: tensor<*x!tf_type.resource>> {tf._composite_device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0", tf.device = "/job:localhost/replica:0/task:0/device:COMPOSITE:0"}) attributes {tf.entry_function = {control_outputs = "IteratorGetNext,IteratorGetNext_1,CrossReplicaSum,AssignAddVariableOp,CrossReplicaSum_1,AssignAddVariableOp_1,CrossReplicaSum_2,AssignAddVariableOp_2,CrossReplicaSum_3,AssignAddVariableOp_3", inputs = "iterator,iterator_1,iterator_2,iterator_3,iterator_4,iterator_5,resnet50_conv1_conv2d_conv1_kernel_140365606309224_handle_inputs_0,resnet50_fc1000_matmul_fc1000_kernel_140365944145960_handle_inputs_0,resnet50_fc1000_biasadd_fc1000_bias_140365944146240_handle_inputs_0,total_140366323758976_handle_inputs_0,count_140366323759312_handle_inputs_0,total_140366323760264_handle_inputs_0,count_140366323760600_handle_inputs_0", outputs = ""}} { + // CHECK: %[[INPUT00:.*]] = "tf.IteratorGetNext" + // CHECK-DAG: %[[SPACETODEPTH00:.*]] = "tf.SpaceToDepth"([[INPUT00:.*]]#0) <{block_size = 2 : i64, data_format = "NHWC"}> : (tensor<2x224x224x3xf32>) -> tensor<2x112x112x12xf32> + %0:2 = "tf.IteratorGetNext"(%arg2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> (tensor<2x224x224x3xf32>, tensor<2x1xf32>) + tf_device.replicate(%0#0 as %arg13: tensor<2x224x224x3xf32>, %0#1 as %arg14: tensor<2x1xf32>, %arg6 as %arg15: tensor<*x!tf_type.resource>>, %arg8 as %arg16: tensor<*x!tf_type.resource>>, %arg7 as %arg17: tensor<*x!tf_type.resource>>, %arg9 as %arg18: tensor<*x!tf_type.resource>>, %arg10 as %arg19: tensor<*x!tf_type.resource>>, %arg11 as %arg20: tensor<*x!tf_type.resource>>, %arg12 as %arg21: tensor<*x!tf_type.resource>>) {_mirrored_variable_indices = [2, 3, 4, 5, 6, 7, 8], devices = {TPU_REPLICATED_CORE_0 = ["/device:TPU:0", "/device:TPU:1"]}, n = 2 : i32} { + %2 = "tf.ReadVariableOp"(%arg15) : (tensor<*x!tf_type.resource>>) -> tensor<7x7x3x64xf32> + %3 = "tf.ReadVariableOp"(%arg16) : (tensor<*x!tf_type.resource>>) -> tensor<1001xf32> + %4 = "tf.ReadVariableOp"(%arg17) : (tensor<*x!tf_type.resource>>) -> tensor<64x1001xf32> + %5 = "tf.ReadVariableOp"(%arg18) : (tensor<*x!tf_type.resource>>) -> tensor + %6 = "tf.ReadVariableOp"(%arg19) : (tensor<*x!tf_type.resource>>) -> tensor + %7 = "tf.ReadVariableOp"(%arg20) : (tensor<*x!tf_type.resource>>) -> tensor + %8 = "tf.ReadVariableOp"(%arg21) : (tensor<*x!tf_type.resource>>) -> tensor + %9:4 = "tf_device.cluster_func"(%arg13, %arg14, %2, %4, %3, %5, %6, %7, %8) {_xla_compile_device_type = "TPU", _replication_info = "cluster_eval_step", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], func = @_func, host_compute_core = [], input_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], num_cores_per_replica = 1 : i64, output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00", "\08\01\1A\01\01\22\01\00"], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor<2x224x224x3xf32>, tensor<2x1xf32>, tensor<7x7x3x64xf32>, tensor<64x1001xf32>, tensor<1001xf32>, tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor, tensor) + "tf.AssignVariableOp"(%arg18, %9#0) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg19, %9#1) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg20, %9#2) : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg21, %9#3) : (tensor<*x!tf_type.resource>>, tensor) -> () + tf_device.return + } + func.return + } + // CHECK-LABEL: func private @_func + // CHECK-SAME: [[FUNCINPUT00:.*]]: tensor<2x112x112x12xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<2x1xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor<7x7x3x64xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg3: tensor<64x1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg4: tensor<1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg5: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg6: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg7: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg8: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + func.func private @_func(%arg0: tensor<2x224x224x3xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg1: tensor<2x1xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg2: tensor<7x7x3x64xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg3: tensor<64x1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg4: tensor<1001xf32> {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg5: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg6: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg7: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, %arg8: tensor {mhlo.is_same_data_across_replicas = true, mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) -> (tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}, tensor {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %0 = "tf.Const"() {value = dense<2.000000e+00> : tensor} : () -> tensor + %1 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %2 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %3 = "tf.Const"() {value = dense<[[0, 1]]> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %4 = "tf.Const"() {value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %5 = "tf.Const"() {value = dense<2.500000e-01> : tensor} : () -> tensor + %6 = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %7 = "tf.Const"() {value = dense<[-1, 1001]> : tensor<2xi32>} : () -> tensor<2xi32> + %8 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi32>} : () -> tensor<2xi32> + %9 = "tf.Const"() {value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + %10 = "tf.Const"() {value = dense<[[0, 0], [3, 3], [3, 3], [0, 0]]> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + %11 = "tf.Pad"(%arg0, %10) : (tensor<2x224x224x3xf32>, tensor<4x2xi32>) -> tensor<2x230x230x3xf32> + %12 = "tf.Cast"(%arg1) {Truncate = false} : (tensor<2x1xf32>) -> tensor<2x1xi64> + %13 = "tf.Reshape"(%12, %9) : (tensor<2x1xi64>, tensor<1xi32>) -> tensor<2xi64> + %14 = "tf.Squeeze"(%arg1) {squeeze_dims = [-1]} : (tensor<2x1xf32>) -> tensor<2xf32> + // CHECK: "tf.Conv2D" + // CHECK-SAME: strides = [1, 1, 1, 1] + // CHECK-SAME: (tensor<2x115x115x12xf32>, tensor<4x4x12x64xf32>) -> tensor<2x112x112x64xf32> + %15 = "tf.Conv2D"(%11, %arg2) {data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "VALID", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true} : (tensor<2x230x230x3xf32>, tensor<7x7x3x64xf32>) -> tensor<2x112x112x64xf32> + %16 = "tf.Mean"(%15, %8) {keep_dims = false} : (tensor<2x112x112x64xf32>, tensor<2xi32>) -> tensor<2x64xf32> + %17 = "tf.MatMul"(%16, %arg3) {transpose_a = false, transpose_b = false} : (tensor<2x64xf32>, tensor<64x1001xf32>) -> tensor<2x1001xf32> + %18 = "tf.BiasAdd"(%17, %arg4) {data_format = "NHWC"} : (tensor<2x1001xf32>, tensor<1001xf32>) -> tensor<2x1001xf32> + %19 = "tf.Reshape"(%18, %7) : (tensor<2x1001xf32>, tensor<2xi32>) -> tensor<2x1001xf32> + %loss, %backprop = "tf.SparseSoftmaxCrossEntropyWithLogits"(%19, %13) : (tensor<2x1001xf32>, tensor<2xi64>) -> (tensor<2xf32>, tensor<2x1001xf32>) + %20 = "tf.Sum"(%loss, %6) {keep_dims = false} : (tensor<2xf32>, tensor<1xi32>) -> tensor + %21 = "tf.Mul"(%20, %5) : (tensor, tensor) -> tensor + %22 = "tf.Sum"(%21, %4) {keep_dims = false} : (tensor, tensor<0xi32>) -> tensor + %23 = "tf.CrossReplicaSum"(%22, %3) : (tensor, tensor<1x2xi32>) -> tensor + %24 = "tf.Softmax"(%18) : (tensor<2x1001xf32>) -> tensor<2x1001xf32> + %25 = "tf.ArgMax"(%24, %2) : (tensor<2x1001xf32>, tensor) -> tensor<2xi64> + %26 = "tf.Cast"(%25) {Truncate = false} : (tensor<2xi64>) -> tensor<2xf32> + %27 = "tf.Equal"(%14, %26) {incompatible_shape_error = true} : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + %28 = "tf.Cast"(%27) {Truncate = false} : (tensor<2xi1>) -> tensor<2xf32> + %29 = "tf.Sum"(%28, %6) {keep_dims = false} : (tensor<2xf32>, tensor<1xi32>) -> tensor + %30 = "tf.CrossReplicaSum"(%29, %3) : (tensor, tensor<1x2xi32>) -> tensor + %31 = "tf.AddV2"(%arg5, %23) : (tensor, tensor) -> tensor + %32 = "tf.CrossReplicaSum"(%1, %3) : (tensor, tensor<1x2xi32>) -> tensor + %33 = "tf.AddV2"(%arg6, %32) : (tensor, tensor) -> tensor + %34 = "tf.AddV2"(%arg7, %30) : (tensor, tensor) -> tensor + %35 = "tf.CrossReplicaSum"(%0, %3) : (tensor, tensor<1x2xi32>) -> tensor + %36 = "tf.AddV2"(%arg8, %35) : (tensor, tensor) -> tensor + func.return %31, %33, %34, %36 : tensor, tensor, tensor, tensor + } +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_tail_with_tobool_op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_tail_with_tobool_op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3aac1fa92f4199924b109067049f404912565397 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_tail_with_tobool_op.mlir @@ -0,0 +1,35 @@ +// RUN: tf-opt %s -tf-replicated-clustering-bridge-v2 -tfrt-lower-cluster-to-runtime-ops-tpu 2>&1 | FileCheck %s + +// This test verifies that the tail extraction is not terminated prematurely +// in handling tf.If op which would end up with excessive host-device +// communication. + +// In this test, all ops other than tf.Rank are marked with outside_compilation +// . So the TPU program should contain tf.Rank op and there should be no +// host-device communication. +// CHECK: tf._TPUCompileMlir +// CHECK-SAME: tf.Rank +// CHECK-NOT: tf._XlaHostComputeMlir +// CHECK-NOT: tf._XlaRecvAtHost +// CHECK-NOT: tf._XlaSendFromHost + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 741 : i32}} { + func.func private @if_then_branch(%arg0: tensor<*xi1>, %arg1: tensor<*xi64>, %arg2: tensor<*xi64>) -> tensor<*xi1> { + func.return %arg0 : tensor<*xi1> + } + func.func private @if_else_branch(%arg0: tensor<*xi1>, %arg1: tensor<*xi64>, %arg2: tensor<*xi64>) -> tensor<*xi1> { + func.return %arg0 : tensor<*xi1> + } + func.func @"tpu_subgraph"(%arg0: tensor<*xi64>) -> tensor<*xi1> { + %cst = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %cst_1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : () -> () + %0 = "tf.Equal"(%arg0, %cst) {_tpu_replicate = "cluster", device = "", incompatible_shape_error = true} : (tensor<*xi64>, tensor) -> tensor<*xi1> + %1 = "tf.Rank"(%0) {_tpu_replicate = "cluster", device = ""} : (tensor<*xi1>) -> tensor<*xi32> + %2 = "tf.Range"(%cst_0, %1, %cst_1) {_tpu_replicate = "cluster", _xla_outside_compilation = "0", device = ""} : (tensor, tensor<*xi32>, tensor) -> tensor<*xi32> + %3 = "tf.All"(%0, %2) {_tpu_replicate = "cluster", _xla_outside_compilation = "0", device = "", keep_dims = false} : (tensor<*xi1>, tensor<*xi32>) -> tensor<*xi1> + %4 = "tf.If"(%3, %3, %arg0, %cst) {_tpu_replicate = "cluster", _xla_outside_compilation = "0", device = "", else_branch = @if_else_branch, is_stateless = false, then_branch = @if_then_branch} : (tensor<*xi1>, tensor<*xi1>, tensor<*xi64>, tensor) -> tensor<*xi1> + func.return %4 : tensor<*xi1> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_update_embedding_enqueue_op_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_update_embedding_enqueue_op_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e19776369bb49151ae383d891a7397681c947d4a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/tpu_update_embedding_enqueue_op_inputs.mlir @@ -0,0 +1,65 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-update-embedding-enqueue-op-inputs | FileCheck %s + +// CHECK-LABEL: func @check_enqueue_ops_update_for_eval +// CHECK-SAME: %[[ARG_0:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_1:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_2:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_3:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_4:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_5:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_6:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_7:[a-z0-9]*]]: tensor +func.func @check_enqueue_ops_update_for_eval(%arg0: tensor, %arg1: tensor, + %arg2 :tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, + %arg6: tensor, %arg7: tensor) -> () { + // CHECK: %[[CONST_0:.*]] = "tf.Const"() + %0 = "tf.Const"() {value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + + // CHECK: %[[CONST_MODE:.*]] = "tf.Const"() <{value = dense<"inference"> : tensor}> {_xla_outside_compilation = "0"} : () -> tensor + // CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%[[ARG_0]], %[[ARG_1]], %[[ARG_2]], %[[ARG_3]], %[[ARG_4]], %[[ARG_5]], %[[CONST_0]], %[[CONST_0]], %[[CONST_0]], %[[CONST_MODE]]) + "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %0, %0, %0, %arg7) {_tpu_embedding_layer = "call1", _xla_outside_compilation = "0", combiners = ["mean", "sum"], device_ordinal = -1 : i64, max_sequence_lengths = [0, 0, 0], table_ids = [1, 1, 0]} : (tensor, tensor, tensor, tensor, tensor, tensor, tensor<0xf32>, tensor<0xf32>, tensor<0xf32>, tensor) -> () + %2:2 = "tf.RecvTPUEmbeddingActivations"() {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D"} : () -> (tensor<2x2xf32>, tensor<4x4xf32>) + func.return +} + +// ----- + +// CHECK-LABEL: func @check_enqueue_ops_update_for_training +// CHECK-SAME: %[[ARG_0:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_1:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_2:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_3:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_4:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_5:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_6:[a-z0-9]*]]: tensor +// CHECK-SAME: %[[ARG_7:[a-z0-9]*]]: tensor +func.func @check_enqueue_ops_update_for_training(%arg0: tensor, %arg1: tensor, + %arg2 :tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, + %arg6: tensor, %arg7: tensor) -> () { + // CHECK: %[[CONST_0:.*]] = "tf.Const"() + %0 = "tf.Const"() {value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + + %2 = "tf.Const"() {value = dense<0.0> : tensor<2x2xf32>} : () -> tensor<2x2xf32> + %3 = "tf.Const"() {value = dense<0.0> : tensor<4x4xf32>} : () -> tensor<4x4xf32> + "tf.SendTPUEmbeddingGradients"(%2, %3) {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D", operandSegmentSizes = array} : (tensor<2x2xf32>, tensor<4x4xf32>) -> () + + // CHECK: %[[CONST_MODE:.*]] = "tf.Const"() <{value = dense<"train"> : tensor}> {_xla_outside_compilation = "0"} : () -> tensor + // CHECK: "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%[[ARG_0]], %[[ARG_1]], %[[ARG_2]], %[[ARG_3]], %[[ARG_4]], %[[ARG_5]], %[[CONST_0]], %[[CONST_0]], %[[CONST_0]], %[[CONST_MODE]]) + "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %0, %0, %0, %arg7) {_tpu_embedding_layer = "call1", _xla_outside_compilation = "0", combiners = ["mean", "sum"], device_ordinal = -1 : i64, max_sequence_lengths = [0, 0, 0], table_ids = [1, 1, 0]} : (tensor, tensor, tensor, tensor, tensor, tensor, tensor<0xf32>, tensor<0xf32>, tensor<0xf32>, tensor) -> () + %4:2 = "tf.RecvTPUEmbeddingActivations"() {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D"} : () -> (tensor<2x2xf32>, tensor<4x4xf32>) + func.return +} + +// ----- + +func.func @check_enqueue_ops_with_different_attr_disallowed(%arg0: tensor, %arg1: tensor, + %arg2 :tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor, + %arg6: tensor, %arg7: tensor, %arg8: tensor) -> () { + %0 = "tf.Const"() {value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + %1 = "tf.SelectV2"(%arg8, %arg6, %arg7) : (tensor, tensor, tensor) -> tensor + // expected-error @+1 {{'tf.EnqueueTPUEmbeddingSparseTensorBatch' op must have a corresponding 'tf.RecvTPUEmbeddingActivations' op}} + "tf.EnqueueTPUEmbeddingSparseTensorBatch"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %0, %0, %0, %1) {_tpu_embedding_layer = "call_123", _xla_outside_compilation = "0", combiners = ["mean", "sum"], device_ordinal = -1 : i64, max_sequence_lengths = [0, 0, 0], table_ids = [1, 1, 0]} : (tensor, tensor, tensor, tensor, tensor, tensor, tensor<0xf32>, tensor<0xf32>, tensor<0xf32>, tensor) -> () + %2:2 = "tf.RecvTPUEmbeddingActivations"() {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D"} : () -> (tensor<2x2xf32>, tensor<4x4xf32>) + func.return +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/transpose-op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/transpose-op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a3091d2a757ec1337c7f9491a142c67c1b87f7a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/transpose-op.mlir @@ -0,0 +1,10 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics + +func.func @out_of_bounds_check(%arg0: tensor<1x4x4x8xf32>) -> tensor<1x4x4x8xf32> { + %0 = "tf.Const"() {value = dense<[0, 3, 1, 2]> : tensor<4xi32>} : () -> tensor<4xi32> + %1 = "tf.Const"() {value = dense<[0, 0x4141, 3, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %2 = "tf.Transpose"(%arg0, %0) : (tensor<1x4x4x8xf32>, tensor<4xi32>) -> tensor<1x8x4x4xf32> + // expected-error @+1 {{'tf.Transpose' op perm[1]=16705 must be in range [-4, 4)}} + %3 = "tf.Transpose"(%2, %1) : (tensor<1x8x4x4xf32>, tensor<4xi32>) -> tensor<1x4x4x8xf32> + func.return %3 : tensor<1x4x4x8xf32> +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/unroll-batch-matmul.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/unroll-batch-matmul.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6a93b9a34b34ef3a5c0d35238f5f42685451eb80 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/unroll-batch-matmul.mlir @@ -0,0 +1,714 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -tf-unroll-batch-matmul %s | FileCheck %s + +//==== V1 tests ==== + +func.func @batchMatMulTwoDim(%arg0: tensor<2x3x4x5xf32>, %arg1: tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<2x3x4x5xf32>, tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulTwoDim + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 4, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 6]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x4x5xf32>, tensor<3xi64>) -> tensor<6x4x5xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x6xf32>, tensor<3xi64>) -> tensor<6x5x6xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulTwoDimAdjXY(%arg0: tensor<2x3x5x4xf32>, %arg1: tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<2x3x5x4xf32>, tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulTwoDimAdjXY + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 4]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 6, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 4]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x4xf32>, tensor<3xi64>) -> tensor<6x5x4xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x5x4xf32>) -> (tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x6x5xf32>, tensor<3xi64>) -> tensor<6x6x5xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x6x5xf32>) -> (tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulOneDim(%arg0: tensor<3x4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulOneDim + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulSingleBatch(%arg0: tensor<1x4x5xf32>, %arg1: tensor<1x5x6xf32>) -> tensor<1x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<1x4x5xf32>, tensor<1x5x6xf32>) -> tensor<1x4x6xf32> + func.return %0 : tensor<1x4x6xf32> + + // CHECK-LABEL: batchMatMulSingleBatch + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%arg0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%arg1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]]) <{axis = 0 : i64}> : (tensor<4x6xf32>) -> tensor<1x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<1x4x6xf32> +} + +// ----- + +func.func @batchMatMulUnbatchedLeft(%arg0: tensor<4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulUnbatchedLeft + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%arg0, %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%arg0, %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulUnbatchedRight(%arg0: tensor<3x4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulUnbatchedRight + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_SPLIT:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulMatrix(%arg0: tensor<4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulMatrix + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- + +func.func @batchMatMulMatrixAdjXY(%arg0: tensor<5x4xf32>, %arg1: tensor<6x5xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMul"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulMatrixAdjXY + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- +// ==== V2 tests ==== + +func.func @batchMatMulV2TwoDim(%arg0: tensor<2x3x4x5xf32>, %arg1: tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<2x3x4x5xf32>, tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2TwoDim + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 4, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 6]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x4x5xf32>, tensor<3xi64>) -> tensor<6x4x5xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x6xf32>, tensor<3xi64>) -> tensor<6x5x6xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2TwoDimAdjXY(%arg0: tensor<2x3x5x4xf32>, %arg1: tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<2x3x5x4xf32>, tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2TwoDimAdjXY + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 4]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 6, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 4]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x4xf32>, tensor<3xi64>) -> tensor<6x5x4xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x5x4xf32>) -> (tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x6x5xf32>, tensor<3xi64>) -> tensor<6x6x5xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x6x5xf32>) -> (tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2Broadcast(%arg0: tensor<2x1x4x5xf32>, %arg1: tensor<1x3x5x6xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<2x1x4x5xf32>, tensor<1x3x5x6xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2Broadcast + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 4, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[3, 5, 6]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x1x4x5xf32>, tensor<3xi64>) -> tensor<2x4x5xf32> + // CHECK: %[[LHS_SPLIT:.*]]:2 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<2x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<1x3x5x6xf32>, tensor<3xi64>) -> tensor<3x5x6xf32> + // CHECK: %[[RHS_SPLIT:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2OneDim(%arg0: tensor<3x4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2OneDim + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2SingleBatch(%arg0: tensor<1x4x5xf32>, %arg1: tensor<1x5x6xf32>) -> tensor<1x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<1x4x5xf32>, tensor<1x5x6xf32>) -> tensor<1x4x6xf32> + func.return %0 : tensor<1x4x6xf32> + + // CHECK-LABEL: batchMatMulV2SingleBatch + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%arg0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%arg1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]]) <{axis = 0 : i64}> : (tensor<4x6xf32>) -> tensor<1x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<1x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2UnbatchedLeft(%arg0: tensor<4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2UnbatchedLeft + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%arg0, %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%arg0, %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2UnbatchedRight(%arg0: tensor<3x4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV2UnbatchedRight + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_SPLIT:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV2Matrix(%arg0: tensor<4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulV2Matrix + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- + +func.func @batchMatMulV2MatrixAdjXY(%arg0: tensor<5x4xf32>, %arg1: tensor<6x5xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulV2MatrixAdjXY + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- + +func.func @batchMatMulV2DynamicSize(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {adj_x = false, adj_y = false} : (tensor, tensor) -> tensor + func.return %0 : tensor + + // CHECK-LABEL: batchMatMulV2DynamicSize + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor, tensor) -> tensor + // CHECK: return %[[MATMUL_1]] : tensor +} + +// ----- +// ==== V3 tests ==== + +func.func @batchMatMulV3TwoDim(%arg0: tensor<2x3x4x5xf32>, %arg1: tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<2x3x4x5xf32>, tensor<2x3x5x6xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3TwoDim + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 4, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 6]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x4x5xf32>, tensor<3xi64>) -> tensor<6x4x5xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x6xf32>, tensor<3xi64>) -> tensor<6x5x6xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3TwoDimAdjXY(%arg0: tensor<2x3x5x4xf32>, %arg1: tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<2x3x5x4xf32>, tensor<2x3x6x5xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3TwoDimAdjXY + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5, 4]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 6, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 4]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[6, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x3x5x4xf32>, tensor<3xi64>) -> tensor<6x5x4xf32> + // CHECK: %[[LHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<6x5x4xf32>) -> (tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>, tensor<1x5x4xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_4:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#3, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_5:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#4, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + // CHECK: %[[LHS_6:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#5, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x5x4xf32>, tensor<2xi64>) -> tensor<5x4xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<2x3x6x5xf32>, tensor<3xi64>) -> tensor<6x6x5xf32> + // CHECK: %[[RHS_SPLIT:.*]]:6 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<6x6x5xf32>) -> (tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>, tensor<1x6x5xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_4:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#3, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_5:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#4, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + // CHECK: %[[RHS_6:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#5, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x6x5xf32>, tensor<2xi64>) -> tensor<6x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_4]], %[[RHS_4]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_5]], %[[RHS_5]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_6]], %[[RHS_6]]) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3Broadcast(%arg0: tensor<2x1x4x5xf32>, %arg1: tensor<1x3x5x6xf32>) -> tensor<2x3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<2x1x4x5xf32>, tensor<1x3x5x6xf32>) -> tensor<2x3x4x6xf32> + func.return %0 : tensor<2x3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3Broadcast + // CHECK-DAG: %[[LHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 4, 5]> : tensor<3xi64>}> + // CHECK-DAG: %[[RHS_RESHAPED_SHAPE:.*]] = "tf.Const"() <{value = dense<[3, 5, 6]> : tensor<3xi64>}> + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + // CHECK-DAG: %[[RESULT_SHAPE:.*]] = "tf.Const"() <{value = dense<[2, 3, 4, 6]> : tensor<4xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]] = "tf.Reshape"(%arg0, %[[LHS_RESHAPED_SHAPE]]) : (tensor<2x1x4x5xf32>, tensor<3xi64>) -> tensor<2x4x5xf32> + // CHECK: %[[LHS_SPLIT:.*]]:2 = "tf.Split"(%[[SPLITTING_AXIS]], %[[LHS_RESHAPED]]) : (tensor, tensor<2x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]] = "tf.Reshape"(%arg1, %[[RHS_RESHAPED_SHAPE]]) : (tensor<1x3x5x6xf32>, tensor<3xi64>) -> tensor<3x5x6xf32> + // CHECK: %[[RHS_SPLIT:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %[[RHS_RESHAPED]]) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_SPLIT]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_4:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_5:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_6:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]], %[[MATMUL_4]], %[[MATMUL_5]], %[[MATMUL_6]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<6x4x6xf32> + // CHECK: %[[RESULT:.*]] = "tf.Reshape"(%[[MATMUL_PACKED]], %[[RESULT_SHAPE]]) : (tensor<6x4x6xf32>, tensor<4xi64>) -> tensor<2x3x4x6xf32> + // CHECK: return %[[RESULT]] : tensor<2x3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3OneDim(%arg0: tensor<3x4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3OneDim + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[LHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_RESHAPED]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3SingleBatch(%arg0: tensor<1x4x5xf32>, %arg1: tensor<1x5x6xf32>) -> tensor<1x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<1x4x5xf32>, tensor<1x5x6xf32>) -> tensor<1x4x6xf32> + func.return %0 : tensor<1x4x6xf32> + + // CHECK-LABEL: batchMatMulV3SingleBatch + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%arg0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%arg1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]]) <{axis = 0 : i64}> : (tensor<4x6xf32>) -> tensor<1x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<1x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3UnbatchedLeft(%arg0: tensor<4x5xf32>, %arg1: tensor<3x5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<3x5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3UnbatchedLeft + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> + // CHECK-DAG: %[[MATMUL_RHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[5, 6]> : tensor<2xi64>}> + + // CHECK: %[[RHS_RESHAPED:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg1) : (tensor, tensor<3x5x6xf32>) -> (tensor<1x5x6xf32>, tensor<1x5x6xf32>, tensor<1x5x6xf32>) + // CHECK: %[[RHS_1:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#0, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_2:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#1, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + // CHECK: %[[RHS_3:.*]] = "tf.Reshape"(%[[RHS_RESHAPED]]#2, %[[MATMUL_RHS_SHAPE]]) : (tensor<1x5x6xf32>, tensor<2xi64>) -> tensor<5x6xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %[[RHS_1]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%arg0, %[[RHS_2]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%arg0, %[[RHS_3]]) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3UnbatchedRight(%arg0: tensor<3x4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<3x4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<5x6xf32>) -> tensor<3x4x6xf32> + func.return %0 : tensor<3x4x6xf32> + + // CHECK-LABEL: batchMatMulV3UnbatchedRight + // CHECK-DAG: %[[SPLITTING_AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + // CHECK-DAG: %[[MATMUL_LHS_SHAPE:.*]] = "tf.Const"() <{value = dense<[4, 5]> : tensor<2xi64>}> : () -> tensor<2xi64> + + // CHECK: %[[LHS_SPLIT:.*]]:3 = "tf.Split"(%[[SPLITTING_AXIS]], %arg0) : (tensor, tensor<3x4x5xf32>) -> (tensor<1x4x5xf32>, tensor<1x4x5xf32>, tensor<1x4x5xf32>) + // CHECK: %[[LHS_1:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#0, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_2:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#1, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + // CHECK: %[[LHS_3:.*]] = "tf.Reshape"(%[[LHS_SPLIT]]#2, %[[MATMUL_LHS_SHAPE]]) : (tensor<1x4x5xf32>, tensor<2xi64>) -> tensor<4x5xf32> + + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%[[LHS_1]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_2:.*]] = "tf.MatMul"(%[[LHS_2]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: %[[MATMUL_3:.*]] = "tf.MatMul"(%[[LHS_3]], %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + + // CHECK: %[[MATMUL_PACKED:.*]] = "tf.Pack"(%[[MATMUL_1]], %[[MATMUL_2]], %[[MATMUL_3]]) <{axis = 0 : i64}> : (tensor<4x6xf32>, tensor<4x6xf32>, tensor<4x6xf32>) -> tensor<3x4x6xf32> + // CHECK: return %[[MATMUL_PACKED]] : tensor<3x4x6xf32> +} + +// ----- + +func.func @batchMatMulV3Matrix(%arg0: tensor<4x5xf32>, %arg1: tensor<5x6xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulV3Matrix + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = false, transpose_b = false}> : (tensor<4x5xf32>, tensor<5x6xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- + +func.func @batchMatMulV3MatrixAdjXY(%arg0: tensor<5x4xf32>, %arg1: tensor<6x5xf32>) -> tensor<4x6xf32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) {adj_x = true, adj_y = true} : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + func.return %0 : tensor<4x6xf32> + + // CHECK-LABEL: batchMatMulV3MatrixAdjXY + // CHECK: %[[MATMUL_1:.*]] = "tf.MatMul"(%arg0, %arg1) <{grad_a = false, grad_b = false, transpose_a = true, transpose_b = true}> : (tensor<5x4xf32>, tensor<6x5xf32>) -> tensor<4x6xf32> + // CHECK: return %[[MATMUL_1]] : tensor<4x6xf32> +} + +// ----- + +func.func @batchMatMulV3MatrixInt8(%arg0: tensor<4x5xi8>, %arg1: tensor<5x6xi8>) -> tensor<4x6xi32> { + %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<4x5xi8>, tensor<5x6xi8>) -> tensor<4x6xi32> + func.return %0 : tensor<4x6xi32> + + // CHECK-LABEL: batchMatMulV3MatrixInt8 + // CHECK: %0 = "tf.BatchMatMulV3"(%arg0, %arg1) : (tensor<4x5xi8>, tensor<5x6xi8>) -> tensor<4x6xi32> + // CHECK: return %0 : tensor<4x6xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/update_control_dependencies.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/update_control_dependencies.mlir new file mode 100644 index 0000000000000000000000000000000000000000..492bc5ca7eb68a78952b876fc4c8a0f38e602df7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/update_control_dependencies.mlir @@ -0,0 +1,427 @@ +// RUN: tf-opt -tf-executor-update-control-dependencies %s --split-input-file --verify-diagnostics | FileCheck %s + +// Test that functions must have a single graph op in their body, otherwise, the +// pass will signal failure. + +// expected-error@+1 {{functions must be of a single Graph with single op Islands: function does not only contain a single tf_executor.graph}} +func.func @multiple_func_body_ops() { + tf_executor.graph { + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + tf_executor.graph { + tf_executor.island { + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Test that functions' graph op must have only island ops wrapping a single op +// and a single fetch op, otherwise, the pass will signal failure. + +func.func @graph_multi_op_island() { + tf_executor.graph { + // expected-error@+1 {{functions must be of a single Graph with single op Islands: tf_executor.island must perfectly wrap a single op}} + tf_executor.island { + "tf.NoOp"() : () -> () + "tf.NoOp"() : () -> () + tf_executor.yield + } + tf_executor.fetch + } + func.return +} + +// ----- + +// Test that external functions aren't supported as they're not deemed +// "export suitable". + +// expected-error@+1 {{functions must be of a single Graph with single op Islands: only single block functions are supported}} +func.func private @unused_external_func() + +// ----- + +func.func @multiple_return_no_controls_needed(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %add1, %add1_control = tf_executor.island wraps "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2, %add2_control = tf_executor.island wraps "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + tf_executor.fetch %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} +// CHECK-LABEL: func @multiple_return_no_controls_needed +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 +// CHECK: } + +func.func @incorrect_control_deps_replaced_with_correct_controls(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %add1, %add1_control = tf_executor.island wraps "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2, %add2_control = tf_executor.island(%add1_control) wraps "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %print, %print_control = tf_executor.island wraps "tf.Print"(%add2) {message = "add2 result"} : (tensor<*xi32>) -> tensor<*xi32> + tf_executor.fetch %add1, %add2, %add2_control : tensor<*xi32>, tensor<*xi32>, !tf_executor.control + } + return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} +// CHECK-LABEL: func @incorrect_control_deps_replaced_with_correct_controls +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: %[[PRINT:.*]], %[[PRINT_control:.*]] = tf_executor.island wraps "tf.Print"(%[[ADD2]]) <{message = "add2 result"}> +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]], %[[PRINT_control]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 +// CHECK: } + +func.func @trailing_print(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %add1, %add1_control = tf_executor.island wraps "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2, %add2_control = tf_executor.island wraps "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %print, %print_control = tf_executor.island wraps "tf.Print"(%add2) { message = "add2 result" } : (tensor<*xi32>) -> (tensor<*xi32>) + tf_executor.fetch %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> +} +// CHECK-LABEL: func @trailing_print +// CHECK: %[[GRAPH:.*]]:2 = tf_executor.graph { +// CHECK: %[[ADD1:.*]], %[[ADD1_control:.*]] = tf_executor.island wraps "tf.Add"(%arg0, %arg1) +// CHECK: %[[ADD2:.*]], %[[ADD2_control:.*]] = tf_executor.island wraps "tf.Add"(%[[ADD1]], %arg1) +// CHECK: %[[PRINT:.*]], %[[PRINT_control:.*]] = tf_executor.island wraps "tf.Print"(%[[ADD2]]) <{message = "add2 result"}> +// CHECK: tf_executor.fetch %[[ADD1]], %[[ADD2]], %[[PRINT_control]] : +// CHECK: } +// CHECK: return %[[GRAPH]]#0, %[[GRAPH]]#1 +// CHECK: } + +func.func @non_aliasing_reads_writes( + %arg0: tensor<*x!tf_type.resource>>, + %arg1: tensor<*x!tf_type.resource>>, + %arg2: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + %read0, %read0_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %assign0_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read1, %read1_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %vh0, %vh0_control = tf_executor.island wraps "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + %read2, %read2_control = tf_executor.island wraps "tf.ReadVariableOp"(%vh0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %assign1_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg1, %read0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %assign2_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %read2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read3, %read3_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + tf_executor.fetch %read3#0 : tensor<32xf32> + } + func.return %graph : tensor<32xf32> +} +// CHECK-LABEL: func @non_aliasing_reads_writes +// CHECK: %[[GRAPH:.*]] = tf_executor.graph { +// CHECK-DAG: %[[READ0:.*]], %[[READ0_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) +// CHECK-DAG: %[[ASSIGN0_CONTROL:.*]] = tf_executor.island(%[[READ0_CONTROL]]) wraps "tf.AssignVariableOp"(%arg0, %arg2) +// CHECK-DAG: %[[READ1:.*]], %[[READ1_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) +// CHECK: %[[VH0:.*]], %[[VH0_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v0"}> +// CHECK: %[[READ2:.*]], %[[READ2_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%[[VH0]]) +// CHECK: %[[ASSIGN1_CONTROL:.*]] = tf_executor.island(%[[READ1_CONTROL]]) wraps "tf.AssignVariableOp"(%arg1, %[[READ0]]) +// CHECK: %[[ASSIGN2_CONTROL:.*]] = tf_executor.island(%[[ASSIGN0_CONTROL]]) wraps "tf.AssignVariableOp"(%arg0, %[[READ2]]) +// CHECK: %[[READ3:.*]], %[[READ3_CONTROL:.*]] = tf_executor.island(%[[ASSIGN2_CONTROL]]) wraps "tf.ReadVariableOp"(%arg0) +// CHECK: tf_executor.fetch %[[READ3]] +// CHECK: } + +func.func @unknown_side_effecting_op(%arg0: tensor<32xf32>) { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> tensor<*x!tf_type.resource>> + %outputs_0, %control_1 = tf_executor.island wraps "tf.VarHandleOp"() {container = "c", shared_name = "v1"} : () -> tensor<*x!tf_type.resource>> + %outputs_2, %control_3 = tf_executor.island wraps "tf.ReadVariableOp"(%outputs) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %control_4 = tf_executor.island wraps "tf.AssignVariableOp"(%outputs_0, %arg0) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %control_5 = tf_executor.island wraps "tf._UnknownSideEffectingOp_"() : () -> () + %outputs_6, %control_7 = tf_executor.island wraps "tf.ReadVariableOp"(%outputs_0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %control_8 = tf_executor.island wraps "tf.AssignVariableOp"(%outputs, %outputs_6) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %control_9 = tf_executor.island wraps "tf.AssignVariableOp"(%outputs_0, %outputs_2) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + tf_executor.fetch + } + return +} +// CHECK-LABEL: func @unknown_side_effecting_op +// CHECK: tf_executor.graph { +// CHECK: %[[VH0:.*]], %[[VH0_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v0"}> +// CHECK: %[[VH1:.*]], %[[VH1_CONTROL:.*]] = tf_executor.island wraps "tf.VarHandleOp"() <{container = "c", shared_name = "v1"}> +// CHECK: %[[READ0:.*]], %[[READ0_CONTROL:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%[[VH0]]) +// CHECK: %[[ASSIGN0_CONTROL:.*]] = tf_executor.island wraps "tf.AssignVariableOp"(%[[VH1]], %arg0) +// CHECK: %[[UNKNOWN_CONTROL:.*]] = tf_executor.island(%[[READ0_CONTROL]], %[[ASSIGN0_CONTROL]]) wraps "tf._UnknownSideEffectingOp_"() +// CHECK: %[[READ1:.*]], %[[READ1_CONTROL:.*]] = tf_executor.island(%[[UNKNOWN_CONTROL]]) wraps "tf.ReadVariableOp"(%[[VH1]]) +// CHECK: %[[ASSIGN1_CONTROL:.*]] = tf_executor.island(%[[UNKNOWN_CONTROL]]) wraps "tf.AssignVariableOp"(%[[VH0]], %[[READ1]]) +// CHECK: %[[ASSIGN2_CONTROL:.*]] = tf_executor.island(%[[READ1_CONTROL]]) wraps "tf.AssignVariableOp"(%[[VH1]], %[[READ0]]) +// CHECK: tf_executor.fetch + +func.func @single_op_island_forward_block_arg(%arg0: tensor) -> (tensor<2048xf32>, tensor) { + %0:2 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {value = dense<0.000000e+00> : tensor<2048xf32>} : () -> tensor<2048xf32> + tf_executor.fetch %outputs, %arg0 : tensor<2048xf32>, tensor + } + return %0#0, %0#1 : tensor<2048xf32>, tensor +} +// CHECK-LABEL: func @single_op_island_forward_block_arg +// CHECK: tf_executor.graph { +// CHECK: %[[outputs:.*]], %[[control:.*]] = tf_executor.island wraps "tf.Const"() <{value = dense<0.000000e+00> : tensor<2048xf32>}> : () -> tensor<2048xf32> +// CHECK: tf_executor.fetch %[[outputs]], %arg0 : tensor<2048xf32>, tensor + +func.func @tpu_load_embedding_ops_sink_controls(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<*x!tf_type.resource>>, %arg2: tensor<*x!tf_type.resource>>, %arg3: tensor<*x!tf_type.resource>>) { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %control_4 = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%outputs, %outputs_0) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table1"} : (tensor<8xf32>, tensor<8xf32>) -> () + %outputs_5, %control_6 = tf_executor.island wraps "tf.ReadVariableOp"(%arg3) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> + %control_7 = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%outputs_2, %outputs_5) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table2"} : (tensor<8xf32>, tensor<8xf32>) -> () + %control_8 = tf_executor.island wraps "tf.UnknownOp"() : () -> () + %control_9 = tf_executor.island wraps "tf.UnknownOp"() : () -> () + %control_10 = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%outputs, %outputs_0) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table3"} : (tensor<8xf32>, tensor<8xf32>) -> () + %control_11 = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%outputs_2, %outputs_5) {config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table4"} : (tensor<8xf32>, tensor<8xf32>) -> () + tf_executor.fetch + } + return +} +// CHECK-LABEL: func @tpu_load_embedding_ops_sink_controls +// CHECK: tf_executor.graph { +// CHECK: %[[outputs:.*]], %[[control:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> +// CHECK: %[[outputs_0:.*]], %[[control_1:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> +// CHECK: %[[outputs_2:.*]], %[[control_3:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> +// CHECK: %[[control_4:.*]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%[[outputs]], %[[outputs_0]]) <{config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table1"}> : (tensor<8xf32>, tensor<8xf32>) -> () +// CHECK: %[[outputs_5:.*]], %[[control_6:.*]] = tf_executor.island wraps "tf.ReadVariableOp"(%arg3) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor<8xf32> +// CHECK: %[[control_7:.*]] = tf_executor.island wraps "tf.LoadTPUEmbeddingAdagradParameters"(%[[outputs_2]], %[[outputs_5]]) <{config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table2"}> : (tensor<8xf32>, tensor<8xf32>) -> () +// CHECK: %[[control_8:.*]] = tf_executor.island(%[[control]], %[[control_1]], %[[control_3]], %[[control_4]], %[[control_6]], %[[control_7]]) wraps "tf.UnknownOp"() : () -> () +// CHECK: %[[control_9:.*]] = tf_executor.island(%[[control_8]]) wraps "tf.UnknownOp"() : () -> () +// CHECK: %[[control_10:.*]] = tf_executor.island(%[[control_9]]) wraps "tf.LoadTPUEmbeddingAdagradParameters"(%[[outputs]], %[[outputs_0]]) <{config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table3"}> : (tensor<8xf32>, tensor<8xf32>) -> () +// CHECK: %[[control_11:.*]] = tf_executor.island(%[[control_9]]) wraps "tf.LoadTPUEmbeddingAdagradParameters"(%[[outputs_2]], %[[outputs_5]]) <{config = "", num_shards = 1 : i64, shard_id = 0 : i64, table_id = -1 : i64, table_name = "table4"}> : (tensor<8xf32>, tensor<8xf32>) -> () +// CHECK: tf_executor.fetch %[[control_10]], %[[control_11]] : !tf_executor.control, !tf_executor.control + +// ----- + +// Tests that we don't create dependencies between ops with same parallel group +// ID but different branch IDs, even if both ops have unknown side effects. +// Also test that the fetch op still depends on all side-effecting ops. +func.func @same_group_different_branches() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_2:.*]] = tf_executor.island wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:1"} : () -> () + // CHECK: tf_executor.fetch %[[control]], %[[control_2]] : !tf_executor.control, !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests that we create dependencies between ops with same parallel group ID and +// same branch ID, if both ops have unknown side effects. +func.func @same_group_same_branch() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_2:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: tf_executor.fetch %[[control_2]] : !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Repeated branches are considered parallel: The 3rd p0:0 has no dependency +// from the previous p0:0. This test demonstrates the current behavior. +// Flatting of nesting structures done by previous passes should never produce +// repeated branches. + +func.func @repeated_branches() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_1:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_2:.*]] = tf_executor.island wraps "tf.C"() + tf_executor.island wraps "tf.C"() {is_stateless = false, _parallel_execution_ids = "p0:1"} : () -> () + // CHECK: %[[control_3:.*]] = tf_executor.island wraps "tf.D"() + tf_executor.island wraps "tf.D"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK-NEXT: tf_executor.fetch %[[control_1]], %[[control_2]], %[[control_3]] : !tf_executor.control, !tf_executor.control, !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests one group with multiple branches. +// We expect the following dependency chains after the pass +// A -> B -> fetch, C -> D -> fetch, E -> fetch. +func.func @one_group_multiple_branches() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_1:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK: %[[control_2:.*]] = tf_executor.island wraps "tf.C"() + tf_executor.island wraps "tf.C"() {is_stateless = false, _parallel_execution_ids = "p0:1"} : () -> () + // CHECK: %[[control_3:.*]] = tf_executor.island(%[[control_2]]) wraps "tf.D"() + tf_executor.island wraps "tf.D"() {is_stateless = false, _parallel_execution_ids = "p0:1"} : () -> () + // CHECK: %[[control_4:.*]] = tf_executor.island wraps "tf.E"() + tf_executor.island wraps "tf.E"() {is_stateless = false, _parallel_execution_ids = "p0:2000"} : () -> () + // CHECK: tf_executor.fetch %[[control_1]], %[[control_3]], %[[control_4]] : !tf_executor.control, !tf_executor.control, !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests nested replica and parallel execute groups. +func.func @nested_replica_and_parallel_execute_groups() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() : () -> () + // CHECK-NEXT: %[[control_2:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {_parallel_execution_ids = "r1:1"} : () -> () + // CHECK-NEXT: %[[control_3:.*]] = tf_executor.island(%[[control_2]]) wraps "tf.C"() + tf_executor.island wraps "tf.C"() {_parallel_execution_ids = "r1:1,p2:1"} : () -> () + // CHECK-NEXT: %[[control_4:.*]] = tf_executor.island(%[[control_2]]) wraps "tf.D"() + tf_executor.island wraps "tf.D"() {_parallel_execution_ids = "r1:1,p2:2"} : () -> () + // CHECK-NEXT: %[[control_5:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {_parallel_execution_ids = "r1:2"} : () -> () + // CHECK-NEXT: %[[control_6:.*]] = tf_executor.island(%[[control_5]]) wraps "tf.C"() + tf_executor.island wraps "tf.C"() {_parallel_execution_ids = "r1:2,p3:1"} : () -> () + // CHECK-NEXT: %[[control_7:.*]] = tf_executor.island(%[[control_5]]) wraps "tf.D"() + tf_executor.island wraps "tf.D"() {_parallel_execution_ids = "r1:2,p3:2"} : () -> () + // CHECK-NEXT: tf_executor.fetch %[[control_3]], %[[control_4]], %[[control_6]], %[[control_7]] : !tf_executor.control, !tf_executor.control, !tf_executor.control, !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests mixed and nested groups and branches. +// We expect the following dependency chains after the pass: +// A -> {B, C} -> D -> E -> fetch. +func.func @mixed_groups_and_branches_nested() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "r1000:0"} : () -> () + // CHECK-NEXT: %[[control_2:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0,r1000:0"} : () -> () + // CHECK-NEXT: %[[control_3:.*]] = tf_executor.island(%[[control]]) wraps "tf.C"() + tf_executor.island wraps "tf.C"() {is_stateless = false, _parallel_execution_ids = "p0:1,r1000:0"} : () -> () + // CHECK-NEXT: %[[control_4:.*]] = tf_executor.island(%[[control_2]], %[[control_3]]) wraps "tf.D"() + tf_executor.island wraps "tf.D"() {is_stateless = false, _parallel_execution_ids = "r2000:0"} : () -> () + // CHECK-NEXT: %[[control_5:.*]] = tf_executor.island(%[[control_4]]) wraps "tf.E"() + tf_executor.island wraps "tf.E"() {is_stateless = false, _parallel_execution_ids = "p0:1,r2000:0"} : () -> () + // CHECK-NEXT: tf_executor.fetch %[[control_5]] : !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests mixed and nested groups and branches where the r1000 group is reused. +// We expect the following dependency chains after the pass: +// A -> {B, C} -> D -> E -> fetch. +func.func @mixed_groups_and_branches_nested_reuse_group() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "r1000:0"} : () -> () + // CHECK-NEXT: %[[control_2:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0,r1000:0"} : () -> () + // CHECK-NEXT: %[[control_3:.*]] = tf_executor.island(%[[control]]) wraps "tf.C"() + tf_executor.island wraps "tf.C"() {is_stateless = false, _parallel_execution_ids = "p0:1,r1000:0"} : () -> () + // CHECK-NEXT: %[[control_4:.*]] = tf_executor.island(%[[control_2]], %[[control_3]]) wraps "tf.D"() + tf_executor.island wraps "tf.D"() {is_stateless = false, _parallel_execution_ids = "r1000:0"} : () -> () + // CHECK-NEXT: %[[control_5:.*]] = tf_executor.island(%[[control_4]]) wraps "tf.E"() + tf_executor.island wraps "tf.E"() {is_stateless = false, _parallel_execution_ids = "p0:1,r1000:0"} : () -> () + // CHECK-NEXT: tf_executor.fetch %[[control_5]] : !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +// Tests that we create dependencies between ops where one op has a parallel +// execution ID and the other has not. +func.func @unspecified_parallel_execution_ids() { + tf_executor.graph { + // CHECK: %[[control:.*]] = tf_executor.island wraps "tf.A"() + tf_executor.island wraps "tf.A"() {is_stateless = false} : () -> () + // CHECK-NEXT: %[[control_2:.*]] = tf_executor.island(%[[control]]) wraps "tf.B"() + tf_executor.island wraps "tf.B"() {is_stateless = false, _parallel_execution_ids = "p0:0"} : () -> () + // CHECK-NEXT: %[[control_3:.*]] = tf_executor.island(%[[control]]) wraps "tf.C"() + tf_executor.island wraps "tf.C"() {is_stateless = false, _parallel_execution_ids = "p0:1"} : () -> () + // CHECK-NEXT: tf_executor.fetch %[[control_2]], %[[control_3]] : !tf_executor.control, !tf_executor.control + tf_executor.fetch + } + func.return +} + +// ----- + +func.func @missing_branch_id() { + tf_executor.graph { + // expected-error@+1 {{Malformed _parallel_execution_ids attribute}} + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "p0:"} : () -> () + tf_executor.fetch + } + func.return +} + +// ----- + +func.func @missing_colon() { + tf_executor.graph { + // expected-error@+1 {{Malformed _parallel_execution_ids attribute}} + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "r01"} : () -> () + tf_executor.fetch + } + func.return +} + +// ----- + +func.func @missing_group_id_prefix() { + tf_executor.graph { + // expected-error@+1 {{Malformed _parallel_execution_ids attribute}} + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "0:0"} : () -> () + tf_executor.fetch + } + func.return +} + +// ----- + +func.func @invalid_group_id_prefix() { + tf_executor.graph { + // expected-error@+1 {{Malformed _parallel_execution_ids attribute}} + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "s0:0"} : () -> () + tf_executor.fetch + } + func.return +} + +// ----- + +func.func @extra_colon() { + tf_executor.graph { + // expected-error@+1 {{Malformed _parallel_execution_ids attribute}} + tf_executor.island wraps "tf.A"() {is_stateless = false, _parallel_execution_ids = "r0:0:1"} : () -> () + tf_executor.fetch + } + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/verify_for_export.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/verify_for_export.mlir new file mode 100644 index 0000000000000000000000000000000000000000..bb6f331749b0234f0b0c82b51805e261ec3034cb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/verify_for_export.mlir @@ -0,0 +1,24 @@ +// RUN: tf-opt %s -split-input-file -tf-verify-for-export -verify-diagnostics | FileCheck %s + +module { + func.func @failsNoIslands() { + // expected-error @+1 {{functions must be of a single Graph with single op Islands: first op in function is not a tf_executor.graph}} + func.return + } +} + +// ----- + +module { + // CHECK-LABEL: func @passesSingleIslandOp + func.func @passesSingleIslandOp() { + // CHECK: _class = ["loc:@class"] + tf_executor.graph { + %c, %control0 = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %a, %control1 = tf_executor.island wraps "tf.A"() {_class = ["loc:@class"]} : () -> (tensor<2xf32>) + %s:2, %control2 = tf_executor.island wraps "tf.Split"(%c, %a) {num_split = 2 : i32} : (tensor, tensor<2xf32>) -> (tensor<1xf32>, tensor<1xf32>) + tf_executor.fetch + } + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/warn_when_using_deprecated_dumps.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/warn_when_using_deprecated_dumps.mlir new file mode 100644 index 0000000000000000000000000000000000000000..db6012e7e2652774d10878e4bd197840638e771e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/warn_when_using_deprecated_dumps.mlir @@ -0,0 +1,17 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics %s + +// Test warning on using deprecated attribute or type in old debug dump. + +func.func @main() { + // expected-error@+1 {{#tf_type.shape}} + "tf.foo"() { shape = #tf.shape} : () -> () + func.return +} + +// ----- + +func.func @main() { + // expected-error@+1 {{!tf_type.string}} + "tf.foo"() : () -> (tensor<*x!tf.string>) + func.return +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/while_licm.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/while_licm.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7192cb440b7368dac937c97bac3aab3a034c620c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/while_licm.mlir @@ -0,0 +1,68 @@ +// RUN: tf-opt -split-input-file -loop-invariant-code-motion %s | FileCheck %s + +// CHECK: while_1([[ARG0:%[^ :]*]]: tensor, [[ARG1:%[^ :]*]]: tensor<1xf32>) +func.func @while_1(%arg0: tensor, %arg1: tensor<1xf32>) -> tensor<1xf32> { + // CHECK: [[CST:%[^ ]*]] = arith.constant dense<1> : tensor + // CHECK: "tf.WhileRegion"([[ARG0]], [[ARG1]]) + // CHECK: (tensor, tensor<1xf32>) -> (tensor, tensor<1xf32>) + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + // cond + { + ^bb0(%condArg0: tensor<*xi32>, %condArg1: tensor<*xf32>): + %0 = "arith.constant" () {value = dense<0> : tensor} : () -> tensor loc("Const") + %1 = "tf.NotEqual"(%condArg0, %0) : (tensor<*xi32>, tensor) -> tensor + "tf.Yield"(%1) : (tensor) -> () + }, + // body + { + ^bb0(%bodyArg0: tensor<*xi32>, %bodyArg1: tensor<*xf32>): + %0 = "arith.constant" () {value = dense<1> : tensor} : () -> tensor loc("Const") + %1 = "tf.Sub"(%bodyArg0, %0) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %2 = "tf.Add"(%bodyArg1, %bodyArg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1, %2) : (tensor<*xi32>, tensor<*xf32>) -> () + } + ) {is_stateless = false} : (tensor, tensor<1xf32>) -> (tensor, tensor<1xf32>) loc("WhileOp") + func.return %0#1 : tensor<1xf32> +} + +// ----- + +// CHECK-LABEL: testWhileRegionisDefinedOutsideOfLoop +func.func @testWhileRegionisDefinedOutsideOfLoop(%arg0 : tensor<4xf32>, %arg1 : tensor) -> tensor<4xf32> { + %a = "tf.Neg"(%arg0) : (tensor<4xf32>) -> tensor<4xf32> + %b = "tf.Abs"(%arg0) : (tensor<4xf32>) -> tensor<4xf32> + // Verify that the Div and Mul are hoisted out of the body + // CHECK: "tf.Div" + // CHECK: constant dense<2.200000e+01> + // CHECK: "tf.Mul" + // CHECK: "tf.WhileRegion" + // Verify that Add and Sub is not hoisted out + // CHECK: "tf.Add" + // CHECK: "tf.Sub" + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ( + { + // condition, check if count has reached 0 + ^bb0(%carg0: tensor<4xf32>, %carg1: tensor): + %zero = arith.constant dense<0> : tensor + %ne = "tf.NotEqual"(%carg1, %zero) : (tensor, tensor) -> tensor + "tf.Yield"(%ne) : (tensor) -> () + }, + { + // loop body + ^bb0(%barg0: tensor<4xf32>, %barg1: tensor): + %add = "tf.Add"(%barg0, %barg0) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %one = arith.constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + + // Some loop invariant math + %li0 = "tf.Div"(%a, %b) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + %cst = arith.constant dense<22.0> : tensor + %li1 = "tf.Mul"(%li0, %cst) : (tensor<4xf32>, tensor) -> tensor<4xf32> + + %final = "tf.Add"(%add, %li1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + "tf.Yield"(%final, %sub) : (tensor<4xf32>, tensor) -> () + } + ) { is_stateless = false } : (tensor<4xf32>, tensor) -> (tensor<4xf32>, tensor) + + func.return %0#0 : tensor<4xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_broadcast.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_broadcast.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b6ddf572f5a736e2250d6ec18e39419b436c53d1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_broadcast.mlir @@ -0,0 +1,50 @@ +// RUN: tf-opt %s -split-input-file -tf-xla-broadcast | FileCheck %s +module attributes {tf.devices = {"/job:tpu_host_worker/replica:0/task:0/device:CPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:1", "/job:tpu_host_worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:1/device:CPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:1", "/job:tpu_host_worker/replica:0/task:1/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1850 : i32}} { +// CHECK-LABEL: func @move_broadcast_non_spmd +func.func @move_broadcast_non_spmd(%arg0: tensor) -> () { + // CHECK: %[[FULL_0:.*]] = "tf.TPUDummyInput"() <{shape = #tf_type.shape<>}> {_ici_weight_distribution_mlir_bridge_marker = true} + // CHECK-NEXT: %[[FULL_1:.*]] = "tf.TPUDummyInput"() <{shape = #tf_type.shape<>}> {_ici_weight_distribution_mlir_bridge_marker = true} + // CHECK-NEXT: tf_device.replicate([%arg0, %[[FULL_0]], %[[FULL_1]], %[[FULL_0]]] as %[[REPVAR:.*]]: tensor) {n = 4 : i32} { + // CHECK-NEXT: %[[ID:.*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + // CHECK-NEXT: %[[IDINSIDE:.*]] = "tf.Identity"(%[[REPVAR]]) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor) -> tensor + // CHECK-NEXT: tf_device.return %[[IDINSIDE]] : tensor + // CHECK-NEXT: }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor + // CHECK-NEXT: "tf_device.cluster"() ({ + // CHECK-NEXT: %[[GROUP:.*]] = "tf.Const"() + // CHECK-SAME: [0, 1, 2, 3] + // CHECK-NEXT: %[[REDUCED:.*]] = "tf.XlaAllReduce"(%[[ID]], %[[GROUP]]) <{mode = "CrossReplica", reduce_op = "Add"}> : (tensor, tensor<1x4xi32>) -> tensor + // CHECK-NEXT: "tf.OpA"(%[[REDUCED]]) : (tensor) -> () + tf_device.replicate {n = 4 : i32} { + "tf_device.cluster"() ({ + "tf.OpA"(%arg0) : (tensor) -> () + tf_device.return + }) {allow_soft_placement = false, computation_shape = [], device_assignment = [0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 4 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\01\10\02\18\02\22\10\00\00\00\00\00\01\00\00\01\00\00\00\01\01\00\00*\02\08\01", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : () -> () + } + func.return +} +} + +// ----- +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2", "/job:worker/replica:0/task:0/device:TPU:3", "/job:worker/replica:0/task:0/device:TPU:4", "/job:worker/replica:0/task:0/device:TPU:5", "/job:worker/replica:0/task:0/device:TPU:6", "/job:worker/replica:0/task:0/device:TPU:7"]} { +// CHECK-LABEL: func @move_broadcast_spmd +func.func @move_broadcast_spmd(%arg0: tensor) -> () { + // CHECK: %[[FULL_0:.*]] = "tf.TPUDummyInput"() <{shape = #tf_type.shape<>}> {_ici_weight_distribution_mlir_bridge_marker = true} + // CHECK-NEXT: tf_device.replicate([%arg0, %[[FULL_0]], %[[FULL_0]], %[[FULL_0]]] as %[[REPVAR:.*]]: tensor) {n = 4 : i32} { + // CHECK-NEXT: %[[ID:.*]] = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + // CHECK-NEXT: %[[IDINSIDE:.*]] = "tf.Identity"(%[[REPVAR]]) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor) -> tensor + // CHECK-NEXT: tf_device.return %[[IDINSIDE]] : tensor + // CHECK-NEXT: }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor + // CHECK-NEXT: "tf_device.cluster"() ({ + // CHECK-NEXT: %[[GROUP:.*]] = "tf.Const"() + // CHECK-SAME: [0, 1, 2, 3] + // CHECK-NEXT: %[[REDUCED:.*]] = "tf.XlaAllReduce"(%[[ID]], %[[GROUP]]) <{mode = "CrossReplica", reduce_op = "Add"}> : (tensor, tensor<1x4xi32>) -> tensor + // CHECK-NEXT: "tf.OpA"(%[[REDUCED]]) : (tensor) -> () + tf_device.replicate {n = 4 : i32} { + "tf_device.cluster"() ({ + "tf.OpA"(%arg0) : (tensor) -> () + tf_device.return + }) {allow_soft_placement = false, computation_shape = [], device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1], host_compute_core = [], num_cores_per_replica = 2 : i64, num_replicas = 4 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01*\02\08\01", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : () -> () + } + func.return +} +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_deserialization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_deserialization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..280a10468f9bfdbe61880f580d08755acd7f7567 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_deserialization.mlir @@ -0,0 +1,66 @@ +// RUN: tf-opt %s -split-input-file -tf-xla-call-module-deserialization | FileCheck %s + +// Tests that `tf.XlaCallModule` with both StableHLO module and TF function +// calls can be deserialized. + +// CHECK-LABEL: module +module { + // CHECK-LABEL: func private @_tf_func + func.func private @_tf_func(%arg0: tensor, %arg1: tensor<*xi32>) { + // CHECK: tf.StreamResults + + // StreamResults is a pseudo op in this test. + "tf.StreamResults"(%arg0, %arg1) : (tensor, tensor<*xi32>) -> () + func.return + } + + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<10xi32>, %[[ARG1:.*]]: tensor<10xi32> + func.func @main(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> { + // CHECK: %[[RESULT:.*]] = "tf.XlaCallModule"(%[[ARG0]], %[[ARG1]]) + // CHECK-NOT: function_list + // CHECK-SAME: module = "" + // CHECK-SAME: _entry_function = @main_0, + // CHECK-SAME: _stablehlo_version = "0.12.0" + + // `module` is stablehlo bytecode for: + // func.func @main(%arg0: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %arg1: tensor<*xi32>) -> (tensor {jax.result_info = ""}) { + // stablehlo.custom_call @tf.call_tf_function(%arg0, %arg1) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + // return %arg0 : tensor + // } + %0 = "tf.XlaCallModule"(%arg0, %arg1) {Sout = [#tf_type.shape], dim_args_spec = [], function_list = [@_tf_func], module = "ML\EFR\07StableHLO_v0.12.0\00\01\19\05\01\05\01\03\05\03\09\07\09\0B\0D\03\8Fm\0F\01?\0B\07\0B\0B\0B\0B\0B\13\0B\0F\133\133\13\13S\0B\0B\0B\0B\0B\0B\0B\0B\0B\13\13\0B\13\13\03/\0B\0B\0B\13\1B\0B\0B\0B\0B\0B\0B\0F\13\0B\0B\0B\0B\0B\0B\0B\13\0B\0F\01\03\0F\03\0D3\07\0B\1B\17\07\02:\03\05\0F\1F\05\11\05\13\05\15\05\17\05\19\03\03\11\13\05\1B\11\01\05\17\01S\15\03\0B\05E\07S\09U\0B[\0DA\17\011\07\03\0B\05?\07]\09?\0BC\0D_\17\01'\07\17\01)\0B\03\13#a%A'c)?+e-?/?1?3g\05\1D\05\1F\05!\05#\05%\05'\05)\05+\05-\17\013\0B\03\039C\05/\17\015\1B\17\017\0B\03\01\1D1\1D3\03\05GQ\0D\05IKMO\1D5\1D7\1D9\1D;\0D\01#\09\03\03W\0D\03YA\1D=\1D?#\0B\1DA\0B\05\1DC\05\03\0D\03ik\1DE\13\0D\01\01\02\04)\03\00\FF\FF\FF\FF\FF\FF\FF\FF\05\1B3\05\11\05\03\07\03\03\11\03\03\03\03\1D\04}\05\01\11\15\0F\07\04m\03\01\09\03\11\19\17\05\03\07\0F\05\03\03\07\03\00\07\055!\05\01\03\09\07;7\03\03\03\01\05\04=\03\05\03\11\1D\1B\05\03\03\07\03\03\03\00\05\04\1F\03\01\06\03\01\05\01\00\9E\07G\1B)\11\0B!\1B\1D\05\1B\1B\03\0F%\1F/!!)#\1F\19)\1F\13\15\1D\15G\11\1F\15\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00return_v1\00custom_call_v1\00call_v1\00xla_call_module_serialization.mlir\00arg_attrs\00function_type\00res_attrs\00sym_name\00sym_visibility\00mhlo.num_partitions\00api_version\00backend_config\00call_target_name\00called_computations\00has_side_effect\00operand_layouts\00output_operand_aliases\00result_layouts\00tf.backend_config\00callee\00\00_stablehlo_f\00jax.arg_info\00x\00mhlo.sharding\00{replicated}\00jax.result_info\00main\00private\00tf.call_tf_function\00called_index\00", platforms = [], version = 5 : i64} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> + // CHECK: return %[[RESULT]] + func.return %0 : tensor<10xi32> + } + + // CHECK-LABEL: func @foo + // CHECK-SAME: %[[ARG0:.*]]: tensor<10xi32>, %[[ARG1:.*]]: tensor<10xi32> + func.func @foo(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> { + // CHECK: %[[RESULT:.*]] = "tf.XlaCallModule"(%[[ARG0]], %[[ARG1]]) + // CHECK-NOT: function_list + // CHECK-SAME: module = "" + // CHECK-SAME: _entry_function = @main_1, + // CHECK-SAME: _stablehlo_version = "0.12.0" + + // `module` is stablehlo bytecode for: + // func.func @main(%arg0: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %arg1: tensor<*xi32>) -> (tensor {jax.result_info = ""}) { + // stablehlo.custom_call @tf.call_tf_function(%arg0, %arg1) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + // return %arg0 : tensor + // } + %0 = "tf.XlaCallModule"(%arg0, %arg1) {Sout = [#tf_type.shape], dim_args_spec = [], function_list = [@_tf_func], module = "ML\EFR\07StableHLO_v0.12.0\00\01\19\05\01\05\01\03\05\03\09\07\09\0B\0D\03\8Fm\0F\01?\0B\07\0B\0B\0B\0B\0B\13\0B\0F\133\133\13\13S\0B\0B\0B\0B\0B\0B\0B\0B\0B\13\13\0B\13\13\03/\0B\0B\0B\13\1B\0B\0B\0B\0B\0B\0B\0F\13\0B\0B\0B\0B\0B\0B\0B\13\0B\0F\01\03\0F\03\0D3\07\0B\1B\17\07\02:\03\05\0F\1F\05\11\05\13\05\15\05\17\05\19\03\03\11\13\05\1B\11\01\05\17\01S\15\03\0B\05E\07S\09U\0B[\0DA\17\011\07\03\0B\05?\07]\09?\0BC\0D_\17\01'\07\17\01)\0B\03\13#a%A'c)?+e-?/?1?3g\05\1D\05\1F\05!\05#\05%\05'\05)\05+\05-\17\013\0B\03\039C\05/\17\015\1B\17\017\0B\03\01\1D1\1D3\03\05GQ\0D\05IKMO\1D5\1D7\1D9\1D;\0D\01#\09\03\03W\0D\03YA\1D=\1D?#\0B\1DA\0B\05\1DC\05\03\0D\03ik\1DE\13\0D\01\01\02\04)\03\00\FF\FF\FF\FF\FF\FF\FF\FF\05\1B3\05\11\05\03\07\03\03\11\03\03\03\03\1D\04}\05\01\11\15\0F\07\04m\03\01\09\03\11\19\17\05\03\07\0F\05\03\03\07\03\00\07\055!\05\01\03\09\07;7\03\03\03\01\05\04=\03\05\03\11\1D\1B\05\03\03\07\03\03\03\00\05\04\1F\03\01\06\03\01\05\01\00\9E\07G\1B)\11\0B!\1B\1D\05\1B\1B\03\0F%\1F/!!)#\1F\19)\1F\13\15\1D\15G\11\1F\15\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00return_v1\00custom_call_v1\00call_v1\00xla_call_module_serialization.mlir\00arg_attrs\00function_type\00res_attrs\00sym_name\00sym_visibility\00mhlo.num_partitions\00api_version\00backend_config\00call_target_name\00called_computations\00has_side_effect\00operand_layouts\00output_operand_aliases\00result_layouts\00tf.backend_config\00callee\00\00_stablehlo_f\00jax.arg_info\00x\00mhlo.sharding\00{replicated}\00jax.result_info\00main\00private\00tf.call_tf_function\00called_index\00", platforms = [], version = 5 : i64} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> + // CHECK: return %[[RESULT]] + func.return %0 : tensor<10xi32> + } + + // CHECK-LABEL: func private @main_0 + // CHECK-SAME: (%[[ARG0:.*]]: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %[[ARG1:.*]]: tensor<*xi32>) -> (tensor {jax.result_info = ""}) attributes {_from_xla_call_module} { + // CHECK: stablehlo.custom_call @tf.call_tf_function(%[[ARG0]], %[[ARG1]]) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + // CHECK: return %arg0 : tensor + // CHECK: } + + // CHECK-LABEL: func private @main_1 + // CHECK-SAME: (%[[ARG0:.*]]: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %[[ARG1:.*]]: tensor<*xi32>) -> (tensor {jax.result_info = ""}) attributes {_from_xla_call_module} { + // CHECK: stablehlo.custom_call @tf.call_tf_function(%[[ARG0]], %[[ARG1]]) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + // CHECK: return %arg0 : tensor + // CHECK: } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_round_trip.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_round_trip.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a5e7e57e1f2bd531099de43afd9158bd3b679344 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_round_trip.mlir @@ -0,0 +1,61 @@ +// RUN: tf-opt %s -split-input-file -tf-xla-call-module-serialization -tf-xla-call-module-deserialization | FileCheck %s + +// Tests that running xla-call-module-serialization followed by +// xla-call-module-deserialization preserves the original module. +// +// Note that function names may be different, but arguments, attributes, +// results, and function body should be the same. + +// CHECK-LABEL: module +module { + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<10xi32>, %[[ARG1:.*]]: tensor<10xi32> + func.func @main(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> { + // CHECK: %[[RESULT:.*]] = "tf.XlaCallModule"(%[[ARG0]], %[[ARG1]]) + // CHECK-SAME: Sout = [#tf_type.shape] + // CHECK-NOT: function_list + // CHECK-SAME: module = "" + // CHECK-SAME: platforms = [] + // CHECK-SAME: version = 5 + // CHECK-SAME: _entry_function = @main_0 + // CHECK-SAME: _stablehlo_module_attrs = {} + // CHECK-SAME: _stablehlo_version = "0.12.0" + + %0 = "tf.XlaCallModule"(%arg0, %arg1) {Sout = [#tf_type.shape], dim_args_spec = [], _entry_function = @main_0, _stablehlo_version = "0.12.0", module = "", platforms = [], version = 5 : i64} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> + // CHECK: return %[[RESULT]] + func.return %0 : tensor<10xi32> + } + + // CHECK-LABEL: func private @_tf_func + func.func private @_tf_func(%arg0: tensor, %arg1: tensor<*xi32>) { + // CHECK: tf.StreamResults + + // StreamResults is a pseudo op in this test. + "tf.StreamResults"(%arg0, %arg1) : (tensor, tensor<*xi32>) -> () + func.return + } + + // CHECK-LABEL: func private @main_0 + // CHECK-SAME: %[[ARG0:.*]]: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"} + // CHECK-SAME: %[[ARG1:.*]]: tensor<*xi32>) + // CHECK-SAME: (tensor {jax.result_info = ""}) + // CHECK-SAME: attributes {_from_xla_call_module} + func.func private @main_0(%arg0: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %arg1: tensor<*xi32>) -> (tensor {jax.result_info = ""}) attributes {_from_xla_call_module} { + // CHECK: stablehlo.custom_call @tf.call_tf_function(%[[ARG0]], %[[ARG1]]) + // CHECK-SAME: { + // CHECK-SAME: api_version = 2 : i32, + // CHECK-SAME: has_side_effect = true, + // CHECK-SAME: tf.backend_config = {called_func = @_tf_func} + // CHECK-SAME: } + stablehlo.custom_call @tf.call_tf_function(%arg0, %arg1) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + // CHECK: call @f + %arg2 = func.call @f(%arg0) : (tensor) -> (tensor) + return %arg2 : tensor + } + + // CHECK-LABEL: func private @f + // CHECK: attributes {_from_xla_call_module} + func.func private @f(%arg0: tensor) -> (tensor) attributes {_from_xla_call_module} { + return %arg0 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_serialization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_serialization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..36ade74fc2b130d2c10b3003996a1b0537c436dd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_call_module_serialization.mlir @@ -0,0 +1,45 @@ +// RUN: tf-opt %s -split-input-file -tf-xla-call-module-serialization | FileCheck %s + +// Tests that stablehlo functions called by XlaCallModuleOp in the top-level +// module can be serialized into bytecode and embedded in XlaCallModuleOp's +// `module` attribute. + +// CHECK-LABEL: module +module { + // CHECK-LABEL: func private @_tf_func + func.func private @_tf_func(%arg0: tensor, %arg1: tensor<*xi32>) { + // CHECK: tf.StreamResults + + // StreamResults is a pseudo op in this test. + "tf.StreamResults"(%arg0, %arg1) : (tensor, tensor<*xi32>) -> () + func.return + } + + // CHECK-NOT: @_stablehlo_f + func.func private @_stablehlo_f(%arg0: tensor) -> (tensor) attributes {_from_xla_call_module} { + return %arg0 : tensor + } + + // CHECK-NOT: @_stablehlo_main_0 + func.func private @_stablehlo_main_0(%arg0: tensor {jax.arg_info = "x", mhlo.sharding = "{replicated}"}, %arg1: tensor<*xi32>) -> (tensor {jax.result_info = ""}) attributes {_from_xla_call_module} { + stablehlo.custom_call @tf.call_tf_function(%arg0, %arg1) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @_tf_func}} : (tensor, tensor<*xi32>) -> () + %arg2 = func.call @_stablehlo_f(%arg0) : (tensor) -> (tensor) + return %arg2 : tensor + } + + // CHECK-LABEL: func @main + // CHECK-SAME: %[[ARG0:.*]]: tensor<10xi32>, %[[ARG1:.*]]: tensor<10xi32> + func.func @main(%arg0: tensor<10xi32>, %arg1: tensor<10xi32>) -> tensor<10xi32> { + // CHECK: %[[RESULT:.*]] = "tf.XlaCallModule"(%[[ARG0]], %[[ARG1]]) + // CHECK-SAME: Sout = [#tf_type.shape] + // CHECK-SAME: dim_args_spec = [] + // CHECK-NOT: _entry_function + // CHECK-NOT: _stablehlo_module_attrs + // CHECK-SAME: function_list = [@_tf_func] + // CHECK-SAME: module = "ML\EFR{{.*}}StableHLO_v0.12.0{{.*}}" + + %0 = "tf.XlaCallModule"(%arg0, %arg1) {Sout = [#tf_type.shape], dim_args_spec = [], _entry_function = @_stablehlo_main_0, _stablehlo_module_attrs = { mhlo.num_partitions = 1 }, _stablehlo_version = "0.12.0", module = "", platforms = [], version = 5 : i64} : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32> + // CHECK: return %[[RESULT]] + func.return %0 : tensor<10xi32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_cluster_formation.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_cluster_formation.mlir new file mode 100644 index 0000000000000000000000000000000000000000..eec1c1f5b2c9382ddf1cb32df7fb9445115677f7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_cluster_formation.mlir @@ -0,0 +1,87 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-xla-cluster-formation | FileCheck %s + +// Check that we outline the partitioned call with `_xla_compile_device_type` +// to a device cluster. +// CHECK-LABEL: func.func @xla_must_compile_true +// CHECK: tf_device.cluster +// CHECK-NEXT: tf.StatefulPartitionedCall +// CHECK-NEXT: tf_device.return +// CHECK: _cluster_outlined_function_name = "stateful_pcall_func" +// CHECK-SAME: allow_soft_placement = true +// CHECK: tf.Const +// CHECK: tf.Add +func.func @xla_must_compile_true(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", _xla_compile_device_type = "CPU", f = @stateful_pcall_func} : (tensor) -> (tensor) + %1 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %2 = "tf.Add"(%0, %1) : (tensor, tensor) -> (tensor) + func.return %2 : tensor +} + +func.func @stateful_pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// Check that we don't outline the partitioned call without +// `_xla_compile_device_type` to a device cluster. +// CHECK-LABEL: func.func @xla_must_compile_false +// CHECK-NOT: tf_device.cluster +func.func @xla_must_compile_false(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @stateful_pcall_func} : (tensor) -> (tensor) + %1 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %2 = "tf.Add"(%0, %1) : (tensor, tensor) -> (tensor) + func.return %2 : tensor +} + +func.func @stateful_pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// Check that we only outline the outermost partitioned call with +// `_xla_compile_device_type` in nested calls to a device cluster. The callee +// of any partitioned call outside of a device cluster is marked with +// `tf._noinline = true` to prevent it getting inlined to perseve runtime +// performance. +// CHECK-LABEL: func.func @nested_calls +func.func @nested_calls(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.While"(%arg0) {cond = @while_cond_func, body = @while_body_func, is_stateless = true} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @while_cond_func +func.func @while_cond_func(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @while_body_func +func.func @while_body_func(%arg0: tensor) -> (tensor) { + // CHECK-NOT: tf_device.cluster + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @outer_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @outer_stateful_pcall_func +// CHECK-SAME: attributes {tf._noinline = true} +func.func @outer_stateful_pcall_func(%arg0: tensor) -> (tensor) { + // CHECK: tf_device.cluster + // CHECK-NEXT: tf.StatefulPartitionedCall + // CHECK-NEXT: tf_device.return + // CHECK: allow_soft_placement = true + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @inner_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @inner_stateful_pcall_func +// CHECK-NOT: tf._noinline +func.func @inner_stateful_pcall_func(%arg0: tensor) -> tensor { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_inline_device_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_inline_device_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3fc9782e8dab8c84b0be3b5c10b4b32bf20531aa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_inline_device_ops.mlir @@ -0,0 +1,76 @@ +// RUN: tf-opt %s -tf-xla-inline-device-ops | FileCheck %s + +// CHECK-LABEL: func @simple_stateful_partitioned_call +// CHECK-NOT: "tf_device.cluster" +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-NEXT: "tf.Const" +func.func @simple_stateful_partitioned_call(%arg0: tensor) -> tensor { + %0 = "tf_device.cluster"() ({ + %2 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @stateful_pcall_func} : (tensor) -> tensor + tf_device.return %2 : tensor + }) {cluster_attr = "cluster_attr"} : () -> tensor + %cst = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %1 = "tf.Add"(%0, %cst) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func @stateful_pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// CHECK-LABEL: func @stateful_partitioned_call_multiple_ops +// CHECK-NOT: "tf_device.cluster" +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-NOT: "tf_device.return" +// CHECK-NEXT: "tf.Const" +func.func @stateful_partitioned_call_multiple_ops(%arg0: tensor) -> tensor { + %0 = "tf_device.cluster"() ({ + %2 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @stateful_pcall_func} : (tensor) -> tensor + %3 = "tf.Const"() {value = dense<4> : tensor} : () -> tensor + %4 = "tf.Add"(%2, %3) : (tensor, tensor) -> (tensor) + tf_device.return %4 : tensor + }) {cluster_attr = "cluster_attr"} : () -> tensor + %5 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %6 = "tf.Add"(%0, %5) : (tensor, tensor) -> tensor + func.return %6 : tensor +} + +// ----- + +// CHECK-LABEL: func @no_stateful_partitioned_call_in_cluster_op +// CHECK-NOT: "tf_device.cluster" +// CHECK-NOT: "tf.StatefulPartitionedCall" +// CHECK: "tf.Const" +func.func @no_stateful_partitioned_call_in_cluster_op(%arg0: tensor) -> tensor { + %0 = "tf_device.cluster"() ({ + %1 = "tf.Const"() {value = dense<4> : tensor} : () -> tensor + tf_device.return %1 : tensor + }) {cluster_attr = "cluster_attr"} : () -> tensor + %2 = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + %3 = "tf.Add"(%2, %0) : (tensor, tensor) -> tensor + func.return %3 : tensor +} + +// ----- + +// CHECK-LABEL: func @multi_return_values_in_cluster_op +// CHECK "tf.Const"() {value = dense<1> : tensor} : () -> tensor +// CHECK "tf.Const"() {value = dense<2.000000e+00> : tensor} : () -> tensor +// CHECK "tf.Const"() {value = dense<3.000000e+00> : tensor} : () -> tensor +// CHECK "tf.Add"(%cst_1, %cst_0) : (tensor, tensor) -> tensor +// CHECK "tf.Const"() {value = dense<4> : tensor} : () -> tensor +// CHECK "tf.Add"(%cst_2, %cst) : (tensor, tensor) -> tensor +func.func @multi_return_values_in_cluster_op(%arg0: tensor) -> () { + %0, %1 = "tf_device.cluster"() ({ + %2 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %3 = "tf.Const"() {value = dense<2.0> : tensor} : () -> tensor + tf_device.return %2, %3 : tensor, tensor + }) {cluster_attr = "cluster_attr"} : () -> (tensor, tensor) + %4 = "tf.Const"() {value = dense<3.0> : tensor} : () -> tensor + %5 = "tf.Add"(%4, %1) : (tensor, tensor) -> tensor + %6 = "tf.Const"() {value = dense<4> : tensor} : () -> tensor + %7 = "tf.Add"(%6, %0) : (tensor, tensor) -> tensor + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_rewrite.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_rewrite.mlir new file mode 100644 index 0000000000000000000000000000000000000000..429a202912d97ba7150e4b02ae0b26258a20dcc6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_rewrite.mlir @@ -0,0 +1,50 @@ +// RUN: tf-opt %s -split-input-file -tf-xla-rewrite | FileCheck %s + + +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:GPU:0"]} { + // CHECK-LABEL: func.func @convert_cluster_func + func.func @convert_cluster_func(%arg0: tensor) -> tensor { + // CHECK: "tf.XlaLaunch"(%arg0) <{function = @func, operandSegmentSizes = array}> : (tensor) -> tensor + %0 = "tf_device.cluster_func"(%arg0) {func = @func} : (tensor) -> tensor + func.return %0 : tensor + } + + func.func @func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:GPU:0"]} { + // CHECK-LABEL: func.func @convert_cluster_func_with_resources_in_order + func.func @convert_cluster_func_with_resources_in_order(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: "tf.XlaLaunch"(%arg1, %arg0) <{function = @func_with_resources_in_order, operandSegmentSizes = array}> : (tensor, tensor) -> tensor + %0 = "tf_device.cluster_func"(%arg1, %arg0) {func = @func_with_resources_in_order} : (tensor, tensor) -> (tensor) + func.return %0 : tensor + } + + func.func @func_with_resources_in_order(%arg0 : tensor, %arg1 : tensor) -> tensor { + func.return %arg0 : tensor + } +} + +// ----- + +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:GPU:0"]} { + // CHECK-LABEL: func.func @convert_cluster_func_with_resources + func.func @convert_cluster_func_with_resources(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: "tf.XlaLaunch"(%arg1, %arg0) <{function = @func_with_resources, operandSegmentSizes = array}> : (tensor, tensor) -> tensor + %0 = "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_resources} : (tensor, tensor) -> tensor + // CHECK: "tf.XlaLaunch"(%arg1, %arg0) <{function = @func_with_resources, operandSegmentSizes = array}> : (tensor, tensor) -> tensor + %1 = "tf_device.cluster_func"(%arg0, %arg1) {func = @func_with_resources} : (tensor, tensor) -> tensor + return %0 : tensor + } + + // CHECK-LABEL: func.func @func_with_resources + // CHECK-SAME: (%arg0: tensor, %arg1: tensor) -> tensor + // CHECK: return %arg0 : tensor + func.func @func_with_resources(%arg0 : tensor, %arg1: tensor) -> tensor { + func.return %arg1 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_sharding_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_sharding_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..6a762b0065de46d618e79bcd355ba4e7aae4503d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_sharding_util_test.cc @@ -0,0 +1,75 @@ +/* Copyright 2023 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" + +#include + +#include +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "xla/xla_data.pb.h" + +inline constexpr llvm::StringRef kXlaShardingAttrName = "_XlaSharding"; + +namespace tensorflow { +namespace { + +TEST(DecodeShardingAttributeTest, CheckInvalidString) { + xla::OpSharding sharding; + EXPECT_TRUE(DecodeShardingAttribute("", sharding).succeeded()); + EXPECT_TRUE(DecodeShardingAttribute("manual", sharding).failed()); +} + +TEST(DecodeShardingAttributeTest, CheckManualShardString) { + xla::OpSharding sharding; + EXPECT_TRUE(DecodeShardingAttribute("{manual}", sharding).succeeded()); + EXPECT_TRUE(sharding.type() == sharding.MANUAL); + EXPECT_EQ(0, sharding.tile_assignment_devices_size()); +} + +TEST(DecodeShardingAttributeTest, CheckMaximalShardString) { + xla::OpSharding sharding; + EXPECT_TRUE( + DecodeShardingAttribute("{maximal device=0}", sharding).succeeded()); + EXPECT_TRUE(sharding.type() == sharding.MAXIMAL); + EXPECT_EQ(1, sharding.tile_assignment_devices_size()); +} + +TEST(ShardingEquivalenceTest, CheckOpShardingEquivalent) { + xla::OpSharding sharding1; + EXPECT_TRUE( + DecodeShardingAttribute("{devices=[2,1]0,1}", sharding1).succeeded()); + xla::OpSharding sharding2; + EXPECT_TRUE( + DecodeShardingAttribute("{devices=[2,1]<=[2]}", sharding2).succeeded()); + EXPECT_TRUE(VerifyShardingEquivalent(sharding1, sharding2).succeeded()); +} + +TEST(ShardingEquivalenceTest, CheckOpShardingNotEquivalent) { + xla::OpSharding sharding1; + EXPECT_TRUE( + DecodeShardingAttribute("{devices=[2,1]1,0}", sharding1).succeeded()); + xla::OpSharding sharding2; + EXPECT_TRUE( + DecodeShardingAttribute("{devices=[2,1]<=[2]}", sharding2).succeeded()); + EXPECT_FALSE(VerifyShardingEquivalent(sharding1, sharding2).succeeded()); +} +} // namespace + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_validate_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_validate_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d67c71b5176a298297652eae5d6b337ad4259301 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/tests/xla_validate_inputs.mlir @@ -0,0 +1,21 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-xla-validate-inputs + +// expected-error @+1 {{expects no nested calls of entry functions as they prevent graph traversal in some passes from working correctly}} +func.func @nested_entry_functions() attributes {tf.entry_function = {}} { + tf_executor.graph { + %control = tf_executor.island wraps "tf.StatefulPartitionedCall"() {config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : () -> () + tf_executor.fetch + } + func.return +} + +func.func @func() attributes {tf.entry_function = {}} { + func.return +} + +// ----- + +// expected-error @+1 {{does not support top-level compilation marker}} +func.func @top_level_compilation_marker() attributes {_xla_compile_device_type = "CPU", tf.entry_function = {}} { + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..799a31c96ab16855f2969dc935930a98f6499b3d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/BUILD @@ -0,0 +1,1080 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("//tensorflow:tensorflow.bzl", "if_not_tflite_converter", "tf_cc_test") +load("//tensorflow:tensorflow.default.bzl", "filegroup", "get_compatible_with_portable") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +# copybara:uncomment_end(google-only) +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +gentbl_cc_library( + name = "tensorflow_canonicalize_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_canonicalize.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "canonicalize.td", + deps = [ + ":rewrite_util_td_files", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + ], +) + +cc_library( + name = "tfe_legalize_tfg", + srcs = [ + "passes.h", + "tfg-to-tfe.cc", + ], + deps = [ + ":tf_device_pass_inc_gen", + ":tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:framework", + "//tensorflow/core/transforms/toposort:Pass", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@shardy//shardy/dialect/sdy/ir:dialect", + ], + alwayslink = 1, +) + +cc_library( + name = "mlprogram", + srcs = [ + "mlprogram.cc", + ], + hdrs = [ + "mlprogram.h", + ], + deps = [ + ":bridge", + ":tensorflow_passes", + ":tf_saved_model_passes", + "//tensorflow/compiler/mlir/tf2xla/internal:clustering_bridge_passes", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalize_tf", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@xla//xla/mlir_hlo:mhlo_passes", + ], +) + +gentbl_cc_library( + name = "decompose_resource_ops_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_decompose_resource_ops.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "decompose_resource_ops.td", + deps = [ + ":rewrite_util_td_files", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:FuncTdFiles", + ], +) + +cc_library( + name = "decompose_resource_ops", + srcs = [ + "decompose_resource_ops.cc", + ], + hdrs = [ + "decompose_resource_ops.h", + ], + deps = [ + ":decompose_resource_ops_inc_gen", + ":rewrite_util", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core:framework", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) + +td_library( + name = "rewrite_util_td_files", + srcs = [ + "rewrite_util.td", + ], + compatible_with = get_compatible_with_portable(), + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + ], +) + +cc_library( + name = "rewrite_util", + srcs = [ + "rewrite_util.cc", + ], + hdrs = [ + "rewrite_util.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/core:framework", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +gentbl_cc_library( + name = "tf_data_optimization_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_tf_data_optimization.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_data_optimization.td", + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:FuncTdFiles", + ], +) + +cc_library( + name = "tf_data_optimization", + srcs = [ + "tf_data_optimization.cc", + ], + hdrs = [ + "tf_data_optimization.h", + ], + deps = [ + ":tf_data_optimization_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@llvm-project//mlir:IR", + ], +) + +cc_library( + name = "unroll_batch_matmul_pass", + srcs = [ + "unroll_batch_matmul.cc", + ], + hdrs = [ + "unroll_batch_matmul.h", + ], + deps = [ + ":tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:framework", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/memory", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AffineAnalysis", + "@llvm-project//mlir:Analysis", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "lift_variables_lib", + srcs = [ + "lift_variables.cc", + ], + hdrs = [ + "lift_variables.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/core:core_cpu", + "//tensorflow/core:core_cpu_lib", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:threadpool_options", + "@com_google_absl//absl/status:statusor", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "mark_initialized_variables_lib", + srcs = [ + "mark_initialized_variables.cc", + ], + hdrs = [ + "mark_initialized_variables.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:session_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/utils:string_container_utils", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "tf_saved_model_freeze_variables", + srcs = [ + "tf_saved_model_freeze_variables.cc", + ], + hdrs = [ + "tf_saved_model_freeze_variables.h", + ], + deps = [ + ":tf_saved_model_freeze_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:resource_value_typed_analyzer", + "//tensorflow/compiler/mlir/tensorflow:session_utils", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework_internal", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tf_saved_model_freeze_utils", + srcs = [ + "tf_saved_model_freeze_utils.cc", + ], + hdrs = [ + "tf_saved_model_freeze_utils.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:framework_internal", + "//tensorflow/core:protos_all_cc", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "initialize_variables_in_session_init", + srcs = [ + "initialize_variables_in_session_init.cc", + ], + hdrs = [ + "initialize_variables_in_session_init.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:session_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework_internal", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tf_saved_model_passes", + srcs = [ + "convert_session_initializer_to_function.cc", + "deduplicate_bound_input_bindings.cc", + "freeze_global_tensors.cc", + "freeze_saved_model_assets.cc", + "lower_globals_to_ml_program.cc", + "lower_variable_ops_to_ml_program.cc", + "optimize_global_tensors.cc", + "remove_vars_in_session_initializer.cc", + "strip_saved_module_metadata.cc", + "unfreeze_mutable_global_tensors.cc", + ], + hdrs = [ + "tf_saved_model_passes.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":tensorflow_passes", + ":tf_saved_model_asset_sinking_pass", + ":tf_saved_model_freeze_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:resource_value_typed_analyzer", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AffineUtils", + "@llvm-project//mlir:Analysis", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:MLProgramDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + ], +) + +gentbl_cc_library( + name = "tf_pass_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = { + "tf_passes.h.inc": [ + "-gen-pass-decls", + "-name=TensorFlow", + ], + "g3doc/_includes/tf_passes.md": ["-gen-pass-doc"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "tf_device_pass_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = { + "tf_device_passes.h.inc": [ + "-gen-pass-decls", + "-name=TensorFlowDevice", + ], + "g3doc/includes/tf_device_passes.md": ["-gen-pass-doc"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_device_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "tf_savedmodel_pass_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = { + "tf_savedmodel_passes.h.inc": [ + "-gen-pass-decls", + "-name=TensorFlowSavedModel", + ], + "g3doc/includes/tf_savedmodel_passes.md": ["-gen-pass-doc"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_savedmodel_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "tf_test_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = { + "test_passes.h.inc": [ + "-gen-pass-decls", + "-name=TensorFlowTest", + ], + "g3doc/includes/tf_test_passes.md": ["-gen-pass-doc"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_test_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "tensorflow_passes", + srcs = [ + "add_functions_for_exported_names.cc", + "annotate_parameter_replication.cc", + "batchmatmul_to_einsum.cc", + "breakup-islands.cc", + "canonicalize_compile_and_replicate_attributes.cc", + "check_control_dependencies.cc", + "cluster_ops_by_policy.cc", + "cluster_outlining.cc", + "cluster_tf_ops_pass.cc", + "collection_ops_util.cc", + "colocate_tpu_copy_with_dynamic_shape.cc", + "constant_op_device_assignment.cc", + "convert_control_to_data_outputs.cc", + "convert_launch_func_to_tf_call.cc", + "convert_tf_control_flow_to_scf.cc", + "convert_to_legacy_compile_and_replicate_attributes.cc", + "decompose_optionals.cc", + "decompose_reduce_dataset.cc", + "decompose_resource_ops_pass.cc", + "device_attribute_to_launch.cc", + "device_index_selector.cc", + "drop_while_shape_invariant.cc", + "einsum.cc", + "executor_island_coarsening.cc", + "executor_tpuv1_inline_tpu_island.cc", + "executor_tpuv1_island_coarsening.cc", + "executor_tpuv1_outline_tpu_island.cc", + "extract_tpu_copy_with_dynamic_shape_op.cc", + "fold_broadcast.cc", + "functional_control_flow_to_cfg.cc", + "functional_control_flow_to_regions.cc", + "fused_kernel_matcher.cc", + "generated_canonicalize.inc", + "generated_optimize.inc", + "gpu_fusion.cc", + "graph_pruning.cc", + "group_by_dialect.cc", + "guarantee_all_funcs_one_use.cc", + "hoist_loop_invariant.cc", + "hoist_replicate_invariant_resource_writes.cc", + "init_text_file_to_import.cc", + "launch_to_device_attribute.cc", + "layout_optimization.cc", + "localize_var_handles.cc", + "lower_quantized.cc", + "mark_input_output_aliases.cc", + "materialize_mlir_passthrough_op.cc", + "merge_control_flow.cc", + "move_tpu_compile_to_front.cc", + "name_anonymous_iterators.cc", + "optimize.cc", + "order_by_dialect.cc", + "order_for_program_key.cc", + "parallel_execute_to_islands.cc", + "prepare_tpu_computation_for_tf_export.cc", + "print.cc", + "promote_resources_to_args.cc", + "readonly_references_to_resources.cc", + "region_control_flow_to_functional.cc", + "remove_unused_arguments.cc", + "remove_unused_while_results.cc", + "replica_id_to_device_ordinal.cc", + "replicate_invariant_op_hoisting.cc", + "replicate_tensor_list_init_ops_pass.cc", + "replicate_to_island.cc", + "resource_device_inference.cc", + "resource_op_lifting.cc", + "resource_op_lifting_cleanup.cc", + "resource_op_lifting_cleanup.h", + "rewrite_tpu_embedding_ops.cc", + "sink_constant.cc", + "stack_ops_decomposition.cc", + "strip_noinline_attribute.cc", + "strip_tf_attributes.cc", + "tensor_array_ops_decomposition.cc", + "tensor_device_copy_conversion.cc", + "tensor_list_ops_decomposition.cc", + "test_resource_alias_analysis.cc", + "tf_data_optimization_pass.cc", + "tf_device_assignment.cc", + "tf_executor_to_functional.cc", + "tf_functional_to_executor.cc", + "tpu_annotate_dynamic_shape_inputs.cc", + "tpu_cluster_cleanup_attributes.cc", + "tpu_colocate_composite_resource_ops.cc", + "tpu_colocate_splits.cc", + "tpu_device_propagation.cc", + "tpu_dynamic_layout_pass.cc", + "tpu_host_computation_expansion.cc", + "tpu_identity_pruning.cc", + "tpu_parallel_execute_sink_resource_write.cc", + "tpu_partitioned_op_conversion.cc", + "tpu_reorder_replicate_and_partitioned_inputs.cc", + "tpu_resource_partitioning.cc", + "tpu_resource_read_for_write.cc", + "tpu_space_to_depth_pass.cc", + "tpu_update_embedding_enqueue_op_inputs.cc", + "update_control_dependencies.cc", + "verify_suitable_for_graph_export_pass.cc", + "xla_call_module_deserialization.cc", + "xla_call_module_serialization.cc", + "xla_inline_device_ops.cc", + "xla_rewrite.cc", + "xla_validate_inputs.cc", + ], + hdrs = [ + "cluster_ops_by_policy.h", + "collection_ops_util.h", + "einsum.h", + "passes.h", + ], + includes = ["include"], + textual_hdrs = [ + "tf_device_passes.h.inc", + "tf_passes.h.inc", + "tf_savedmodel_passes.h.inc", + ], + visibility = ["//visibility:public"], + deps = [ + ":cluster_formation", + ":decompose_resource_ops", + ":decompose_resource_ops_inc_gen", + ":lower_tf_lib", + ":shape_inference_pass", + ":tensorflow_optimize_inc_gen", + ":tf_data_optimization", + ":tf_device_pass_inc_gen", + ":tf_pass_inc_gen", + ":tf_savedmodel_pass_inc_gen", + ":tfe_legalize_tfg", + ":unroll_batch_matmul_pass", + ":verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:call_graph_util", + "//tensorflow/compiler/mlir/tensorflow:cluster_util", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/compiler/mlir/tensorflow:parallel_execute_util", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:stablehlo_custom_call_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_op_interfaces", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_side_effects", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_traits", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tf_ops_layout_helper", + "//tensorflow/compiler/mlir/tensorflow:tf_saved_model_inc_gen", + "//tensorflow/compiler/mlir/tensorflow:topological_sort", + "//tensorflow/compiler/mlir/tensorflow:tpu_cluster_util", + "//tensorflow/compiler/mlir/tensorflow:tpu_embedding_ops_registry", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow:verification_utils", + "//tensorflow/compiler/mlir/tensorflow:verify_suitable_for_graph_export", + "//tensorflow/compiler/mlir/tensorflow:visitor", + "//tensorflow/compiler/mlir/tensorflow:xla_call_module_attrs", + "//tensorflow/compiler/mlir/tensorflow:xla_rewrite_util", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:tpu_metadata_utils", + "//tensorflow/compiler/mlir/tf2xla/internal/inference:inference_metrics_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/compiler/mlir/tf2xla/transforms:split_into_island_per_op_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:xla_legalize_tf", + "//tensorflow/compiler/mlir/tf2xla/transforms:xla_legalize_tf_with_tf2xla", + "//tensorflow/compiler/mlir/utils:validators", + "//tensorflow/compiler/tf2xla:side_effect_util", + "//tensorflow/compiler/tf2xla/kernels:xla_call_module_loader", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/ir/types:Dialect", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:logging", + "//tensorflow/core/platform:random", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/protobuf/tpu:dynamic_padding_proto_cc", + "//tensorflow/core/tpu:tpu_embedding_optimization_parameters_utils", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:variant", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AffineAnalysis", + "@llvm-project//mlir:AffineUtils", + "@llvm-project//mlir:Analysis", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ControlFlowDialect", + "@llvm-project//mlir:ControlFlowInterfaces", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:QuantOps", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + "@shardy//shardy/dialect/sdy/ir:dialect", + "@stablehlo//:chlo_ops", + "@stablehlo//:stablehlo_ops", + "@stablehlo//:stablehlo_portable_api", + "@stablehlo//:stablehlo_serialization", + "@stablehlo//:version", + "@stablehlo//:vhlo_ops", + "@tsl//tsl/platform:error_logging", + "@xla//xla:window_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla:xla_proto_cc", + "@xla//xla/hlo/builder:sharding_builder", + "@xla//xla/mlir_hlo", + ], +) + +cc_library( + name = "cluster_formation", + srcs = ["cluster_formation.cc"], + textual_hdrs = [ + "tf_passes.h.inc", + ], + deps = [ + ":tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:cluster_util", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/core:framework", + "//tensorflow/core:portable_gif_internal", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], + alwayslink = 1, +) + +cc_library( + name = "bridge", + srcs = ["bridge.cc"], + hdrs = ["bridge.h"], + deps = [ + ":tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/core:lib_proto_parsing", + "//tensorflow/core/platform:status", + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "verify_no_outside_compilation_markers_pass", + srcs = ["verify_no_outside_compilation_markers_pass.cc"], + hdrs = ["passes.h"], + textual_hdrs = [ + "tf_device_passes.h.inc", + "tf_passes.h.inc", + ], + deps = [ + ":tf_device_pass_inc_gen", + ":tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@shardy//shardy/dialect/sdy/ir:dialect", + ], +) + +tf_cc_test( + name = "verify_no_outside_compilation_markers_pass_test", + srcs = ["verify_no_outside_compilation_markers_pass_test.cc"], + deps = [ + ":tensorflow_passes", + "//tensorflow/compiler/mlir/tf2xla/transforms:test_utils", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "shape_inference_pass", + srcs = [ + "passes.h", + "shape_inference.cc", + "shape_inference_pass.cc", + ], + hdrs = [ + "shape_inference.h", + ], + deps = [ + ":tf_device_pass_inc_gen", + ":tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:shape_inference_utils", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/translate/tools:parsers", + "//tensorflow/compiler/tf2xla/kernels:xla_call_module_loader", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/ir/types:Dialect", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@shardy//shardy/dialect/sdy/ir:dialect", + "@stablehlo//:stablehlo_serialization", + "@stablehlo//:version", + "@xla//xla:shape_util", + "@xla//xla:window_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/translate/hlo_to_mhlo:hlo_utils", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo", + "@xla//xla/service:shape_inference", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/util:env_var", + ], +) + +cc_library( + name = "tensorflow_test_passes", + testonly = True, # Ensure alwayslink does not leak in the codebase. + srcs = [ + "init_text_file_to_import_test_pass.cc", + "initialize_variables_in_session_init_test_pass.cc", + "lift_variables_test_pass.cc", + "lower_tf_test_pass.cc", + "mark_initialized_variables_test_pass.cc", + "resource_analyzer_test_pass.cc", + "test_cluster_ops_by_policy.cc", + "test_passes.h.inc", + "test_side_effect_analysis.cc", + "tf_saved_model_freeze_variables_test_pass.cc", + ], + hdrs = [ + "test_passes.h", + ], + deps = [ + ":initialize_variables_in_session_init", + ":lift_variables_lib", + ":lower_tf_lib", + ":mark_initialized_variables_lib", + ":tensorflow_passes", + ":tf_saved_model_freeze_variables", + ":tf_saved_model_passes", + ":tf_test_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:fake_session", + "//tensorflow/compiler/mlir/tensorflow:resource_value_typed_analyzer", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/core:core_cpu", + "//tensorflow/core:core_cpu_lib", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:status", + "//tensorflow/core/platform:threadpool_options", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + ], + alwayslink = 1, +) + +cc_library( + name = "graph_optimization_pass", + srcs = ["graph_optimization_pass.cc"], + hdrs = ["graph_optimization_pass.h"], + deps = [ + ":tensorflow_passes", + "//tensorflow/compiler/mlir:mlir_graph_optimization_pass", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +cc_library( + name = "graph_optimization_pass_registration", + srcs = ["graph_optimization_pass_registration.cc"], + deps = [ + ":graph_optimization_pass", + "//tensorflow/compiler/mlir:mlir_graph_optimization_pass", + "//tensorflow/compiler/mlir:mlir_graph_optimization_pass_registration", + ], + alwayslink = 1, +) + +cc_library( + name = "constant_fold_utils", + srcs = [ + "constant_fold_utils.cc", + ], + hdrs = [ + "constant_fold_utils.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_traits", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/tfrt/fallback:fallback_state", + "//tensorflow/core/tfrt/fallback:op_kernel_runner", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tf_dialect_passes", + srcs = [ + "constant_fold.cc", + "decode_attributes_hook.cc", + ], + hdrs = [ + "constant_fold.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":constant_fold_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core:direct_session", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/ops", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ] + if_not_tflite_converter([ + "//tensorflow/core:all_kernels", + ]), + alwayslink = 1, +) + +cc_library( + name = "tf_dialect_lib", + deps = [ + ":tf_dialect_passes", + "@llvm-project//mlir:AllPassesAndDialects", + ], +) + +cc_library( + name = "tf_graph_optimization_pass", + srcs = ["tf_graph_optimization_pass.cc"], + hdrs = ["tf_graph_optimization_pass.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tf2xla/api/v2:graph_to_tf_executor", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/compiler/tf2xla:functionalize_control_flow_pass_registration", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:ops", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +filegroup( + name = "tensorflow_optimize_td_files", + srcs = [ + "optimize.td", + ], +) + +gentbl_cc_library( + name = "tensorflow_optimize_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_optimize.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "optimize.td", + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:FuncTdFiles", + ], +) + +# This is a separate library so that external passes can link only this library +# without linking any of the other tensorflow passes. +gentbl_cc_library( + name = "lower_tf_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_lower_tf.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "lower_tf.td", + deps = [ + ":rewrite_util_td_files", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:FuncTdFiles", + ], +) + +cc_library( + name = "lower_tf_lib", + srcs = [ + "lower_tf.cc", + ], + hdrs = [ + "lower_tf.h", + ], + deps = [ + ":lower_tf_inc_gen", + ":rewrite_util", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core:framework", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "set_tpu_infeed_layout", + srcs = ["set_tpu_infeed_layout.cc"], + hdrs = ["set_tpu_infeed_layout.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@xla//xla:shape_util", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo", + "@xla//xla/tpu:c_api_conversions", + "@xla//xla/tpu:c_api_decl", + "@xla//xla/tpu:tpu_executor_api", + "@xla//xla/tpu:tpu_executor_c_api_hdrs", + ], +) + +cc_library( + name = "tf_saved_model_asset_sinking_pass", + srcs = ["tf_saved_model_asset_sinking_pass.cc"], + hdrs = ["tf_saved_model_asset_sinking_pass.h"], + deps = [ + ":tf_savedmodel_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tsl//tsl/platform:path", + ], +) + +tf_cc_test( + name = "shape_inference_test", + srcs = ["shape_inference_test.cc"], + deps = [ + ":shape_inference_pass", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@xla//xla/tsl/platform:env", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/add_functions_for_exported_names.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/add_functions_for_exported_names.cc new file mode 100644 index 0000000000000000000000000000000000000000..be3d7e7d673b170b47a4e05f5ca1e07d39f29631 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/add_functions_for_exported_names.cc @@ -0,0 +1,125 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +#define GEN_PASS_DEF_ADDFUNCTIONSFOREXPORTEDNAMESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +struct AddFunctionsForExportedNamesPass + : public impl::AddFunctionsForExportedNamesPassBase< + AddFunctionsForExportedNamesPass> { + void runOnOperation() override; +}; + +// Set the (array of) exported name(s) of a (public) function to just +// contain the given string. +void SetExportedName(func::FuncOp f, StringRef name) { + OpBuilder b(f); + f->removeAttr(kTfSavedModelExportedNamesAttr); + f->setAttr(kTfSavedModelExportedNamesAttr, b.getStrArrayAttr({name})); +} + +// Convert a savedmodel public function into a private function. +// This means we need to remove any attributes that are only allowed +// on exported (public) functions. +void Unexport(func::FuncOp f) { + f.setVisibility(mlir::SymbolTable::Visibility::Private); + f->removeAttr(kTfSavedModelExportedNamesAttr); + for (int i = 0; i < f.getNumArguments(); ++i) { + llvm::ArrayRef attrs = + mlir::function_interface_impl::getArgAttrs(f, i); + for (NamedAttribute a : attrs) { + if (a.getName().strref().starts_with("tf_saved_model.")) { + f.removeArgAttr(i, a.getName()); + } + } + } + for (int i = 0; i < f.getNumResults(); ++i) { + for (NamedAttribute a : f.getResultAttrs(i)) { + if (a.getName().strref().starts_with("tf_saved_model.")) { + f.removeResultAttr(i, a.getName()); + } + } + } +} + +void AddFunctionsForExportedNamesPass::runOnOperation() { + Block& module_body = getOperation().getRegion().front(); + for (auto f : + llvm::make_early_inc_range(getOperation().getOps())) { + auto exported_names = GetExportedNames(f); + if (exported_names.empty() || !f.isPublic()) continue; + + if (exported_names.size() == 1 && exported_names[0] == f.getName()) { + // Functions that already export themselves with their MLIR name + // we can leave alone. This saves one level of indirection. + return; + } + + f->removeAttr(kTfSavedModelExportedNamesAttr); // so we don't clone it + + // Rename to avoid name collisions with itself. + f.setName(StringAttr::get(f->getContext(), f.getName() + "_internal")); + + for (StringRef name : llvm::reverse(exported_names)) { + // Create a "trampoline" function with the given name. So given + // func bar(...) {exported_names = ["foo"]} + // we create + // func foo(...) { + // return bar(...) + // } + func::FuncOp other = f.cloneWithoutRegions(); + other.setName(name); + SetExportedName(other, name); + module_body.push_front(other); + other.addEntryBlock(); + OpBuilder builder(other.getRegion()); + auto call_op = mlir::func::CallOp::create( + builder, f.getLoc(), f.getFunctionType().getResults(), f.getSymName(), + other.getRegion().getArguments()); + mlir::func::ReturnOp::create(builder, f.getLoc(), call_op.getResults()); + } + + Unexport(f); + } +} + +} // namespace + +std::unique_ptr> +CreateAddFunctionsForExportedNamesPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/annotate_parameter_replication.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/annotate_parameter_replication.cc new file mode 100644 index 0000000000000000000000000000000000000000..3e546dca2b0c91cc762ec88e38ca91dcd9c3f726 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/annotate_parameter_replication.cc @@ -0,0 +1,109 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +constexpr char kReplicationAttr[] = "mhlo.is_same_data_across_replicas"; +constexpr char kMirroredVariableIndicesAttr[] = "_mirrored_variable_indices"; + +#define GEN_PASS_DEF_ANNOTATEPARAMETERREPLICATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +// Analyzes the inputs to ClusterFuncOps in the module, and annotates their +// invoked functions whether each input has the same data across replicas. +struct AnnotateParameterReplicationPass + : public impl::AnnotateParameterReplicationPassBase< + AnnotateParameterReplicationPass> { + void runOnOperation() override; +}; + +// Returns the first value in the chain of operands, which is not defined by a +// tf.IdentityOp or a tf.ReadVariableOp. +Value SkipIdentityAndReadVariable(Value v) { + while (auto op = v.getDefiningOp()) { + if (!isa(op)) break; + v = op->getOperand(0); + } + return v; +} + +void AnnotateParameterReplicationPass::runOnOperation() { + ModuleOp m = getOperation(); + OpBuilder builder(m.getContext()); + m.walk([&](tf_device::ClusterFuncOp cluster_func) { + auto replicate = cluster_func->getParentOfType(); + if (!replicate) return; + auto mirrored_variable_indices_attr = + replicate->getAttrOfType(kMirroredVariableIndicesAttr); + llvm::SmallDenseSet mirrored_replicate_args; + if (mirrored_variable_indices_attr) { + for (const auto& mirrored_index : mirrored_variable_indices_attr) { + mirrored_replicate_args.insert( + mlir::cast(mirrored_index).getInt()); + } + } + auto func = + llvm::cast(m.lookupSymbol(cluster_func.getFunc())); + for (auto entry : llvm::enumerate(cluster_func.getOperands())) { + auto operand = SkipIdentityAndReadVariable(entry.value()); + auto block_arg = mlir::dyn_cast(operand); + if (block_arg && block_arg.getOwner() == &replicate.GetBody()) { + // Only mirrored args of ReplicateOp can be annotated. + if (mirrored_replicate_args.count(block_arg.getArgNumber()) == 0) { + continue; + } + } else if (!operand.getParentRegion()->isProperAncestor( + &replicate.getBody())) { + // Not a replication-invariant operand. + continue; + } + func.setArgAttr(entry.index(), kReplicationAttr, + builder.getBoolAttr(true)); + } + }); +} + +} // namespace + +std::unique_ptr> +CreateAnnotateParameterReplicationPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/batchmatmul_to_einsum.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/batchmatmul_to_einsum.cc new file mode 100644 index 0000000000000000000000000000000000000000..56cad34b304b7a814f12157950bf66ca9b8fd24e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/batchmatmul_to_einsum.cc @@ -0,0 +1,109 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/matmul_bcast.h" + +namespace mlir { +namespace TF { + +namespace { + +// Replace TF BatchMatMul by TF Einsum op +template +class ConvertTFBatchMatMulToEinsumOp + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BatchMatMulOpType op, + PatternRewriter& rewriter) const override { + Value input_lhs = op.getX(); + Value input_rhs = op.getY(); + + // LHS and RHS must be a ranked tensor type + auto lhs_type = mlir::dyn_cast(input_lhs.getType()); + auto rhs_type = mlir::dyn_cast(input_rhs.getType()); + + if (!lhs_type || !rhs_type) return failure(); + + auto lhs_shape = lhs_type.getShape(); + auto rhs_shape = rhs_type.getShape(); + + // Ensure that input ranks are at least 2. + const int dims_a = lhs_shape.size(); + const int dims_b = rhs_shape.size(); + if (dims_a < 2 || dims_b < 2) { + return failure(); + } + + // einsum equation for batchmatmul + std::string equation("...mk,...kn->...mn"); + if (op.getAdjX()) std::swap(equation[3], equation[4]); + if (op.getAdjY()) std::swap(equation[6 + 3], equation[6 + 4]); + + rewriter.replaceOpWithNewOp( + op, op.getType(), + /*inputs=*/ValueRange({input_lhs, input_rhs}), + /*equation=*/equation); + + return success(); + } +}; + +#define GEN_PASS_DEF_BATCHMATMULTOEINSUMPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct BatchMatMulToEinsumPass + : public impl::BatchMatMulToEinsumPassBase { + void runOnOperation() override; +}; + +void BatchMatMulToEinsumPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + auto func = getOperation(); + + patterns.add, + ConvertTFBatchMatMulToEinsumOp>( + &getContext()); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +std::unique_ptr> CreateBatchMatMulToEinsumPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/breakup-islands.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/breakup-islands.cc new file mode 100644 index 0000000000000000000000000000000000000000..60cfcbedda3dec1a28a00a47fbccd0e522caca17 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/breakup-islands.cc @@ -0,0 +1,396 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +// This pass is used in preparation for Graph export. +// The GraphDef exporter expects each op to be in its own island. +// This pass puts the IR in that form. +// +// We do this as an IR->IR transform to keep the Graph exporter as simple as +// possible. + +namespace mlir { + +namespace { + +class BreakUpIslands : public TF::PerFunctionAggregateAnalysisConsumerPass< + BreakUpIslands, TF::SideEffectAnalysis> { + void getDependentDialects(DialectRegistry& registry) const override { + registry.insert(); + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(BreakUpIslands) + + StringRef getArgument() const final { return "tf-executor-break-up-islands"; } + + StringRef getDescription() const final { + return "Transform from TF control dialect to TF executor dialect."; + } + + void runOnFunction(func::FuncOp func, + const TF::SideEffectAnalysis::Info& side_effect_analysis); + + void BreakUpIsland(tf_executor::IslandOp island_op, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + llvm::DenseMap>* + new_control_inputs); +}; + +// Returns true if the operation is a stateful If, Case, or While op. +bool IsStatefulFunctionalControlFlowOp(Operation* op) { + if (!isa(op)) { + return false; + } + + if (auto is_stateless = op->getAttrOfType("is_stateless")) { + return !is_stateless.getValue(); + } + return false; +} + +// Add control dependencies from stateful control-flow ops to graph fetch op. +// This is needed to avoid that such control-flow ops get pruned because of a +// bug in common runtime (see b/185483669). +void AddStatefulControlFlowDependencies(tf_executor::GraphOp graph_op) { + llvm::SmallDenseSet graph_fetches; + for (Value value : graph_op.GetFetch().getFetches()) { + graph_fetches.insert(value); + } + for (Operation& op : graph_op.GetBody().without_terminator()) { + auto island = dyn_cast(&op); + if (!island) continue; + if (!island.WrapsSingleOp()) continue; + Operation& wrapped_op = island.GetBody().front(); + if (!IsStatefulFunctionalControlFlowOp(&wrapped_op)) continue; + if (graph_fetches.contains(island.getControl())) continue; + + graph_op.GetFetch().getFetchesMutable().append(island.getControl()); + } +} + +void BreakUpIslands::runOnFunction( + func::FuncOp func, + const TF::SideEffectAnalysis::Info& side_effect_analysis) { + auto graph_op_range = func.front().without_terminator(); + tf_executor::GraphOp graph_op; + + if (llvm::hasSingleElement(graph_op_range)) + graph_op = dyn_cast(func.front().front()); + + if (!graph_op) { + func.emitError("expected function to contain only a graph_op"); + signalPassFailure(); + return; + } + + // New control inputs to be added. For an operation x, new_control_inputs[x] + // contains all control inputs that need to be added to x as operands. + llvm::DenseMap> new_control_inputs; + // Iterate in reverse order to avoid invalidating Operation* stored in + // new_control_inputs. + for (auto& item : + llvm::make_early_inc_range(llvm::reverse(graph_op.GetBody()))) { + if (auto island = dyn_cast(&item)) { + BreakUpIsland(island, side_effect_analysis, &new_control_inputs); + } + } + OpBuilder builder(func); + + // For every op, add new control inputs in reverse order so that the ops don't + // get invalidated. + llvm::SmallVector operands; + llvm::SmallPtrSet defining_ops; + llvm::SmallVector types; + for (auto& item : + llvm::make_early_inc_range(llvm::reverse(graph_op.GetBody()))) { + auto it = new_control_inputs.find(&item); + if (it == new_control_inputs.end()) continue; + auto& new_control_inputs_for_item = it->second; + builder.setInsertionPoint(&item); + OperationState state(item.getLoc(), item.getName()); + types.assign(item.result_type_begin(), item.result_type_end()); + state.addTypes(types); + for (Region& region : item.getRegions()) { + state.addRegion()->takeBody(region); + } + // Assign existing operands for item. + operands.assign(item.operand_begin(), item.operand_end()); + + // Collect defining ops for existing operands. + defining_ops.clear(); + for (Value operand : operands) { + defining_ops.insert(operand.getDefiningOp()); + } + for (Value new_control_input : llvm::reverse(new_control_inputs_for_item)) { + // Add new control input if its defining op is not already a defining + // op for some other operand. Update defining_ops. + if (defining_ops.insert(new_control_input.getDefiningOp()).second) { + operands.push_back(new_control_input); + } + } + state.addOperands(operands); + Operation* new_op = builder.create(state); + item.replaceAllUsesWith(new_op); + new_op->setAttrs(item.getAttrDictionary()); + item.erase(); + } + AddStatefulControlFlowDependencies(graph_op); +} + +// Populates an empty IslandOp and with a NoOp or Identity/IdentityN depending +// on if there are any data results. +void PopulateEmptyIsland(tf_executor::IslandOp island) { + OpBuilder builder(&island.GetBody(), island.GetBody().begin()); + tf_executor::YieldOp yield = island.GetYield(); + if (yield.getNumOperands() == 0) { + TF::NoOp::create(builder, island.getLoc(), TypeRange{}, ValueRange{}); + } else if (yield.getNumOperands() == 1) { + Value operand = yield.getOperand(0); + auto identity = TF::IdentityOp::create(builder, island.getLoc(), + operand.getType(), operand); + yield.setOperand(0, identity.getOutput()); + } else { + auto identity_n = TF::IdentityNOp::create( + builder, island.getLoc(), yield.getOperandTypes(), yield.getOperands()); + for (auto it : llvm::enumerate(identity_n.getResults())) + yield.setOperand(it.index(), it.value()); + } +} + +// Helper that creates an island. If `sub_op` is not nullptr, it will be moved +// to the island. Otherwise a NoOp will be added to the island. +tf_executor::IslandOp CreateIsland(TypeRange result_types, + ValueRange control_inputs, + const tf_executor::ControlType& control_type, + const Location& loc, Operation* sub_op, + tf_executor::IslandOp original_island) { + OpBuilder builder(original_island); + auto island = tf_executor::IslandOp::create(builder, loc, result_types, + control_type, control_inputs); + island.getBody().push_back(new Block); + Block* block = &island.getBody().back(); + OpBuilder island_builder(original_island); + island_builder.setInsertionPointToEnd(block); + if (sub_op) { + sub_op->replaceAllUsesWith(island.getOutputs()); + sub_op->moveBefore(block, block->begin()); + tf_executor::YieldOp::create(island_builder, loc, sub_op->getResults()); + } else { + TF::NoOp::create(island_builder, island.getLoc(), TypeRange{}, + ValueRange{}); + tf_executor::YieldOp::create(island_builder, loc, ValueRange{}); + } + return island; +} + +// A struct that contains the operations in an island that need explicit control +// dependencies added going into and out of the island to capture inter-island +// dependencies properly. +struct IslandSourcesAndSinks { + // Sub-ops that need a control dependency going into the island. This includes + // sub-ops that do not depend on other sub-ops in the island and functional + // control ops (e.g. if, while, case) with side effects that must not take + // effect before the previous island is finished executing. + llvm::SmallPtrSet sources; + + // Sub-ops that need a control dependency going out of the island. This + // includes sub-ops that do not have other sub-ops in the island depending on + // them (excluding yield) and functional control ops (e.g. if, while, case) + // with side effects that must take effect before the next island starts + // executing. + llvm::SmallPtrSet sinks; +}; + +// Finds IslandSourcesAndSinks for an unmodified island. +IslandSourcesAndSinks FindSourcesAndSinksInIsland( + tf_executor::IslandOp island, + const TF::SideEffectAnalysis::Info& side_effect_analysis) { + IslandSourcesAndSinks result; + auto island_body = island.GetBody().without_terminator(); + for (Operation& sub_op : island_body) { + auto predecessors = side_effect_analysis.DirectControlPredecessors(&sub_op); + result.sinks.insert(&sub_op); + // Remove predecessor from sinks. + for (auto predecessor : predecessors) result.sinks.erase(predecessor); + bool has_in_island_operands = false; + for (auto operand : sub_op.getOperands()) { + auto defining_op = operand.getDefiningOp(); + if (!defining_op || defining_op->getParentOp() != island) continue; + has_in_island_operands = true; + + // Remove operands from sinks. + // We don't remove the operand if it is a stateful functional control flow + // op to work around an issue in LowerFunctionalOpsPass where the operand + // dependency isn't enough to ensure the side effects take place + // (b/185483669). + if (!IsStatefulFunctionalControlFlowOp(defining_op)) { + result.sinks.erase(defining_op); + } + } + if (predecessors.empty() && (!has_in_island_operands || + IsStatefulFunctionalControlFlowOp(&sub_op))) { + result.sources.insert(&sub_op); + } + } + return result; +} + +// Converts a single island into multiple islands (one for each op). The islands +// are chained together by control flow values. +void BreakUpIslands::BreakUpIsland( + tf_executor::IslandOp island_op, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + llvm::DenseMap>* + new_control_inputs) { + auto island_body = island_op.GetBody().without_terminator(); + // Populate islands that are empty (only yield). + if (island_body.empty()) { + PopulateEmptyIsland(island_op); + return; + } + + // Skip islands that are already only a single op. + if (island_op.WrapsSingleOp()) return; + + auto control_type = tf_executor::ControlType::get(&getContext()); + auto island_control_inputs = llvm::to_vector<4>(island_op.getControlInputs()); + // Add control dependencies for yields of values defined by other islands to + // the island that defines that fetched value. + for (auto fetch : island_op.GetYield().getFetches()) { + if (!fetch.getDefiningOp()) { + // Skip, because there is no op to add control to (eg: function args). + continue; + } else if (fetch.getDefiningOp()->getParentOp() == island_op) { + // Skip, because it is the same island. + continue; + } else if (auto other_island_op = llvm::dyn_cast( + fetch.getDefiningOp())) { + island_control_inputs.push_back(other_island_op.getControl()); + } else { + // TODO(parkers): Any defining op that has a control output can be handled + // just like an island. + fetch.getDefiningOp()->emitError("fetching non-island as dependency"); + return signalPassFailure(); + } + } + // If there are multiple control inputs, create an empty island to group them. + if (island_control_inputs.size() > 1) { + auto new_island = CreateIsland({}, island_control_inputs, control_type, + island_op.getLoc(), nullptr, island_op); + island_control_inputs.clear(); + island_control_inputs.push_back(new_island.getControl()); + } + // Find sources and sinks inside the original island. + IslandSourcesAndSinks sources_and_sinks = + FindSourcesAndSinksInIsland(island_op, side_effect_analysis); + // The corresponding control output of the new island created for each sub-op. + llvm::SmallDenseMap new_control_for_sub_ops; + // Control outputs of newly created islands that are sinks. + llvm::SmallVector sink_island_controls; + // For each operation in the island, construct a new island to wrap the op, + // yield all the results, and replace all the usages with the results of the + // new island. + for (auto& sub_op : llvm::make_early_inc_range(island_body)) { + const auto predecessors = + side_effect_analysis.DirectControlPredecessors(&sub_op); + // Get the controls from the predecessors. + llvm::SmallVector predecessor_controls; + predecessor_controls.reserve(predecessors.size()); + for (auto predecessor : predecessors) { + predecessor_controls.push_back(new_control_for_sub_ops[predecessor]); + } + // If sub_op is a source, use island_control_inputs, because that's required + // by inter-islands dependencies; otherwise, we do not need to include + // island_control_inputs, since they must have been tracked by the (direct + // or indirect) control predecessors or operands. + ArrayRef control = sources_and_sinks.sources.count(&sub_op) > 0 + ? island_control_inputs + : predecessor_controls; + auto new_island = + CreateIsland(sub_op.getResultTypes(), control, control_type, + sub_op.getLoc(), &sub_op, island_op); + new_control_for_sub_ops[&sub_op] = new_island.getControl(); + if (sources_and_sinks.sinks.count(&sub_op)) { + sink_island_controls.push_back(new_island.getControl()); + } + } + // Create control outputs for the sinks. + assert(!sink_island_controls.empty()); + // If there are multiple control outputs, create an empty island to group + // them. + if (sink_island_controls.size() > 1) { + auto new_island = CreateIsland({}, sink_island_controls, control_type, + island_op.getLoc(), nullptr, island_op); + sink_island_controls.clear(); + sink_island_controls.push_back(new_island.getControl()); + } + assert(sink_island_controls.size() == 1); + auto& sink_island_control = sink_island_controls[0]; + island_op.getControl().replaceAllUsesWith(sink_island_control); + // All existing outputs need to add sink_island_control as control input. + // GraphOp, YieldOp and NextIterationSourceOp don't have control inputs so + // exclude them below. + for (Value out : island_op.getOutputs()) { + for (auto& use : out.getUses()) { + Operation* owner = use.getOwner(); + if (owner->getDialect() == island_op->getDialect() && + !llvm::isa(owner)) { + (*new_control_inputs)[owner].push_back(sink_island_control); + // Note that we cannot assume that the island containing `owner` is a + // direct parent: + // For example, ops with regions usually don't expose values used in a + // region to the op's interface which means that the usage of a value + // can be 2 or more levels below an island (see b/242920486). + } else if (auto other_island_op = + owner->getParentOfType()) { + (*new_control_inputs)[other_island_op].push_back(sink_island_control); + } else { + owner->emitOpError("adding control dependency not supported"); + return signalPassFailure(); + } + } + } + for (auto item : + llvm::zip(island_op.getOutputs(), island_op.GetYield().getFetches())) + std::get<0>(item).replaceAllUsesWith(std::get<1>(item)); + island_op.erase(); +} + +} // namespace + +std::unique_ptr> CreateBreakUpIslandsPass() { + return std::make_unique(); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc new file mode 100644 index 0000000000000000000000000000000000000000..8c9d53d9374b4121964489bced25fe7a2dbb226d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc @@ -0,0 +1,60 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/bridge.h" + +#include "absl/log/log.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/core_platform_payloads.pb.h" + +namespace mlir { +namespace TF { + +absl::Status RunBridgeWithStandardPipeline(ModuleOp module, bool enable_logging, + bool enable_inliner) { + PassManager bridge(module.getContext()); + + StandardPipelineOptions pipeline_options; + pipeline_options.enable_inliner.setValue(enable_inliner); + CreateTFStandardPipeline(bridge, pipeline_options); + + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + constexpr char kBridgeComponent[] = "TFXLABridge"; + if (enable_logging || VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile(kStandardPipelineBefore, module, "", &bridge); + if (VLOG_IS_ON(2)) { + tensorflow::tf2xla::internal::EnablePassIRPrinting(bridge, + kBridgeComponent); + } + } + LogicalResult result = bridge.run(module); + (void)result; + if (enable_logging || VLOG_IS_ON(1)) + tensorflow::DumpMlirOpToFile(kStandardPipelineAfter, module, "", &bridge); + return diag_handler.ConsumeStatus(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.h new file mode 100644 index 0000000000000000000000000000000000000000..8e5406481e3895f72bc24834e217b52b9a3db384 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/bridge.h @@ -0,0 +1,47 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_BRIDGE_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_BRIDGE_H_ + +#include + +#include "absl/base/attributes.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/lib/core/status.h" + +namespace mlir { +namespace TF { + +inline constexpr char kStandardPipelineBefore[] = "standard_pipeline_before"; +inline constexpr char kStandardPipelineAfter[] = "standard_pipeline_after"; + +// Runs all passes involved in transforming or optimizing an MLIR graph without +// any target specialization. When enable_logging is true, enables +// tensorflow::BridgeLogger. When enable_inliner is true, enables the inliner +// pass. +ABSL_DEPRECATED( + "This is legacy code and is unsupported. Use at your own risk. Use " + "tf2xla/api/v2/* for specific functionality") +absl::Status RunBridgeWithStandardPipeline(ModuleOp module, bool enable_logging, + bool enable_inliner); +} // namespace TF + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_BRIDGE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize.td new file mode 100644 index 0000000000000000000000000000000000000000..7456bb8d671a4e873110380c795a22c7ec6a0eda --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize.td @@ -0,0 +1,396 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is the canonicalize pattern definition file. + +include "mlir/IR/OpBase.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" +include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td" + +/// TODO(b/130756570): Support OpBase constraints in PatternRewrites. +def SingleResultAndOperandHaveSameElementType : Constraint< + CPred<"getElementTypeOrSelf($0) == getElementTypeOrSelf($1)">>; + +def SingleResultAndOperandHaveSameType : Constraint< + CPred<"$0.getType() == $1.getType()">>; + +def IsRank2Tensor : Type, "Rank 2 tensor">; + +def IsRank4Tensor : Type, "Rank 4 tensor">; + +// Checks if all the users is ReadVariableOp. +def HasOnlyReadVariableOpUsers : Constraint< + CPred<"llvm::all_of($0.getUsers(), [](mlir::OpOperand op) { " + "return llvm::isa(op.getOwner()); })">>; + +def CopyAttrs: NativeCodeCallVoid<"CopyDeviceAndUnderscoredAttributesAdaptor($0, $1)">; + +//===----------------------------------------------------------------------===// +// Add op patterns. +//===----------------------------------------------------------------------===// + +def AddToAddV2 : Pat< + (TF_AddOp:$src TF_NumberTensor:$arg0, TF_NumberTensor:$arg1), + (TF_AddV2Op:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// AddV2 op patterns. +//===----------------------------------------------------------------------===// + +def AddV2OfNegLeft : Pat< + (TF_AddV2Op:$src (TF_NegOp $arg0), $arg1), (TF_SubOp:$dest $arg1, $arg0), + [], [(CopyAttrs $src, $dest)]>; + +def AddV2OfNegRight : Pat< + (TF_AddV2Op:$src $arg0, (TF_NegOp $arg1)), (TF_SubOp:$dest $arg0, $arg1), [], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Gather op patterns. +//===----------------------------------------------------------------------===// + +class GetI32Attr: NativeCodeCall< + "$_builder.getI32IntegerAttr(" # x # ")">; + +class GetI64Attr: NativeCodeCall< + "$_builder.getI64IntegerAttr(" # x # ")">; + +// The deprecated validate_indices attribute is NOT verified during +// canonicalization because it is neither supported in the old bridge +// nor in the reference TF implementation. +def GatherToV2 : Pat< + (TF_GatherOp:$src $params, $indices, $ignored_validate_indices), + (TF_GatherV2Op:$dest $params, $indices, (TF_ConstOp (GetI32Attr<0>)), + (GetI64Attr<0>)), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// BatchMatMul op patterns. +//===----------------------------------------------------------------------===// + +// Static shaped operands in a legal BatchMatMul op will have matching batch +// dimensions and can be upgraded to the BatchMatMulV2 op. Canonicalizing +// dynamically shaped operands is not correct as that will execute ops that +// have non matching batch dimensions but are broadcastable which should fail +// with V1. +def BatchMatMulToV2 : Pat< + (TF_BatchMatMulOp:$src AnyStaticShapeTensor:$x, AnyStaticShapeTensor:$y, + $adj_x, $adj_y, $grad_x, $grad_y), + (TF_BatchMatMulV2Op:$dest $x, $y, $adj_x, $adj_y, $grad_x, $grad_y), + [], [(CopyAttrs $src, $dest)]>; + +def BatchMatMulToMatMul : Pat< + (TF_BatchMatMulOp:$src $x, $y, $adj_x, $adj_y, $grad_x, $grad_y), + (TF_MatMulOp:$dest $x, $y, $adj_x, $adj_y, $grad_x, $grad_y), + [(IsRank2Tensor $x), (IsRank2Tensor $y)], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// BatchMatMulV2 op patterns. +//===----------------------------------------------------------------------===// + +def BatchMatMulV2ToMatMul : Pat< + (TF_BatchMatMulV2Op:$src $x, $y, $adj_x, $adj_y, $grad_x, $grad_y), + (TF_MatMulOp:$dest $x, $y, $adj_x, $adj_y, $grad_x, $grad_y), + [(IsRank2Tensor $x), (IsRank2Tensor $y)], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// BatchToSpace op patterns. +//===----------------------------------------------------------------------===// + +def BatchToSpaceBlockSizeToBlockShape : NativeCodeCall< + "DenseElementsAttr::get(RankedTensorType::get({2}, $_builder.getI64Type()), " + "ArrayRef{$0.getValue(), $0.getValue()})">; + +def BatchToSpaceToBatchToSpaceND : Pat< + (TF_BatchToSpaceOp:$src $input, $crops, $block_size), + (TF_BatchToSpaceNDOp:$dest $input, + (TF_ConstOp (BatchToSpaceBlockSizeToBlockShape $block_size)), $crops), + [(IsRank4Tensor $input), (IsRank2Tensor $crops)], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// BiasAddV1 op patterns. +//===----------------------------------------------------------------------===// + +def BiasAddV1ToBiasAdd : Pat< + (TF_BiasAddV1Op:$src $arg0, $arg1), + (TF_BiasAddOp:$dest $arg0, $arg1, + ConstantAttr), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Bitcast op patterns. +//===----------------------------------------------------------------------===// + +// Parameter attributes are part of function attributes, which is not affected +// by pattern rewrites. Therefore we do not need to explicilty copy attributes +// over. +def BitcastSameType : Pat< + (TF_BitcastOp:$res $arg), + (replaceWithValue $arg), + [(SingleResultAndOperandHaveSameElementType $res, $arg)]>; + +def BitcastNested : Pat< + (TF_BitcastOp:$src (TF_BitcastOp $arg)), (TF_BitcastOp:$dest $arg), [], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Concat op patterns. +//===----------------------------------------------------------------------===// + +def ConvertToConcatV2 : Pat< + (TF_ConcatOp:$src $axis, $inputs), (TF_ConcatV2Op:$dest $inputs, $axis), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Div op patterns. +//===----------------------------------------------------------------------===// + +/// Favor Mul over Div. +def DivWithSqrtDivisor : Pat< + (TF_DivOp:$src $arg0, (TF_SqrtOp $arg1)), + (TF_MulOp:$dest1 $arg0, (TF_RsqrtOp:$dest2 $arg1)), [], + [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +//===----------------------------------------------------------------------===// +// Log op patterns. +//===----------------------------------------------------------------------===// + +def LogOfSoftmax : Pat< + (TF_LogOp:$src (TF_SoftmaxOp $arg)), (TF_LogSoftmaxOp:$dest $arg), [], + [(CopyAttrs $src, $dest)]>; + +// Canonicalize: Log(1.0 + x) to Log1p(x) +// +// We currently do this rewrite only if the constant `1` is a scalar, because +// it is safely broadcastable to any shape. To be able to canonicalize when +// constant values is not a scalar, we have to first prove that it is +// broadcastable to `x`, which requires static shape information. +def LogToLog1p : Pat< + (TF_LogOp:$src (TF_AddV2Op $arg, + (TF_ConstOp ConstantAttr, "1.0f">))), + (TF_Log1pOp:$dest $arg), [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// LogicalNot op patterns. +//===----------------------------------------------------------------------===// + +def LogicalNotOfEqual : Pat< + (TF_LogicalNotOp:$src (TF_EqualOp $arg0, $arg1, $shape_error)), + (TF_NotEqualOp:$dest $arg0, $arg1, $shape_error), [], + [(CopyAttrs $src, $dest)]>; + +def LogicalNotOfNotEqual : Pat< + (TF_LogicalNotOp:$src (TF_NotEqualOp $arg0, $arg1, $shape_error)), + (TF_EqualOp:$dest $arg0, $arg1, $shape_error), [], [(CopyAttrs $src, $dest)]>; + +def LogicalNotOfGreater : Pat< + (TF_LogicalNotOp:$src (TF_GreaterOp $arg0, $arg1)), + (TF_LessEqualOp:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +def LogicalNotOfGreaterEqual : Pat< + (TF_LogicalNotOp:$src (TF_GreaterEqualOp $arg0, $arg1)), + (TF_LessOp:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +def LogicalNotOfLess : Pat<( + TF_LogicalNotOp:$src (TF_LessOp $arg0, $arg1)), + (TF_GreaterEqualOp:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +def LogicalNotOfLessEqual : Pat< + (TF_LogicalNotOp:$src (TF_LessEqualOp $arg0, $arg1)), + (TF_GreaterOp:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// MatrixSetDiag op patterns. +//===----------------------------------------------------------------------===// + +class GetStrAttr: NativeCodeCall< + "$_builder.getStringAttr(\"" # x # "\")">; + +def MatrixSetDiagToV3 : Pat< + (TF_MatrixSetDiagOp:$src $input, $diag), + (TF_MatrixSetDiagV3Op:$dest $input, $diag, (TF_ConstOp (GetI32Attr<0>)), + (GetStrAttr<"RIGHT_LEFT">)), + [], [(CopyAttrs $src, $dest)]>; + +// MatrixSetDiagToV2 op implicitly used LEFT_LEFT alignment. +def MatrixSetDiagV2ToV3 : Pat< + (TF_MatrixSetDiagV2Op:$src $input, $diag, $k), + (TF_MatrixSetDiagV3Op:$dest $input, $diag, $k, (GetStrAttr<"LEFT_LEFT">)), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// MatrixDiag Op. +//===----------------------------------------------------------------------===// + +def MatrixDiagToV3 : Pat< + (TF_MatrixDiagOp:$src $diag), + (TF_MatrixDiagV3Op:$dest $diag, (TF_ConstOp (GetI32Attr<0>)), + (TF_ConstOp (GetI32Attr<-1>)), (TF_ConstOp (GetI32Attr<-1>)), + (TF_ConstOp (GetScalarOfType<0> $diag)), (GetStrAttr<"RIGHT_LEFT">)), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// QuantizeAndDequantizeV2 op patterns. +//===----------------------------------------------------------------------===// + +def QuantizeAndDequantizeV2ToQuantizeAndDequantizeV4 : Pat< + (TF_QuantizeAndDequantizeV2Op:$src $inputs, $min, $max, $signed_input, + $num_bits, $range_given, $round_mode, $narrow_range, $axis), + (TF_QuantizeAndDequantizeV4Op:$dest $inputs, $min, $max, $signed_input, + $num_bits, $range_given, $round_mode, $narrow_range, $axis), + [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// RealDiv op patterns. +//===----------------------------------------------------------------------===// + +def RealDivWithSqrtDivisor : Pat< + (TF_RealDivOp:$src $arg0, (TF_SqrtOp $arg1)), + (TF_MulOp:$dest1 $arg0, (TF_RsqrtOp:$dest2 $arg1)), + [], [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +// Replace division by a constant with a multiplication by a reciprocal of that +// constant. Floating point division can be ~10x more expensive than a +// multiplication. +def RealDivWithConstDivisor : Pat< + (TF_RealDivOp:$src $arg0, (TF_ConstOp FloatElementsAttr<32>:$value)), + (TF_MulOp:$dest1 $arg0, (TF_ReciprocalOp:$dest2 (TF_ConstOp $value))), + [], [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +//===----------------------------------------------------------------------===// +// Reshape op patterns. +//===----------------------------------------------------------------------===// + +def RedundantReshape : Pat< + (TF_ReshapeOp:$src (TF_ReshapeOp $arg, $unused), $shape), + (TF_ReshapeOp:$dest $arg, $shape), [], [(CopyAttrs $src, $dest)]>; + +def ReshapeToSelfShape : Pat< + (TF_ReshapeOp:$op $x, (TF_ShapeOp $x)), (replaceWithValue $x), + [(SingleResultAndOperandHaveSameType $x, $op)]>; + +//===----------------------------------------------------------------------===// +// Square op patterns. +//===----------------------------------------------------------------------===// + +// Restrict the pattern to the types supported by SquaredDifferenceOp. +def TF_NumberTensorNoSmallInt: TensorOf<[TF_Bfloat16, TF_Complex128, TF_Complex64, + TF_Float16, TF_Float32, TF_Int32, TF_Int64]>; + +def SquareOfSub : Pat< + (TF_SquareOp:$src (TF_SubOp TF_NumberTensorNoSmallInt:$arg0, + TF_NumberTensorNoSmallInt:$arg1)), + (TF_SquaredDifferenceOp:$dest $arg0, $arg1), [], [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Sub op patterns. +//===----------------------------------------------------------------------===// + +def SubOfNeg : Pat< + (TF_SubOp:$src $arg0, (TF_NegOp $arg1)), (TF_AddV2Op:$dest $arg0, $arg1), [], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// TruncateDiv op patterns. +//===----------------------------------------------------------------------===// + +def TruncateDivWithSqrtDivisor : Pat< + (TF_TruncateDivOp:$src $arg0, (TF_SqrtOp $arg1)), + (TF_MulOp:$dest1 $arg0, (TF_RsqrtOp:$dest2 $arg1)), [], + [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +//===----------------------------------------------------------------------===// +// Xdivy op patterns. +//===----------------------------------------------------------------------===// + +def XdivyWithSqrtDivisor : Pat< + (TF_XdivyOp:$src $arg0, (TF_SqrtOp $arg1)), + (TF_MulNoNanOp:$dest1 (TF_RsqrtOp:$dest2 $arg1), $arg0), [], + [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + + +//===----------------------------------------------------------------------===// +// Cast op followed by a ReadVariable op can be folded into the ReadVariable +//===----------------------------------------------------------------------===// + +def ReadVariableOfCast : Pat< + (TF_ReadVariableOp:$src (TF_CastOp:$output $x, BoolAttr:$Truncate)), + (TF_ReadVariableOp:$dest $x), [(HasOnlyReadVariableOpUsers $output)], + [(CopyAttrs $src, $dest)]>; + +//===----------------------------------------------------------------------===// +// Canonicalize tf.Variable ops to tf.VariableV2 ops +//===----------------------------------------------------------------------===// + +def VariableToVariableV2 : Pat< + (TF_VariableOp:$src $shape, $container, $shard_name), + (TF_VariableV2Op:$dest $shape, $container, $shard_name), + [], [(CopyAttrs $src, $dest)]>; + + +//===----------------------------------------------------------------------===// +// Canonicalize tf.HashTable and its user ops to V2 versions. +//===----------------------------------------------------------------------===// +def HashTableAndInitializeTableToV2 : Pat< + (TF_InitializeTableFromTextFileOp:$src (TF_HashTableOp $container, + $shared_name, $use_node_name_sharing, $key_dtype, $value_dtype), $filename, + $key_index, $value_index, $vocab_size, $delimiter, $offset), + (TF_InitializeTableFromTextFileV2Op:$dest1 (TF_HashTableV2Op:$dest2 $container, + $shared_name, $use_node_name_sharing, $key_dtype, $value_dtype), + $filename, $key_index, $value_index, $vocab_size, $delimiter, $offset), + [], [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +def HashTableAndLookupTableSizeToV2 : Pat< + (TF_LookupTableSizeOp:$src (TF_HashTableOp $container, $shared_name, + $use_node_name_sharing, $key_dtype, $value_dtype)), + (TF_LookupTableSizeV2Op:$dest1 (TF_HashTableV2Op:$dest2 $container, $shared_name, + $use_node_name_sharing, $key_dtype, $value_dtype)), + [], [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +def HashTableAndLookupTableFindToV2 : Pat< + (TF_LookupTableFindOp:$src (TF_HashTableOp $container, $shared_name, + $use_node_name_sharing, $key_dtype, $value_dtype), $keys, $default_value), + (TF_LookupTableFindV2Op:$dest1 (TF_HashTableV2Op:$dest2 $container, $shared_name, + $use_node_name_sharing, $key_dtype, $value_dtype), $keys, $default_value), + [], [(CopyAttrs $src, $dest1), (CopyAttrs $src, $dest2)]>; + +//===----------------------------------------------------------------------===// +// Canonicalize tf.Maximum of zero to tf.Relu +//===----------------------------------------------------------------------===// + +def IsInteger32Pred: CPred< + "getElementTypeOrSelf($0.getType()).isInteger(32)">; + +// Whether the transformation is compatible with the device if given. +// Currently, Relu with int32 is not supported on GPU. +def IsDeviceCompatible: Constraint< + Neg>>; + +// TODO(b/239736969): Remove the device compatibility constraint to make the +// canonicalization device independent, by adding an int32 GPU kernel for Relu. +def MaximumOfZeroToRelu : Pat< + (TF_MaximumOp:$maximum_op $x, $y), (TF_ReluOp:$dest $x), + [(IsConstantValueOf<0> $y), (IsDeviceCompatible $maximum_op)], + [(CopyAttrs $maximum_op, $dest)]>; + +//===----------------------------------------------------------------------===// +// Canonicalize tf.Relu of Minimul six to tf.Relu6 +//===----------------------------------------------------------------------===// + +def ReluOfMinimum6ToRelu6 : Pat< + (TF_ReluOp:$src (TF_MinimumOp $x, $y)), (TF_Relu6Op:$dest $x), + [(IsConstantValueOf<6> $y)], [(CopyAttrs $src, $dest)]>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize_compile_and_replicate_attributes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize_compile_and_replicate_attributes.cc new file mode 100644 index 0000000000000000000000000000000000000000..5bbbac34177c2ff63a17617c1fa9fb86009bb746 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/canonicalize_compile_and_replicate_attributes.cc @@ -0,0 +1,115 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass converts existing compilation and replication +// attributes into unified attributes. For example, A _tpu_replicate=X +// should be replaced with _xla_compile_device_type=TPU and +// _replication_info=X attributes by the conversion. An _XlaMustCompile=true +// should be replaced with _xla_compile_device_type with the value of device +// attribute. + +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Debug.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/core/util/device_name_utils.h" + +#define DEBUG_TYPE "tf-canonicalize-compile-and-replicate-attributes" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_CANONICALIZECOMPILEANDREPLICATEATTRIBUTESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct CanonicalizeCompileAndReplicateAttributesPass + : public impl::CanonicalizeCompileAndReplicateAttributesPassBase< + CanonicalizeCompileAndReplicateAttributesPass> { + void runOnOperation() override; +}; + +void CanonicalizeCompileAndReplicateAttributesPass::runOnOperation() { + func::FuncOp func_op = getOperation(); + ModuleOp module_op = func_op->getParentOfType(); + mlir::OpBuilder builder(module_op.getContext()); + + auto walk_result = func_op->walk([&](mlir::Operation* op) { + // Convert `_tpu_replicate`. + if (op->hasAttr(TF::kTpuReplicateAttr)) { + op->setAttr(tensorflow::kReplicationInfoAttr, + op->getAttr(TF::kTpuReplicateAttr)); + op->removeAttr(tensorflow::kTpuReplicateAttr); + op->setAttr(tensorflow::kCompileDeviceTypeAttr, + builder.getStringAttr(tensorflow::kTpuDevice)); + } + + // Convert `_XlaMustCompile`. + if (op->hasAttr(tensorflow::kMustCompileAttr)) { + bool must_compile_attr_val = + op->getAttrOfType(tensorflow::kMustCompileAttr).getValue(); + op->removeAttr(tensorflow::kMustCompileAttr); + if (!must_compile_attr_val) { + if (op->hasAttr(tensorflow::kCompileDeviceTypeAttr)) { + op->emitOpError() + << "has both '" << tensorflow::kMustCompileAttr + << " = false' and '" << tensorflow::kCompileDeviceTypeAttr + << "' attribute which contradicts each other"; + return mlir::WalkResult::interrupt(); + } + return mlir::WalkResult::advance(); + } + if (op->hasAttr(tensorflow::kCompileDeviceTypeAttr)) { + return mlir::WalkResult::advance(); + } + auto device_attr = op->getAttrOfType(tensorflow::kDeviceAttr); + if (!device_attr) { + op->setAttr(tensorflow::kCompileDeviceTypeAttr, + builder.getStringAttr(tensorflow::kEmptyDevice)); + return mlir::WalkResult::advance(); + } + tensorflow::DeviceNameUtils::ParsedName parsed_name; + tensorflow::DeviceNameUtils::ParseFullOrLocalName(device_attr.getValue(), + &parsed_name); + auto device_type = builder.getStringAttr(parsed_name.type); + if (failed(IsValidDeviceTypeOrEmpty(device_type))) { + op->emitOpError() << "'" << tensorflow::kDeviceAttr << "'" + << " has invalid value"; + return mlir::WalkResult::interrupt(); + } + op->setAttr(tensorflow::kCompileDeviceTypeAttr, device_type); + } + + return mlir::WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) signalPassFailure(); +} + +} // namespace + +std::unique_ptr> +CreateCanonicalizeCompileAndReplicateAttributesPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/check_control_dependencies.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/check_control_dependencies.cc new file mode 100644 index 0000000000000000000000000000000000000000..db8297eaad8f82f5c9e3628dec0251d5cb777a59 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/check_control_dependencies.cc @@ -0,0 +1,265 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" + +// This pass is used for checking control dependencies. It is under "transforms" +// because it adds op warnings and could potentially fix unexpected control +// dependencies in the future (therefore changing the IR). + +namespace mlir { +namespace tf_executor { + +namespace { + +#define GEN_PASS_DEF_EXECUTORCHECKCONTROLDEPENDENCIESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class TFExecutorCheckControlDependencies + : public impl::ExecutorCheckControlDependenciesPassBase< + TFExecutorCheckControlDependencies> { + public: + void runOnOperation() override; +}; + +using ResourceIdVec = llvm::SmallVector>; +// MapVector provides both fast lookup and deterministic iteration order. +using IslandToIslandMapVec = + llvm::MapVector>; +using IslandToIslandHashMap = llvm::SmallDenseMap; + +// Returns true iff the IDs could potentially point to the same resource. +// +// Note: This doesn't yet take self-dependent-only resources into account. We +// would need to return false if one resource is unknown and the other one is +// self-dependent-only. This could cause unreported extra dependencies for such +// cases. +bool IsPotentiallySameResource(TF::ResourceId resource_id, + TF::ResourceId other_resource_id) { + return (resource_id == TF::kUnknownResourceId || + other_resource_id == TF::kUnknownResourceId || + resource_id == other_resource_id); +} + +// Returns true iff there is any dependency between the IDs in `resource_ids` +// and the IDs in `other_resource_ids`. +// Note that this can be made more efficient if necessary. For current use cases +// this runtime is negligible (typically at least one of the resource ID vectors +// is small). +bool ResourceIdsHaveDependency(const ResourceIdVec& resource_ids, + const ResourceIdVec& other_resource_ids) { + for (const auto& [resource_id, read_only] : resource_ids) { + for (const auto& [other_resource_id, other_read_only] : + other_resource_ids) { + if (IsPotentiallySameResource(resource_id, other_resource_id) && + !(read_only && other_read_only)) { + return true; + } + } + } + return false; +} + +// Returns true iff there should be a dependency between `source_op` and +// `target_op`. +bool ShouldOpsHaveDependency( + Operation* source_op, Operation* target_op, + const TF::SideEffectAnalysis::Info& analysis_for_func) { + const ResourceIdVec& source_op_resource_ids = + analysis_for_func.GetResourceIds(source_op); + const ResourceIdVec& target_op_resource_ids = + analysis_for_func.GetResourceIds(target_op); + return ResourceIdsHaveDependency(source_op_resource_ids, + target_op_resource_ids); +} + +// Returns true iff the op wrapped by `island` is an intermediate op used for +// grouping control dependencies. We don't want to report warnings for these +// ops, unless they belong to a control path between two side-effecting ops that +// should not have any dependencies. +bool IsIntermediateOp(IslandOp island) { + // These two side-effect-free ops are known to be used for control dependency + // grouping (e.g., in `BreakUpIslands` pass). + return isa(island.GetBody().front()); +} + +// Finds a path from `source_op` to `target_op` and stores it in `path`, in +// reverse order (from target to source). Returns false if no path was found, +// true otherwise. +// Note that there can be multiple paths, we don't care about which one we find. +// BFS guarantees that we find a path with minimal number of edges. +bool FindPathBfs(IslandOp source_op, IslandOp target_op, + std::vector& path) { + std::queue worklist; + // Stores predecessor pointers. + IslandToIslandHashMap pred_map; + + // BFS starting from `source_op`, stop when `target_op` is reached. + worklist.push(source_op); + while (!worklist.empty()) { + IslandOp curr_op = worklist.front(); + worklist.pop(); + + if (curr_op == target_op) break; + + for (Operation* user : curr_op.getControl().getUsers()) { + auto user_island = dyn_cast(user); + if (!user_island) continue; + // We have labeled `user_island` before so it also must have been added to + // `worklist` before. + if (pred_map.count(user_island) > 0) continue; + + worklist.push(user_island); + pred_map[user_island] = curr_op; + } + } + + // Construct path by following predecessor pointers. + IslandOp curr_op = target_op; + while (curr_op != source_op) { + // If we don't have a predecessor pointer here, then there is no + // source-target path. + if (pred_map.count(curr_op) == 0) return false; + path.push_back(curr_op); + curr_op = pred_map[curr_op]; + } + path.push_back(source_op); + return true; +} + +// Emits dependency warning for `op`. +void EmitDependencyWarningForOp(Operation* op, int path_idx, int node_idx, + absl::string_view pos_str) { + op->emitWarning("unexpected control dependency path: ") + << "path " << path_idx << ", node " << node_idx << " (" << pos_str << ")"; +} + +// Constructs path from `source_op` to `target_op` and emits warnings. +// +// Note that in case of many reported paths this can be inefficient since we +// perform one BFS per path. However, the vast majority of paths should not be +// reported so this should be fine (otherwise we have bigger problems). Also, +// the expected path length is very small (typically < 5) which means that BFS +// searches should terminate quickly (this makes BFS preferable to DFS). +// If higher performance is needed, the paths can be cached during the first +// traversal in function `CheckControlDependenciesForFunc`. +void EmitDependencyWarningsForPath(IslandOp source_op, IslandOp target_op, + int path_idx) { + std::vector path; + bool found_path = FindPathBfs(source_op, target_op, path); + if (!found_path) { + // This shouldn't happen, returning an error instead of asserting so it + // doesn't go unnoticed if it ever happens. + target_op.emitError("no path to target op found, cannot emit warnings"); + return; + } + + // Emit warnings for path. + int node_idx = 0; + for (auto iter = path.rbegin(); iter != path.rend(); ++iter) { + Operation* op = *iter; + std::string pos_str; + if (node_idx == 0) { + pos_str = "source"; + } else if (node_idx + 1 == path.size()) { + pos_str = "target"; + } else { + pos_str = "intermediate"; + } + EmitDependencyWarningForOp(op, path_idx, node_idx++, pos_str); + } +} + +void CheckControlDependenciesForFunc( + func::FuncOp func, const TF::SideEffectAnalysis::Info& analysis_for_func, + int& path_idx) { + IslandToIslandMapVec op_to_control_sources; + + // Traverse islands in topological order. + func.walk([&](IslandOp source_island) { + for (Operation* user : source_island.getControl().getUsers()) { + auto target_island = dyn_cast(user); + if (!target_island) continue; + + if (IsIntermediateOp(source_island)) { + // Add all sources from intermediate op to target op's sources. + op_to_control_sources[target_island].append( + op_to_control_sources[source_island]); + } else { + // Add source island to target op's sources. + op_to_control_sources[target_island].push_back(source_island); + } + } + }); + + // Find all control sources for `target_op` for which we should not have a + // dependency and emit corresponding warnings. + for (const auto& [target_op, control_sources] : op_to_control_sources) { + if (IsIntermediateOp(target_op)) continue; + for (IslandOp source_op : control_sources) { + if (!ShouldOpsHaveDependency(source_op, target_op, analysis_for_func)) { + EmitDependencyWarningsForPath(source_op, target_op, path_idx++); + } + } + } +} + +void TFExecutorCheckControlDependencies::runOnOperation() { + ModuleOp module = getOperation(); + // This pass assumes that all functions are suitable for export, i.e., each + // function has a single tf_executor.graph op and all islands wrap single + // ops. + if (failed(tensorflow::VerifyExportSuitable(module))) { + module.emitOpError() << "not suitable for checking control dependencies"; + return; + } + TF::SideEffectAnalysis side_effect_analysis(module); + // Use a global path index across functions to make it easier to follow one + // path for debugging purposes. + int path_idx = 0; + for (auto func : module.getOps()) { + if (func.isExternal()) continue; + const auto& analysis_for_func = + side_effect_analysis.GetAnalysisForFunc(func); + CheckControlDependenciesForFunc(func, analysis_for_func, path_idx); + } +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorCheckControlDependenciesPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_formation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_formation.cc new file mode 100644 index 0000000000000000000000000000000000000000..b49075eb6ed72af39bf26c2de0b00389150ed12e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_formation.cc @@ -0,0 +1,183 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation forms clusters from instructions in same island and +// assigned to save devices. Clusters are represented as regions. +// Note that side-effecting ops are not correctly handled yet. + +#include +#include + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +#define GEN_PASS_DEF_CLUSTERFORMATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ClusterFormationPass + : public impl::ClusterFormationPassBase { + void runOnOperation() override; +}; + +void ReplaceLiveOutExternalUses(llvm::ArrayRef live_outs, + tf_device::LaunchOp launch_op) { + Region* launch_op_region = &launch_op.getBody(); + for (const auto& p : llvm::zip(live_outs, launch_op.getResults())) { + Value from = std::get<0>(p); + // TODO(jingpu): move this to RegionUtils.h in MLIR core. + for (auto& use : llvm::make_early_inc_range(from.getUses())) { + if (launch_op_region->isAncestor(use.getOwner()->getParentRegion())) + continue; + use.set(std::get<1>(p)); + } + } +} + +// Get all escaped live-out values of a region. +void GetLiveOuts(Region* region, llvm::SmallVectorImpl* live_outs) { + live_outs->clear(); + + for (Operation& op : region->front()) { + for (Value v : op.getResults()) { + // A value is live-out if any of its users are not inside value producer's + // region. + bool is_live_out = llvm::any_of(v.getUsers(), [&](Operation* user) { + return !region->isAncestor(user->getParentRegion()); + }); + + if (is_live_out) live_outs->emplace_back(v); + } + } +} + +// Build a `tf_device.launch` op with a region that contains all the operations +// in given cluster. Then all ops in cluster are replaced by `tf_device.launch`. +void BuildLaunchForCluster(const TF::Cluster& c, OpBuilder* builder) { + // Set insertion point to right after all operations in cluster. + builder->setInsertionPoint(c.ops.back()->getNextNode()); + + // Create a stand-alone region to hold all instructions in the cluster. + Region region; + region.push_back(new Block); + + // Move all operations in cluster to newly created region, stripping their + // "device" attribute since launch op already carries device information. + Block* block = ®ion.front(); + for (Operation* op : c.ops) { + op->moveBefore(block, block->end()); + op->removeAttr(builder->getStringAttr("device")); + } + + // Get all escaped live-out values of region, they are used later to determine + // return values and types of launch op. + llvm::SmallVector live_outs; + GetLiveOuts(®ion, &live_outs); + + // Build a `tf_device.return` op at end of region, with all live-out values + // as operand. + OpBuilder return_builder(builder->getContext()); + return_builder.setInsertionPointToEnd(block); + tf_device::ReturnOp::create(return_builder, return_builder.getUnknownLoc(), + live_outs); + + llvm::SmallVector live_out_types; + live_out_types.reserve(live_outs.size()); + for (Value v : live_outs) { + live_out_types.emplace_back(v.getType()); + } + + tf_device::LaunchOp launch_op = tf_device::LaunchOp::create( + *builder, builder->getUnknownLoc(), builder->getStringAttr(c.target), + live_out_types); + + // Attach the region to launch_op. + launch_op.getBody().takeBody(region); + + // Replace any external uses of live-out values with return values of launch + // op. So live-out values no longer escape the region. + ReplaceLiveOutExternalUses(live_outs, launch_op); + + // Ensure that users of the launch op's results appear after the launch op + // in order to preserve the dominance property. + TF::ReorderOpResultUses(launch_op); +} + +std::string GetDevice(Operation* op) { + auto device_attr = op->getAttrOfType("device"); + return device_attr ? device_attr.getValue().str() : ""; +} + +bool CanBeIgnoredInCluster(Operation* op) { + auto device_attr = op->getAttrOfType("device"); + return !device_attr || device_attr.getValue().empty(); +} + +void BuildClusters(Block& block, OpBuilder builder, + const TF::SideEffectAnalysis::Info& side_effect_analysis) { + auto all_clusters = TF::BuildAllClusters(block, side_effect_analysis, + GetDevice, CanBeIgnoredInCluster); + for (const auto& [device, clusters] : all_clusters) { + for (const auto& cluster : clusters) { + BuildLaunchForCluster(cluster, &builder); + } + } +} + +void ClusterFormationPass::runOnOperation() { + auto module = getOperation(); + auto& side_effect_analysis = getAnalysis(); + + for (auto func : module.getOps()) { + if (func.isExternal()) continue; + OpBuilder builder(func.getContext()); + const TF::SideEffectAnalysis::Info& info = + side_effect_analysis.GetAnalysisForFunc(func); + + // Operates on individual blocks independently of if they are directly in + // the function body or if they are nested in individual + // `tf_executor.island`. + for (Block& block : func.getBody()) BuildClusters(block, builder, info); + func.walk([&](tf_executor::IslandOp island) { + BuildClusters(island.GetBody(), builder, info); + }); + } +} + +} // namespace + +std::unique_ptr> CreateClusterFormationPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.cc new file mode 100644 index 0000000000000000000000000000000000000000..8d3cbfe221b1bb92314677de4f8f9d78e7f8535c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.cc @@ -0,0 +1,747 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h" + +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +#define DEBUG_TYPE "cluster-ops-by-policy" + +namespace mlir { +namespace TFDevice { + +// -------------------------------------------------------------------------- // +// ValueConstraint. +// -------------------------------------------------------------------------- // + +ValueConstraint Merge(ValueConstraint a, ValueConstraint b) { + return a > b ? a : b; +} + +LogicalResult IsStaticallyResolved(Value value, ValueConstraint constraint) { + // Resolve constraints inferred from the tensor type. + if (auto tensor = mlir::dyn_cast(value.getType())) { + if (constraint == ValueConstraint::kRank && tensor.hasRank()) + return success(); + if (constraint == ValueConstraint::kShape && tensor.hasStaticShape()) + return success(); + } + + return failure(); +} + +raw_ostream &operator<<(raw_ostream &os, const ValueConstraint &constraint) { + auto str = [](ValueConstraint constraint) -> StringRef { + switch (constraint) { + case ValueConstraint::kRank: + return "rank"; + case ValueConstraint::kShape: + return "shape"; + case ValueConstraint::kValue: + return "value"; + default: + llvm_unreachable("unknown value constraint"); + } + }; + + os << str(constraint); + return os; +} + +// -------------------------------------------------------------------------- // +// ValuesConstraintSet. +// -------------------------------------------------------------------------- // + +void ValuesConstraintSet::Insert(ValueRange values, + ValueConstraint constraint) { + for (Value value : values) Insert(value, constraint); +} + +std::pair ValuesConstraintSet::Insert( + Value value, ValueConstraint constraint) { + auto emplaced = constraints_.try_emplace(value, constraint); + ValueConstraint persisted = emplaced.first->getSecond(); + + // We've just inserted a new constraint for the value. + if (emplaced.second) return {persisted, true}; + + // Update existing constraint with a new one. + auto merged = Merge(constraint, persisted); + return {constraints_[value] = merged, merged != persisted}; +} + +void ValuesConstraintSet::Walk( + llvm::function_ref walk) const { + for (auto &kv : constraints_) walk(kv.getFirst(), kv.getSecond()); +} + +std::optional ValuesConstraintSet::GetConstraint( + Value value) const { + auto it = constraints_.find(value); + if (it == constraints_.end()) return std::nullopt; + return it->getSecond(); +} + +bool ValuesConstraintSet::HasConstraint(Value value) const { + return GetConstraint(value).has_value(); +} + +void ValuesConstraintSet::MergeAll(const ValuesConstraintSet &other) { + other.Walk([this](Value value, ValueConstraint constraint) { + Insert(value, constraint); + }); +} + +ValuesConstraintSet &ValuesConstraintSet::Resolve() { + llvm::SmallDenseSet resolved; + Walk([&](Value value, ValueConstraint constraint) { + if (succeeded(IsStaticallyResolved(value, constraint))) + resolved.insert(value); + }); + for (Value value : resolved) constraints_.erase(value); + return *this; +} + +ValuesConstraintSet &ValuesConstraintSet::Reset() { + constraints_.clear(); + return *this; +} + +size_t ValuesConstraintSet::Size() const { return constraints_.size(); } + +bool ValuesConstraintSet::Empty() const { return constraints_.empty(); } + +// -------------------------------------------------------------------------- // +// Discovering clusters of operations based on the policy. +// -------------------------------------------------------------------------- // + +namespace { +constexpr char kDeviceAttr[] = "device"; + +// A type that abstracts over types that have uses accessible via `getUses`. +using Source = PointerUnion; + +// We use union-find algorithm to build clusters of connected operations based +// on the user provided policy. If an operation can be clustered (one of the +// user provided policies accepts it under given constraints), it will become +// a "member" that will participate in the union-find cluster construction. +// +// A block argument can also become a member (or even a root member), however +// only operations will become a part of the outline `tf_device.cluster`, block +// arguments will stay as block arguments, and will later become cluster +// function inputs. +struct Member { + Member(unsigned root, Source source, Operation *insertion_point, + ValuesConstraintSet constraints = {}) + : root(root), + source(source), + insertion_point(insertion_point), + constraints(constraints) {} + + unsigned root; + Source source; + + // After construction: + // For basic block argument source this will be a first operation in the + // basic block, and for operations it will be an operation iself. + // + // During the union-find cluster formation: + // The root member will have the location in the basic block, where the + // cluster operation will be inserted. We use the location of the last + // operation in the cluster, so that during cluster construction we can + // ensure that all operands are above the insertion point, and all users are + // below the insertion point. + // + // Example: + // + // %0 = "clustered_op"(...) + // %1 = "non_clustered_op"(...) + // %2 = "clustered_op"(%1) <<<--- insert cluster here + // %3 = "cluster_result_user"(%1, %2) + // + // By using `%2` location as an insertion point we ensure that all operands + // (%1 in this example) dominate the cluster operation, and that the cluster + // operation dominates all the users (%3 in this example). + Operation *insertion_point; + + // After construction: + // A set of constraints on the clustered operation operands that must be + // satisfied in order to add operation to the cluster. For basic block + // source this will be always empty. + // + // During the union-find cluster formation: + // The root member will have constraints merged from all of the cluster + // members. + ValuesConstraintSet constraints; +}; + +using Members = llvm::SmallVector; + +struct ClusteringState { + // Storage backing an array based union-find algorithm for clustering. Index + // in this vector is the member id. + llvm::SmallVector members; + + // Mapping from the member operation (block argument) to the member id. + llvm::SmallDenseMap member_ids; + + // Puts `a` and `b` members into the same cluster if it is possible. Returns + // success if union operation was completed successfully, otherwise returns + // failure. + // + // Members can be clustered together: + // 1. This will not break dominance property of the IR. + // 2. New clustering policy constraints can be propagated through the + // already clustered operations. + LogicalResult Union(unsigned a, unsigned b, + const ClusteringPolicySet &policies); + + bool IsMember(Operation *op) const; + unsigned FindRoot(unsigned id); + + // Verifies that merging `src_root` cluster with a `dst_root` cluster, and + // inserting it at `insertion_point` location will not break the dominance + // property: all users of the `src_root` cluster results must be below the + // insertion point in the block. + LogicalResult VerifyDominanceProperty(unsigned src_root, unsigned dst_root, + Operation *insertion_point); + + // Verifies that all constraints on the values defined by the `dst_root` + // cluster can be propagated through the nodes in the `src_root` cluster, and + // updates `src_root` constraints on success. + LogicalResult VerifyValueConstraints(unsigned src_root, unsigned dst_root, + const ClusteringPolicySet &policies); +}; + +} // namespace + +bool ClusteringState::IsMember(Operation *op) const { + return member_ids.find(op) != member_ids.end(); +} + +unsigned ClusteringState::FindRoot(unsigned id) { + if (members[id].root == id) return id; + return members[id].root = FindRoot(members[id].root); +} + +LogicalResult ClusteringState::VerifyDominanceProperty( + unsigned src_root, unsigned dst_root, Operation *insertion_point) { + // TODO(ezhulenev): Optimize this linear scan with a map lookup. + for (auto &member : members) { + unsigned root = FindRoot(member.root); + if (root != src_root) continue; + + // Block arguments do not really participate in clustering, they are only + // used to connect independent operation using the same argument. + if (member.source.is()) continue; + + Operation *op = member.source.dyn_cast(); + assert(op && "member operation must be not null"); + + for (Operation *user : op->getUsers()) { + // Skip users in other blocks. + if (user->getBlock() != op->getBlock()) continue; + + // Skip users is in the `dst_root` or `src_root` clusters, if we'll merge + // roots they'll become a single cluster and will not violate the + // dominance property after that. + auto it = member_ids.find(user); + if (it != member_ids.end() && (FindRoot(it->getSecond()) == dst_root || + FindRoot(it->getSecond()) == src_root)) + continue; + + if (user->isBeforeInBlock(insertion_point)) { + LLVM_DEBUG(llvm::dbgs() + << " Failure: user is before the insertion point: " + << *user << "\n";); + return failure(); + } + } + } + + return success(); +} + +LogicalResult ClusteringState::VerifyValueConstraints( + unsigned src_root, unsigned dst_root, const ClusteringPolicySet &policies) { + // Propagate constraints only through operations in the `src_root` cluster. + auto filter = [&](Operation *op) -> bool { + auto it = member_ids.find(op); + return it != member_ids.end() && FindRoot(it->getSecond()) == src_root; + }; + + // Start from all operations in the `src_root` cluster. + llvm::SmallVector worklist; + for (Member &member : members) + if (Operation *op = member.source.dyn_cast()) + if (FindRoot(member.root) == src_root) worklist.emplace_back(op); + + // Collect `dst_root` constraints that are applicable to the values defined in + // the `src_root` cluster. + ValuesConstraintSet constraints = members[src_root].constraints; + members[dst_root].constraints.Walk([&](Value v, ValueConstraint constraint) { + Operation *op = v.getDefiningOp(); + if (op && filter(op)) constraints.Insert(v, constraint); + }); + + // Update `src_root` constraints only if we can propagate them. + if (succeeded(PropagateValuesConstraints(worklist, filter, policies, + constraints, /*resolve=*/true))) { + members[src_root].constraints = constraints; + return success(); + } + + return failure(); +} + +LogicalResult ClusteringState::Union(unsigned a, unsigned b, + const ClusteringPolicySet &policies) { + unsigned a_root = FindRoot(a); + unsigned b_root = FindRoot(b); + + // Already members of the same cluster. + if (a_root == b_root) return failure(); + + // Verify that merging two clusters will not break dominance property. + Operation *a_insertion_point = members[a_root].insertion_point; + Operation *b_insertion_point = members[b_root].insertion_point; + bool a_is_before_b = a_insertion_point->isBeforeInBlock(b_insertion_point); + + // Use clusters position in the block to select merging src and dst. + unsigned src_root = a_is_before_b ? a_root : b_root; // merge `src_root` ... + unsigned dst_root = a_is_before_b ? b_root : a_root; // ... into `dst_root` + Operation *insertion_point = + a_is_before_b ? b_insertion_point : a_insertion_point; + + // Print operations in the `root` cluster to debug stream. + auto debug_clustered_ops = [&](unsigned root) { + for (Member &member : members) + if (FindRoot(member.root) == root) { + if (auto *op = member.source.dyn_cast()) { + llvm::dbgs() << " " << *op << "\n"; + } else if (auto *arg = member.source.dyn_cast()) { + llvm::dbgs() << " " << *arg; + } + } + }; + (void)debug_clustered_ops; + + LLVM_DEBUG({ + llvm::dbgs() << "\n\n--- Try to merge cluster:\n"; + debug_clustered_ops(src_root); + llvm::dbgs() << "\n--- With cluster:\n"; + debug_clustered_ops(dst_root); + LLVM_DEBUG(llvm::dbgs() << "\n--- Diagnostics:\n"); + }); + + // Check if merging `src_root` with `dst_root` will not violate SSA dominance + // property (all operands before the cluster, all results after the cluster). + if (failed(VerifyDominanceProperty(src_root, dst_root, insertion_point))) + return failure(); + + // Check if `dst_root` constraints can be propagated to the `src_root` + // constraints. + if (failed(VerifyValueConstraints(src_root, dst_root, policies))) + return failure(); + + // Set `dst_root` as a new root for `src_root`. + members[src_root].root = dst_root; + // Update insertion point of the new root. + members[dst_root].insertion_point = insertion_point; + // Merge all constraints from `src_root` into `dst_root`. + members[dst_root].constraints.MergeAll(members[src_root].constraints); + + LLVM_DEBUG(llvm::dbgs() << " Clusters successfully merged\n"); + + return success(); +} + +// Returns constraints on the operands specified by the clustering policy if the +// operation can be clustered (constraints could be empty). Otherwise return +// empty optional. +static std::optional CanBeClustered( + Operation *op, const ClusteringPolicySet &policies, + const std::function &filter) { + // Check that op has no side effects. This guarantees that we will not + // reorder side-effecting ops during cluster formation. + if (!isMemoryEffectFree(op)) return std::nullopt; + + // Operation rejected by the custom filter. + if (filter && !filter(op)) return std::nullopt; + + // Initially we do not have any constraints on the operation results. + ValuesConstraintSet result_constraints; + + for (auto &policy : policies.policies()) { + ValuesConstraintSet operands_constraints; + if (succeeded(policy->MatchAndUpdateConstraints(op, result_constraints, + operands_constraints))) + return operands_constraints.Resolve(); + } + + return std::nullopt; +} + +// Compute initial clustering state based on the clustering polocy. +static ClusteringState InitializeClusteringState( + Block *block, const ClusteringPolicySet &policies, + const std::function &filter) { + ClusteringState state; + + // Create members for all block arguments. + for (BlockArgument &arg : block->getArguments()) { + if (!arg.getUsers().empty()) + state.members.emplace_back(state.members.size(), &arg, &block->front()); + } + + int num_bbarg_members = state.members.size(); + (void)num_bbarg_members; + + // Create members for operations that can be clustered based on the policy. + for (Operation &op : block->getOperations()) { + if (auto constraints = CanBeClustered(&op, policies, filter)) + state.members.emplace_back(state.members.size(), &op, &op, *constraints); + } + + // Initialize mapping from the member operation (block argument) to the id. + for (const auto &tuple : llvm::enumerate(state.members)) { + state.member_ids.try_emplace(tuple.value().source, tuple.index()); + } + + LLVM_DEBUG(llvm::dbgs() << "Found " + << (state.members.size() - num_bbarg_members) + << " clustering candidate operations in the block\n"); + + return state; +} + +// Users of the `source` that are candidates for clustering. +static llvm::SmallVector GetClusteringCandidates( + const ClusteringState &state, Source source) { + // Users of operation result must be in the same block and placed on the same + // device. + if (auto op = source.dyn_cast()) { + auto range = llvm::make_filter_range(op->getUsers(), [&](Operation *user) { + bool same_block = user->getBlock() == op->getBlock(); + bool same_device = op->getAttr(kDeviceAttr) == user->getAttr(kDeviceAttr); + return same_block && same_device && state.IsMember(user); + }); + return {range.begin(), range.end()}; + } + + // Users of block argument must be in the same block. + if (auto arg = source.dyn_cast()) { + auto range = llvm::make_filter_range(arg->getUsers(), [&](Operation *user) { + bool same_block = user->getBlock() == arg->getOwner(); + return same_block && state.IsMember(user); + }); + return {range.begin(), range.end()}; + } + + llvm_unreachable("Unexpected type in the union."); +} + +// Cluster members with their result users. Returns `true` if merged at least a +// pair of members into a new cluster. +static bool RunClusteringPass(ClusteringState &state, + const ClusteringPolicySet &policies) { + bool clustered = false; + + for (const auto &tuple : llvm::enumerate(state.members)) { + size_t member_id = tuple.index(); + Member &member = tuple.value(); + + llvm::SmallVector users = + GetClusteringCandidates(state, member.source); + + // Process candidates according to their order in the block to minimize + // the number of dominance property violations. + llvm::sort(users, [](auto *a, auto *b) { return a->isBeforeInBlock(b); }); + + for (Operation *user : users) { + auto user_member_id = state.member_ids.lookup(user); + if (succeeded(state.Union(member_id, user_member_id, policies))) + clustered = true; + } + } + + return clustered; +} + +llvm::SmallVector FindClustersInTheBlock( + Block *block, const ClusteringPolicySet &policies, + std::function filter) { + // It is impossible to build a cluster in the empty block. + if (block->empty()) return {}; + + ClusteringState state = InitializeClusteringState(block, policies, filter); + + // Run clustering passes until the convergence. Limit the number of iterations + // to guard from the infinite loop in presence of bugs. + constexpr int max_iterations = 100; + for (unsigned i = 0; i < max_iterations; ++i) + if (!RunClusteringPass(state, policies)) break; + + // Form clusters found by the union-find algorithm. + llvm::DenseMap root_clusters; + + for (Member &member : state.members) { + Cluster &cluster = root_clusters[state.FindRoot(member.root)]; + + // If member is a root of the cluster, copy inferred constraints. + if (state.FindRoot(member.root) == member.root) + cluster.constraints = std::move(member.constraints); + + // Add operation to the cluster. + if (auto op = member.source.dyn_cast()) + cluster.operations.emplace_back(op); + } + + llvm::SmallVector clusters; + for (auto &kv : root_clusters) { + Cluster &cluster = kv.getSecond(); + // Skip degenerate clusters formed by a single basic block argument. + if (!cluster.operations.empty()) clusters.emplace_back(std::move(cluster)); + } + + LLVM_DEBUG(llvm::dbgs() << "Found " << clusters.size() << " clusters\n"); + + return clusters; +} + +// -------------------------------------------------------------------------- // +// Create `tf_device.cluster` operation from the discovered ops cluster. +// -------------------------------------------------------------------------- // + +tf_device::ClusterOp CreateClusterOp(Cluster &cluster, StringAttr policy) { + // Find all the values that are used outside of the cluster. These values + // will be returned from the created cluster operation. + llvm::DenseSet in_cluster; + for (Operation *op : cluster.operations) in_cluster.insert(op); + + llvm::SetVector return_values; + llvm::SmallVector return_types; + + for (Operation *op : cluster.operations) + for (OpOperand &use : op->getUses()) { + // User is inside the cluster. + if (in_cluster.contains(use.getOwner())) continue; + // Do not return the same value multiple times. + if (return_values.contains(use.get())) continue; + + return_values.insert(use.get()); + return_types.emplace_back(use.get().getType()); + } + + // Sort matched operations by their position in the block. + llvm::sort(cluster.operations, [](Operation *a, Operation *b) -> bool { + return a->isBeforeInBlock(b); + }); + + // Create tf_device::ClusterOp before the last operation in the block that + // is a part of a match set. + auto back = cluster.operations.back(); + auto loc = back->getLoc(); + OpBuilder builder(back); + + auto cluster_op = + tf_device::ClusterOp::create(builder, loc, return_types, policy); + + // Create block in cluster_op's region and move 'cluster.operations' into + // it. + auto block = builder.createBlock(&cluster_op.getBody()); + auto block_end = block->end(); + for (auto op : cluster.operations) op->moveBefore(block, block_end); + + // Add 'tf_device::ReturnOp' at the end of the block. + builder.setInsertionPointToEnd(block); + tf_device::ReturnOp::create(builder, loc, return_values.getArrayRef()); + + // Set device attribute + if (auto device = back->getAttr(kDeviceAttr)) + cluster_op->setAttr(kDeviceAttr, device); + + // Update all users of the operations moved into the cluster region. + for (auto tuple : llvm::zip(return_values, cluster_op.getResults())) { + Value old_value = std::get<0>(tuple); + Value new_value = std::get<1>(tuple); + old_value.replaceUsesWithIf(new_value, [&](OpOperand &operand) -> bool { + // Do not update users in the same cluster. + return operand.getOwner()->getBlock() != block; + }); + } + + return cluster_op; +} + +// -------------------------------------------------------------------------- // +// Helper functions for value constraints propagations and analysis. +// -------------------------------------------------------------------------- // + +mlir::LogicalResult PropagateValuesConstraints( + llvm::ArrayRef root, std::function filter, + const ClusteringPolicySet &policies, ValuesConstraintSet &constraints, + bool resolve, bool emit_remarks) { + // A set of constraints for operation results. + llvm::DenseMap op_results_constraints; + assert(filter && "filter predicate must be defined"); + + // Use initial constraints to initialize op results constraints. + for (std::pair pair : constraints) { + Value value = pair.first; + ValueConstraint constraint = pair.second; + + // Value must be defined by an operation and accepted by the filter. + Operation *op = value.getDefiningOp(); + if (!op || !filter(op)) continue; + + op_results_constraints[op].Insert(value, constraint); + } + + // Keep a worklist of operations that need their constraints to be updated. + llvm::SetVector worklist; + for (Operation *op : root) worklist.insert(op); + + while (!worklist.empty()) { + Operation *op = worklist.pop_back_val(); + + // Use results constraints to infer operands constraints. + const ValuesConstraintSet &results = op_results_constraints[op]; + ValuesConstraintSet operands; + + // Walk through all policies until we find one that matches the operation. + bool updated = false; + for (auto &policy : policies.policies()) { + auto matched = + policy->MatchAndUpdateConstraints(op, results, operands.Reset()); + if (succeeded(matched)) { + updated = true; + break; + } + } + + // Signal a failure if could not propagate non-empty constraints on the + // operation results to the operands. + if (!updated && !results.Empty()) { + if (emit_remarks) { + std::string err_msg; + llvm::raw_string_ostream os(err_msg); + for (unsigned i = 0; i < op->getNumResults(); ++i) + os << " " << i << ":" << results.GetConstraint(op->getResult(i)); + op->emitError(llvm::formatv( + "failed to propagate results constraints:{0}", os.str())); + } + return failure(); + } + + // Update results constraints based on inferred operands constraints. + operands.Walk([&](Value value, ValueConstraint constraint) { + // Resolve constraint based on the static type information. + if (resolve && succeeded(IsStaticallyResolved(value, constraint))) return; + + // Update constraint for a value. + auto updated = constraints.Insert(value, constraint); + if (!updated.second) return; + + // Maybe update constaint on the operation result, but do not follow + // operations that are not accepted by the filter predicate. + Operation *op = value.getDefiningOp(); + if (!op || !filter(op)) return; + + // Add updated operation to the worklist. + auto inserted = op_results_constraints[op].Insert(value, updated.first); + if (inserted.second) worklist.insert(op); + }); + } + + return success(); +} + +mlir::LogicalResult PropagateValuesConstraints( + mlir::Region ®ion, const ClusteringPolicySet &policies, + ValuesConstraintSet &constraints, bool resolve, bool emit_remarks) { + // Propagate constraints for all operations in the region. + llvm::SmallVector worklist; + region.walk([&](Operation *op) { worklist.emplace_back(op); }); + + // Propagate constraints only through operations inside the `region`. + auto filter = [&](Operation *op) -> bool { + return region.findAncestorBlockInRegion(*op->getBlock()); + }; + + return PropagateValuesConstraints(worklist, filter, policies, constraints, + resolve, emit_remarks); +} + +void EmitValueConstraintsRemarks(const ValuesConstraintSet &constraints) { + constraints.Walk([](Value value, ValueConstraint constraint) { + for (OpOperand &operand : value.getUses()) + operand.getOwner()->emitRemark( + llvm::formatv("operand #{0} constrained to: {1}", + operand.getOperandNumber(), constraint)); + }); +} + +void EmitInputsConstraintsRemarks(func::FuncOp func, + const ValuesConstraintSet &constraints) { + constraints.Walk([&](Value value, ValueConstraint constraint) { + if (auto arg = mlir::dyn_cast(value)) + if (arg.getOwner() == &func.getBody().front()) + func.emitRemark(llvm::formatv("input #{0} constrained to: {1}", + arg.getArgNumber(), constraint)); + }); +} + +LogicalResult InferFunctionBodyValuesConstraints( + func::FuncOp func, ValuesConstraintSet &constraints) { + for (unsigned i = 0; i < func.getNumResults(); ++i) { + auto str = func.getResultAttrOfType(i, "tf.constraint"); + if (!str) continue; + + ValueConstraint constraint = StringSwitch(str.getValue()) + .Case("rank", ValueConstraint::kRank) + .Case("shape", ValueConstraint::kShape) + .Case("value", ValueConstraint::kValue); + + // Propagate constraints through function return operations. + for (Block &block : func.getBody()) { + func::ReturnOp ret = dyn_cast(block.back()); + if (ret) constraints.Insert(ret.getOperand(i), constraint); + } + } + + return success(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h new file mode 100644 index 0000000000000000000000000000000000000000..e96b0851d31da20e680d93ddc882949957999dff --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h @@ -0,0 +1,294 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CLUSTER_OPS_BY_POLICY_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CLUSTER_OPS_BY_POLICY_H_ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" + +namespace mlir { +namespace TFDevice { + +// -------------------------------------------------------------------------- // +// ValueConstraint. +// -------------------------------------------------------------------------- // + +// In order to be clustered operation can require its operands to satisfy +// some constraints (e.g. reduction operation can require reduction dimension +// operand to be a constant value). +enum class ValueConstraint { + // Operand must have statically known rank. + kRank = 0, + // Operand must have statically known shape (all dimensions are known at + // compile time). + kShape = 1, + // Operand must have statically known value (operand must be defined by a + // constant operation). + kValue = 2, +}; + +// Returns the more restrictive constraint of `a` and `b`: +// +// Value >> Shape >> Rank +// +// If you know the value, you always know the shape and the rank. If you know +// the shape, you always know the rank. +ValueConstraint Merge(ValueConstraint a, ValueConstraint b); + +// Returns success if constraint can be resolved statically based on the value +// type, e.g. `shape` constraint can be resolved if the value is a tensor of +// statically known shape. +LogicalResult IsStaticallyResolved(Value value, ValueConstraint constraint); + +raw_ostream& operator<<(raw_ostream& os, const ValueConstraint& constraint); + +// -------------------------------------------------------------------------- // +// ValuesConstraintSet. +// -------------------------------------------------------------------------- // + +// A set of constraints for values, that either operation results or operands. +class ValuesConstraintSet { + using ConstraintsMap = llvm::SmallDenseMap; + using ConstIterator = typename ConstraintsMap::const_iterator; + + public: + ValuesConstraintSet() = default; + + // Inserts a new constraint for the `value`. If the `value` already has some + // constraint, it will merge it with a new one, and will return a new + // constraint value. Returned pair has a constraint value that was set for + // a value, and a boolean flag that is true if the constraint was updated. + std::pair Insert(Value value, + ValueConstraint constraint); + + // Inserts constraints for multiple values. + void Insert(ValueRange value, ValueConstraint constraint); + + // Walk all the constraints owned by this set. + void Walk(llvm::function_ref walk) const; + + // Returns the constraint of the value if it exists, or None otherwise. + std::optional GetConstraint(Value value) const; + bool HasConstraint(Value value) const; + + // Merges all constrains from the other constraints set into this one. + void MergeAll(const ValuesConstraintSet& other); + + // Remove constraints that can be statically resolved from the type of the + // constrained value (see `IsStaticallyResolved` defined above). + ValuesConstraintSet& Resolve(); + + // Reset all constraints. + ValuesConstraintSet& Reset(); + + // Return the number of constrained values in the set. + size_t Size() const; + + // Returns true if the constraint set is empty. + bool Empty() const; + + ConstIterator begin() const { return constraints_.begin(); } + ConstIterator end() const { return constraints_.end(); } + + private: + llvm::SmallDenseMap constraints_; +}; + +// -------------------------------------------------------------------------- // +// ClusteringPolicy. +// -------------------------------------------------------------------------- // + +// Clustering policy specifies if the operation can be clustered (in practice it +// usually means that operation can be added to a cluster that will be later +// compiled) given the set of constraints on its results, and might propagate or +// create new constraints on the operation operands. +// +// Clustering policy must make a local decision just for a single operation. It +// is the responsibility of a clustering pass to combine all these individual +// operations constraints to form a valid cluster. +// +// Example: compilation using XLA (MHLO) lowering +// +// %0 = "tf.Transpose"(%input, %perm) +// : (tensor, tensor<2xi32>) -> tensor +// +// XLAs `mhlo.transpose` operation requires permutation to be an attribute +// (compile time value), so it means that if we want to put `tf.Transpose` +// into a cluster that will be compiled with XLA, the `%perm` operand must +// be a known compiled time value, e.g. result of a `tf.Const` operation. +// +class ClusteringPolicy { + public: + virtual ~ClusteringPolicy() = default; + + // Returns success if an operation can be clustered given the constraints on + // the operation results. Updates operands constraits to satisfy all the + // results constraints. + virtual LogicalResult MatchAndUpdateConstraints( + Operation* operation, const ValuesConstraintSet& results, + ValuesConstraintSet& operands) const = 0; +}; + +// Clustering policy for a specific operation type. +template +class OpClusteringPolicy : public ClusteringPolicy { + public: + LogicalResult MatchAndUpdateConstraints( + Operation* operation, const ValuesConstraintSet& results, + ValuesConstraintSet& operands) const final { + if (auto op = dyn_cast(operation)) + return MatchAndUpdateConstraints(op, results, operands); + return failure(); + } + + virtual LogicalResult MatchAndUpdateConstraints( + OpTy op, const ValuesConstraintSet& results, + ValuesConstraintSet& operands) const = 0; +}; + +// -------------------------------------------------------------------------- // +// ClusteringPolicySet. +// -------------------------------------------------------------------------- // + +// A set of clustering policies for different operations. +class ClusteringPolicySet { + public: + using Policies = std::vector>; + + const Policies& policies() const { return policies_; } + + // Add an instance of each of the policy types 'Ts'. Return a reference to + // `this` for chaining insertions. + template + ClusteringPolicySet& Add() { + (void)std::initializer_list{0, (AddImpl(), 0)...}; + return *this; + } + + // ClusteringPolicySet is move only type. + ClusteringPolicySet() = default; + ClusteringPolicySet(const ClusteringPolicySet&) = delete; + ClusteringPolicySet(ClusteringPolicySet&&) = default; + ClusteringPolicySet& operator=(const ClusteringPolicySet&) = delete; + ClusteringPolicySet& operator=(ClusteringPolicySet&&) = default; + + private: + template + void AddImpl(Args&&... args) { + static_assert(std::is_base_of::value, + "T must implement ClusteringPolicy"); + policies_.emplace_back(std::make_unique(std::forward(args)...)); + } + + std::vector> policies_; +}; + +// -------------------------------------------------------------------------- // +// Discovering clusters of operations based on the policy. +// -------------------------------------------------------------------------- // + +// Cluster groups together operations in the single basic block based on the +// given clustering policy set. Clusters can be outlined into nested modules +// later device specific compilation (e.g. for TFRT JIT compiler). +struct Cluster { + llvm::SmallVector operations; + ValuesConstraintSet constraints; +}; + +// Returns clusters of operations in the given `block` based on the provided +// clustering policy. If `filter` is defined, it will be used to filter +// operations that can be considered for clustering based on the policy. +// +// TODO(ezhulenev): Additional filter function is a workaround for customizing +// clustering policies at runtime for experimentation. In the long term, +// clustering policy should be enough. +llvm::SmallVector FindClustersInTheBlock( + Block* block, const ClusteringPolicySet& policies, + std::function filter = {}); + +// Creates a `tf_device.cluster` operation from the clustered operations. +tf_device::ClusterOp CreateClusterOp(Cluster& cluster, StringAttr policy = {}); + +// -------------------------------------------------------------------------- // +// Helper functions for value constraints propagations and analysis. +// -------------------------------------------------------------------------- // + +// Propagates initial constraints on the values defined by the `constraints` set +// with operations in the `root` as a starting point, using user provided set of +// clustering policies. +// +// Filter predicate specifies if constraints should be propagated across the +// given operation. Operations in the root set will be also filtered using +// the `filter` predicate. +// +// Optionally resolve constraints that can be statically satisfied by the +// value type, and stop constraints propagation early. +// +// Optionally emits remarks attached to operation that failed to propagate +// results constraints to its operands (for testing purpose). +// +// Returns failure if constraints can't be propagated through some of the +// operations accepted by the filter (there is no clustering policy for an +// operation, or constraints can't be satisfied by the policy), and attaches +// error diagnostics to the operation that prevented constraints propagation. +mlir::LogicalResult PropagateValuesConstraints( + llvm::ArrayRef root, std::function filter, + const ClusteringPolicySet& policies, ValuesConstraintSet& constraints, + bool resolve = false, bool emit_remarks = false); + +// Propagates initial constraints on the values in the `region` to the other +// values in the same region, using user provided set of clustering policies. +mlir::LogicalResult PropagateValuesConstraints( + mlir::Region& region, const ClusteringPolicySet& policies, + ValuesConstraintSet& constraints, bool resolve = false, + bool emit_remarks = false); + +// Emits constraints remarks for all operations that use constrained values. +void EmitValueConstraintsRemarks(const ValuesConstraintSet& constraints); + +// Emits constraints remarks for function inputs that are in the constraints +// set (entry block arguments have constraints). +void EmitInputsConstraintsRemarks(func::FuncOp func, + const ValuesConstraintSet& constraints); + +// Infers constraints for the values in the function body from the function +// results attributes. +// +// Example: +// func @test(...) -> (tensor {tf.constraint = "shape"}) { +// ..... +// %v = "some_operation"() : () -> tensor +// return %v : tensor +// } +LogicalResult InferFunctionBodyValuesConstraints( + func::FuncOp func, ValuesConstraintSet& constraints); + +} // namespace TFDevice +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CLUSTER_OPS_BY_POLICY_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_outlining.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_outlining.cc new file mode 100644 index 0000000000000000000000000000000000000000..2c03c6c9757821d6f07a0cd65ac94fc1e1e99c04 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_outlining.cc @@ -0,0 +1,195 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +constexpr char kFuncAttr[] = "func"; + +#define GEN_PASS_DEF_CLUSTEROUTLININGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ClusterOutliningPass + : public impl::ClusterOutliningPassBase { + void runOnOperation() override; +}; + +#define GEN_PASS_DEF_LAUNCHOUTLININGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct LaunchOutliningPass + : public impl::LaunchOutliningPassBase { + void runOnOperation() override; +}; + +void ReplaceClusterReturnWithReturn(tf_device::ReturnOp cluster_return_op, + OpBuilder* builder) { + func::ReturnOp::create(*builder, cluster_return_op.getLoc(), + cluster_return_op.getOperands()); + cluster_return_op.erase(); +} + +// Builds a function that outlines region attached to cluster_op or launch_op, +// and inserts built function into given module. +template +func::FuncOp BuildFunction(llvm::ArrayRef live_ins, ClusterOrLaunchOp op, + SymbolTable* symbol_table, OpBuilder* builder) { + llvm::SmallVector operand_types; + operand_types.reserve(live_ins.size()); + for (Value v : live_ins) operand_types.emplace_back(v.getType()); + + auto func_type = builder->getFunctionType(operand_types, op.getResultTypes()); + + std::string func_name; + if (auto outlined_func_name = op->template getAttrOfType( + TF::kClusterOutlinedFunctionNameAttr)) { + op->removeAttr(TF::kClusterOutlinedFunctionNameAttr); + func_name = outlined_func_name.str(); + } else { + func_name = "_func"; + } + + func::FuncOp outlined_func = + func::FuncOp::create(op.getLoc(), func_name, func_type); + + // This function is not externally visible and marking it private would allow + // symbol-dce pass to remove it when it is not referenced anymore. + outlined_func.setPrivate(); + + // Create function body. + Block* outlined_func_block = outlined_func.addEntryBlock(); + + // Replace uses of live-in values within cluster_op region with function + // arguments. + Region& op_region = op.getBody(); + for (auto p : llvm::zip(live_ins, outlined_func_block->getArguments())) { + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), op_region); + } + + // Move all instructions in cluster_op into outlined_function's only block. + auto& op_body = op.GetBody().getOperations(); + outlined_func_block->getOperations().splice( + outlined_func_block->end(), op_body, op_body.begin(), op_body.end()); + + // Replace `tf_device.return` terminator with `std.return` in function + // body. + auto return_op = + cast(outlined_func_block->getTerminator()); + builder->setInsertionPoint(return_op); + ReplaceClusterReturnWithReturn(return_op, builder); + + symbol_table->insert(outlined_func); + return outlined_func; +} + +// Outlines body of `tf_device.cluster` into a function and create a +// `tf_device.cluster_func` to invoke that function. `tf_device.cluster` is +// removed afterwards.` +void OutlineCluster(tf_device::ClusterOp cluster_op, SymbolTable* symbol_table, + OpBuilder* builder) { + llvm::SetVector live_ins; + getUsedValuesDefinedAbove(cluster_op.getBody(), cluster_op.getBody(), + live_ins); + + func::FuncOp outlined_func = + BuildFunction(live_ins.getArrayRef(), cluster_op, symbol_table, builder); + cluster_op->setAttr( + builder->getStringAttr(kFuncAttr), + mlir::SymbolRefAttr::get(builder->getContext(), outlined_func.getName())); + + builder->setInsertionPoint(cluster_op); + auto cluster_func_op = tf_device::ClusterFuncOp::create( + *builder, cluster_op.getLoc(), + outlined_func.getFunctionType().getResults(), live_ins.getArrayRef(), + cluster_op->getAttrs()); + cluster_op.replaceAllUsesWith(cluster_func_op); + cluster_op.erase(); +} + +// Outlines body of `tf_device.launch` into a function and create a +// `tf_device.launch_func` to invoke that function. `tf_device.launch` is +// removed afterwards.` +void OutlineLaunch(tf_device::LaunchOp launch_op, SymbolTable* symbol_table, + OpBuilder* builder) { + llvm::SetVector live_ins; + getUsedValuesDefinedAbove(launch_op.getBody(), launch_op.getBody(), live_ins); + + func::FuncOp outlined_func = + BuildFunction(live_ins.getArrayRef(), launch_op, symbol_table, builder); + launch_op->setAttr( + builder->getStringAttr(kFuncAttr), + mlir::SymbolRefAttr::get(builder->getContext(), outlined_func.getName())); + + builder->setInsertionPoint(launch_op); + auto cluster_func_op = tf_device::LaunchFuncOp::create( + *builder, launch_op.getLoc(), + outlined_func.getFunctionType().getResults(), live_ins.getArrayRef(), + launch_op->getAttrs()); + + launch_op.replaceAllUsesWith(cluster_func_op); + launch_op.erase(); +} + +void ClusterOutliningPass::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTable symbol_table(module); + OpBuilder builder(module.getContext()); + module.walk([&](tf_device::ClusterOp cluster) { + OutlineCluster(cluster, &symbol_table, &builder); + }); +} + +void LaunchOutliningPass::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTable symbol_table(module); + OpBuilder builder(module.getContext()); + module.walk([&](tf_device::LaunchOp launch) { + OutlineLaunch(launch, &symbol_table, &builder); + }); +} + +} // namespace + +std::unique_ptr> CreateClusterOutliningPass() { + return std::make_unique(); +} + +std::unique_ptr> CreateLaunchOutliningPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_tf_ops_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_tf_ops_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..28a601c227456cfe13261e5a7c0a7bd1c832d569 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/cluster_tf_ops_pass.cc @@ -0,0 +1,351 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass clusters the TensorFlow ops by host. The program generated by this +// pass will have one function per host where all operations in the same +// function are placed on the same host. Each result of the per-host function +// will have a "tf.device" attribute which specifies the device assignment of +// the result. +// +// The pass currently assumes that there is no circular dependency among the +// per-host functions. For example, if there exists an operation placed on +// host_A that consumes the result of an operation placed on host_B, then there +// does not exist any operation placed on host_B that conumes any result of any +// operation placed on host_A. + +#include +#include +#include +#include + +#include "mlir/IR/Builders.h" +#include "mlir/Pass/Pass.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TF { +namespace { + +using DeviceNameUtils = ::tensorflow::DeviceNameUtils; +using ParsedName = ::tensorflow::DeviceNameUtils::ParsedName; + +constexpr const char *kHostAttr = "host"; +constexpr const char *kDeviceAttr = "device"; +constexpr const char *kTFDeviceAttr = "tf.device"; +// TODO(donglin): Handle the case where the address of localhost is different +// from /job:localhost/replica:0/task:0. +constexpr const char *kLocalhost = "/job:localhost/replica:0/task:0"; +constexpr const char *kErrorMessage = + "The operation that uses the operand is on a different host than the " + "operation that defines the op. This pass does not support cross-host data " + "transfer yet"; + +// The host address is identified by the job/replicate/task in the device name. +std::string GetHost(llvm::StringRef device) { + ParsedName parsed_name; + DeviceNameUtils::ParseFullName(device.str(), &parsed_name); + std::string result = DeviceNameUtils::ParsedNameToString( + DeviceNameUtils::AddressSpace(parsed_name)); + return result.empty() ? kLocalhost : result; +} + +std::string GetHost(Operation *op) { + std::string device = ""; + if (StringAttr attr = op->getAttrOfType(kDeviceAttr)) { + device = attr.getValue().str(); + } + return GetHost(device); +} + +// The device is considered to be on the localhost iff one of the following is +// true: +// 1) None of the job/replica/task is specified in the device name. +// 2) The job/replica/task in the device name are explicitly specified as +// /job:localhost/replica:0/task:0. +bool IsOnLocalHost(llvm::StringRef device) { + std::string host = GetHost(device); + return host == kLocalhost; +} + +// This structure contains the metadata of the per-host function. All operations +// in this function should be on the same host. +struct FunctionMetadata { + // The original function name before partition. + llvm::StringRef original_name; + // The insertion point of partition functions. + Block::iterator insertion_point; + // The partitioned function name. + llvm::StringRef partition_name; + // The input values of the function. + llvm::SmallVector inputs; + // The result values of the function. + llvm::SmallVector results; + // The devices of the input values. It should have the same size as inputs. + llvm::SmallVector input_devices; + // The devices of the result values. It should have the same size as results. + llvm::SmallVector result_devices; + // The operations to be included in the body of the function. + llvm::SmallVector ops; + + func::FuncOp partition_op; +}; + +// Returns a map that maps the host address to the metadata of the function +// for that remote host. The metadata of the function specifies the input +// values, result values, result devices and the operations to be included in +// the function body. +std::optional> GetFunctionMetadatas( + func::FuncOp func_op) { + llvm::StringMap metadatas; + WalkResult result = func_op.getBody().walk([&](Operation *op) { + std::string op_host = GetHost(op); + FunctionMetadata &func_metadata = metadatas[op_host]; + func_metadata.original_name = func_op.getName(); + func_metadata.insertion_point = ++Block::iterator(func_op); + func_metadata.ops.push_back(op); + + for (Value value : op->getOperands()) { + std::string value_device = ""; + + // If the value is defined as an argument of the func_op, adds it to + // the argument list of the function that uses this op. + if (BlockArgument block_arg = mlir::dyn_cast(value)) { + if (StringAttr attr = func_op.getArgAttrOfType( + block_arg.getArgNumber(), kTFDeviceAttr)) { + value_device = attr.getValue().str(); + } + + if (GetHost(value_device) != op_host) { + op->emitOpError() << kErrorMessage; + return WalkResult::interrupt(); + } + + if (llvm::find(func_metadata.inputs, value) == + func_metadata.inputs.end()) { + func_metadata.inputs.push_back(value); + func_metadata.input_devices.push_back(value_device); + } + continue; + } + + Operation *defining_op = value.getDefiningOp(); + std::string defining_op_host = GetHost(defining_op); + FunctionMetadata &defining_func_metadata = metadatas[defining_op_host]; + + if (StringAttr attr = + defining_op->getAttrOfType(kDeviceAttr)) { + value_device = attr.getValue().str(); + } + + // If the value is used as an operand of the terminator op, adds it to + // the result list of function that defines this op. + if (op->hasTrait()) { + if (llvm::find(defining_func_metadata.results, value) == + defining_func_metadata.results.end()) { + defining_func_metadata.results.push_back(value); + defining_func_metadata.result_devices.push_back(value_device); + } + continue; + } + + if (defining_op_host != op_host) { + op->emitOpError() << kErrorMessage; + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) return std::nullopt; + + return metadatas; +} + +// Creates functions in the given module using the given FunctionMetadatas. +void CreateFunctions(ModuleOp module_op, + llvm::StringMap &metadatas) { + MLIRContext *context = module_op.getContext(); + SymbolTable symbol_table(module_op); + for (auto &iter : metadatas) { + llvm::StringRef host = iter.first(); + FunctionMetadata &metadata = iter.second; + + // Do not create any new function for the operations on the localhost. + if (IsOnLocalHost(host)) continue; + + llvm::SmallVector input_types; + llvm::SmallVector result_types; + for (Value input : metadata.inputs) { + input_types.push_back(input.getType()); + } + for (Value result : metadata.results) { + result_types.push_back(result.getType()); + } + + // Replaces ':' and '/' with '_' in the host name and uses the resulting + // string as the function name. + std::string func_name = + absl::StrCat(iter.second.original_name.str(), ":", host.str()); + std::replace(func_name.begin(), func_name.end(), ':', '_'); + std::replace(func_name.begin(), func_name.end(), '/', '_'); + + FunctionType func_type = + FunctionType::get(context, input_types, result_types); + Location loc = metadata.ops.front()->getLoc(); + func::FuncOp func_op = func::FuncOp::create(loc, func_name, func_type); + // Sets the device attribute for every input and every result of the + // function. + for (int i : llvm::seq(0, metadata.input_devices.size())) { + func_op.setArgAttr(i, kTFDeviceAttr, + StringAttr::get(context, metadata.input_devices[i])); + } + for (int i : llvm::seq(0, metadata.result_devices.size())) { + func_op.setResultAttr( + i, kTFDeviceAttr, + StringAttr::get(context, metadata.result_devices[i])); + } + + func_op->setAttr(kHostAttr, StringAttr::get(context, host)); + func_op.setPublic(); + Block *block = func_op.addEntryBlock(); + + // Clones and moves the operations into the function's body. And the cloned + // operation should use the arguments of the newly created func_op as + // appropriate. + OpBuilder builder(block, block->end()); + IRMapping mapping; + for (int i : llvm::seq(0, metadata.inputs.size())) { + Value original_value = metadata.inputs[i]; + Value new_value = func_op.getArgument(i); + mapping.map(original_value, new_value); + } + for (Operation *op : metadata.ops) { + builder.clone(*op, mapping); + } + // Creates the ReturnOp so that the per-host function returns the + // correct values of the cloned operations. + llvm::SmallVector results_after_mapping; + for (Value result : metadata.results) { + results_after_mapping.push_back(mapping.lookupOrDefault(result)); + } + func::ReturnOp::create(builder, loc, results_after_mapping); + symbol_table.insert(func_op, metadata.insertion_point++); + // Record the actual name. The symbol table might rename the FuncOp if there + // is name collision. + metadata.partition_name = func_op.getName(); + } +} + +// Creates a tf_device.remote_run call for every remote function. And replaces +// usages of the results of the original operations with the results of the +// tf_device.remote_run calls. +void CreateRemoteRunCalls(MLIRContext *context, + const llvm::StringMap &metadatas) { + IRMapping mapping; + for (auto &iter : metadatas) { + llvm::StringRef host = iter.first(); + const FunctionMetadata &metadata = iter.second; + + // Do not create tf_device.remote_run call for the operations already placed + // on the localhost. + if (IsOnLocalHost(host)) continue; + + // Creates the tf_device.remote_run operation. + OpBuilder builder(metadata.ops.back()); + llvm::SmallVector result_types; + for (Value result : metadata.results) { + result_types.push_back(result.getType()); + } + Location loc = metadata.ops.front()->getLoc(); + llvm::SmallVector inputs_after_mapping; + for (Value input : metadata.inputs) { + inputs_after_mapping.push_back(mapping.lookupOrDefault(input)); + } + + tf_device::RemoteRunOp remote_run_op = tf_device::RemoteRunOp::create( + builder, loc, result_types, host, metadata.partition_name, + inputs_after_mapping); + // Clones the tf_device.remote_run operation to replace its callee args with + // the results of the other tf_device.remote_run operations using the + // `mapping` as appropriate. + Operation *cloned_remote_run_op = + builder.clone(*remote_run_op.getOperation(), mapping); + remote_run_op.erase(); + + // Replaces usages of the results of the original operations with the + // results of the tf_device.remote_run operations. + for (int i : llvm::seq(0, metadata.results.size())) { + Value original_value = metadata.results[i]; + Value new_value = cloned_remote_run_op->getResult(i); + original_value.replaceAllUsesWith(new_value); + mapping.map(original_value, new_value); + } + } +} + +#define GEN_PASS_DEF_CLUSTERTFOPSBYHOSTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class ClusterTFOpsByHostPass + : public impl::ClusterTFOpsByHostPassBase { + void runOnOperation() override { + MLIRContext *context = &getContext(); + ModuleOp module_op = getOperation(); + SmallVector original_func; + for (auto func_op : module_op.getOps()) { + original_func.push_back(func_op); + } + for (auto func_op : original_func) { + std::optional> metadatas = + GetFunctionMetadatas(func_op); + if (!metadatas) { + signalPassFailure(); + return; + } + + CreateFunctions(module_op, *metadatas); + CreateRemoteRunCalls(context, *metadatas); + + // Erases the original operations which have been cloned in the remote + // functions. + for (auto &iter : *metadatas) { + llvm::StringRef host = iter.first(); + FunctionMetadata &metadata = iter.second; + // Do not erase operations placed on the localhost. + if (IsOnLocalHost(host)) continue; + + for (int i = metadata.ops.size() - 1; i >= 0; i--) { + metadata.ops[i]->erase(); + } + } + } + } +}; + +} // namespace + +std::unique_ptr> CreateClusterTFOpsByHostPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..7173a467fe882de67e0d2f50ca91512c1e9a2d9c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.cc @@ -0,0 +1,333 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h" + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" + +namespace mlir { +namespace TF { +namespace collection_ops_util { + +Value CreateScalarConst(int32_t value, OpBuilder builder, Location loc) { + auto attr = DenseIntElementsAttr::get( + tensorflow::GetTypeFromTFTensorShape({}, builder.getI32Type()), value); + return TF::ConstOp::create(builder, loc, attr); +} + +Value GetR1Const(ArrayRef r1, OpBuilder builder, Location loc, + int bitwidth) { + llvm::SmallVector values; + int64_t rank = r1.size(); + values.reserve(rank); + for (int i = 0; i < rank; ++i) values.push_back(APInt(bitwidth, r1[i])); + auto result_type = tensorflow::GetTypeFromTFTensorShape( + {rank}, IntegerType::get(builder.getContext(), bitwidth)); + return TF::ConstOp::create(builder, loc, + DenseElementsAttr::get(result_type, values)); +} + +Value GetIndicesForElement(Value index, Value buffer, OpBuilder builder, + Location loc) { + auto buffer_type = mlir::cast(buffer.getType()); + if (buffer_type.getShape().size() == 1) return index; + // Create a concat of index and trailing zeros. + llvm::SmallVector zeros(buffer_type.getShape().size() - 1, 0); + auto zeros_tensor = GetR1Const(zeros, builder, loc); + return TF::ConcatV2Op::create( + builder, loc, + ArrayRef{tensorflow::GetTypeFromTFTensorShape( + {static_cast(buffer_type.getShape().size())}, + getElementTypeOrSelf(index.getType()))}, + ArrayRef{index, zeros_tensor, CreateScalarConst(0, builder, loc)}); +} + +Value GetElement(Value index, Value buffer, OpBuilder builder, Location loc, + bool keep_slice_shape) { + auto buffer_type = mlir::cast(buffer.getType()); + // Create a slice then reshape to remove the leading trivial dimension of + // size 1. + llvm::SmallVector slice_size = + llvm::to_vector<8>(buffer_type.getShape()); + slice_size[0] = 1; + auto size_const = GetR1Const(slice_size, builder, loc); + auto slice_type = tensorflow::GetTypeFromTFTensorShape( + slice_size, buffer_type.getElementType()); + auto slice = TF::SliceOp::create( + builder, loc, ArrayRef{slice_type}, + ArrayRef{buffer, GetIndicesForElement(index, buffer, builder, loc), + size_const}); + if (keep_slice_shape) return slice; + auto element_type = tensorflow::GetTypeFromTFTensorShape( + buffer_type.getShape().drop_front(), buffer_type.getElementType()); + auto reshape = TF::ReshapeOp::create( + builder, loc, ArrayRef{element_type}, + ArrayRef{slice, + GetR1Const(element_type.getShape(), builder, loc)}); + return reshape.getOutput(); +} + +Value SetElement(Value index, Value buffer, Value element, OpBuilder builder, + Location loc) { + auto buffer_type = mlir::cast(buffer.getType()); + // Reshape the element to add a leading dimension of size 1 if th element does + // not have that dimension, then perform a dynamic update slice. + auto slice_shape = llvm::to_vector<8>(buffer_type.getShape()); + slice_shape[0] = 1; + auto slice_type = tensorflow::GetTypeFromTFTensorShape( + slice_shape, buffer_type.getElementType()); + auto update_slice = element; + if (element.getType() != slice_type) { + update_slice = TF::ReshapeOp::create( + builder, loc, ArrayRef{slice_type}, + ArrayRef{element, GetR1Const(slice_shape, builder, loc)}); + } + return TF::XlaDynamicUpdateSliceOp::create( + builder, loc, ArrayRef{buffer.getType()}, + ArrayRef{buffer, update_slice, + GetIndicesForElement(index, buffer, builder, loc)}) + .getOutput(); +} + +TensorType GetSizeType(OpBuilder builder) { + return tensorflow::GetTypeFromTFTensorShape({1}, builder.getIntegerType(32)); +} + +Value ReshapeScalarToSizeType(OpBuilder builder, Value scalar, Location loc) { + auto size_type = GetSizeType(builder); + return TF::ReshapeOp::create( + builder, loc, ArrayRef{size_type}, + ArrayRef{scalar, GetR1Const(size_type.getShape(), builder, loc)}); +} + +LogicalResult CreateInitBufferValue(ArrayRef element_shape, + Value max_size, Operation* op, + Type element_dtype, OpBuilder builder, + Value* buffer) { + auto max_count_op = max_size.getDefiningOp(); + if (!max_count_op) return op->emitOpError("unknown max element count"); + auto max_count_const_op = llvm::dyn_cast(max_count_op); + if (!max_count_const_op) return op->emitOpError("unknown max element count"); + int64_t max_size_const = + (*max_count_const_op.getValue().getValues().begin()) + .getSExtValue(); + return CreateInitBufferValue(element_shape, max_size_const, op, element_dtype, + builder, buffer); +} + +LogicalResult CreateInitBufferValue(ArrayRef element_shape, + int64_t max_size, Operation* op, + Type element_dtype, OpBuilder builder, + Value* buffer) { + llvm::SmallVector buffer_shape; + buffer_shape.push_back(max_size); + for (int64_t dim : element_shape) { + buffer_shape.push_back(dim); + } + auto zero = CreateScalarConst(0, builder, op->getLoc()); + if (getElementTypeOrSelf(zero.getType()) != element_dtype) { + zero = TF::CastOp::create( + builder, op->getLoc(), + ArrayRef{tensorflow::GetTypeFromTFTensorShape({}, element_dtype)}, + ArrayRef{zero}); + } + auto buffer_type = + tensorflow::GetTypeFromTFTensorShape(buffer_shape, element_dtype); + auto broadcast = TF::BroadcastToOp::create( + builder, op->getLoc(), ArrayRef{buffer_type}, + ArrayRef{zero, GetR1Const(buffer_shape, builder, op->getLoc())}); + *buffer = broadcast.getOutput(); + return success(); +} + +std::optional GetElementTypeFromAccess( + Value collection, ModuleOp module, + llvm::function_ref(Operation*)> infer_from_op) { + for (auto& use : collection.getUses()) { + if (auto while_op = llvm::dyn_cast(use.getOwner())) { + auto body = while_op.body_function(); + assert(body); + auto type_from_body = GetElementTypeFromAccess( + body.getArgument(use.getOperandNumber()), module, infer_from_op); + if (type_from_body.has_value()) return type_from_body; + } else if (auto if_op = llvm::dyn_cast(use.getOwner())) { + auto then_branch = if_op.then_function(); + auto else_branch = if_op.else_function(); + assert(then_branch && else_branch); + auto type_from_then = GetElementTypeFromAccess( + then_branch.getArgument(use.getOperandNumber() - 1), module, + infer_from_op); + if (type_from_then.has_value()) return type_from_then; + auto type_from_else = GetElementTypeFromAccess( + else_branch.getArgument(use.getOperandNumber() - 1), module, + infer_from_op); + if (type_from_else.has_value()) return type_from_else; + } else if (auto call = llvm::dyn_cast(use.getOwner())) { + auto callee = dyn_cast(call.resolveCallable()); + auto type_from_callee = GetElementTypeFromAccess( + callee.getArgument(use.getOperandNumber()), module, infer_from_op); + if (type_from_callee.has_value()) return type_from_callee; + } else if (llvm::isa(use.getOwner())) { + auto type_from_alias = GetElementTypeFromAccess( + use.getOwner()->getResult(use.getOperandNumber()), module, + infer_from_op); + if (type_from_alias.has_value()) return type_from_alias; + } else if (auto type = infer_from_op(use.getOwner())) { + if (!type) continue; + auto elem_type = mlir::dyn_cast(*type); + if (elem_type && elem_type.hasStaticShape()) return elem_type; + } + } + return std::nullopt; +} + +// Creates a ReadVariableOp on a local variable. +Value ReadLocalVariable(Value local_var, OpBuilder builder, Location loc) { + return TF::ReadVariableOp::create( + builder, loc, + ArrayRef{mlir::cast( + getElementTypeOrSelf(local_var.getType())) + .getSubtypes()[0]}, + ArrayRef{local_var}) + .getValue(); +} + +// Creates an AssignVariableOp on a local variable. +TF::AssignVariableOp WriteLocalVariable(Value local_var, Value value, + OpBuilder builder, Location loc) { + return TF::AssignVariableOp::create(builder, loc, ArrayRef{}, + ArrayRef{local_var, value}); +} + +Value AccumulateBuffers(Value a, Value b, OpBuilder builder, Location loc) { + if (getElementTypeOrSelf(a.getType()) == builder.getI1Type()) { + return TF::LogicalOrOp::create(builder, loc, ArrayRef{a.getType()}, + ArrayRef{a, b}); + } + return TF::AddV2Op::create(builder, loc, ArrayRef{a.getType()}, + ArrayRef{a, b}); +} + +namespace { + +int64_t GetFirstIfIndicesAreContiguous(Value indices) { + auto type = mlir::dyn_cast(indices.getType()); + if (!type) return -1; + auto indices_op = indices.getDefiningOp(); + if (!indices_op) return -1; + auto const_op = llvm::dyn_cast(indices_op); + if (!const_op) return -1; + int64_t last_index = -1; + int64_t first_index = -1; + for (const auto& ind : const_op.getValue().getValues()) { + if (last_index == -1) { + last_index = ind.getSExtValue(); + first_index = last_index; + continue; + } + if (last_index + 1 != ind.getSExtValue()) return -1; + last_index++; + } + return first_index; +} + +} // namespace + +Value GatherElements(Value indices, Value buffer, OpBuilder builder, + Location loc) { + auto buffer_type = mlir::cast(buffer.getType()); + auto result_shape = llvm::to_vector<8>(buffer_type.getShape()); + result_shape[0] = + mlir::cast(indices.getType()).getDimSize(0); + int64_t maybe_contiguous_start = GetFirstIfIndicesAreContiguous(indices); + if (maybe_contiguous_start >= 0) { + llvm::SmallVector slice_starts(result_shape.size(), 0); + slice_starts[0] = maybe_contiguous_start; + auto slice_type = tensorflow::GetTypeFromTFTensorShape( + result_shape, buffer_type.getElementType()); + return TF::SliceOp::create( + builder, loc, ArrayRef{slice_type}, + ArrayRef{buffer, GetR1Const(slice_starts, builder, loc), + GetR1Const(result_shape, builder, loc)}); + } + auto result_type = tensorflow::GetTypeFromTFTensorShape( + result_shape, buffer_type.getElementType()); + return TF::GatherV2Op::create( + builder, loc, ArrayRef{result_type}, + ArrayRef{buffer, indices, CreateScalarConst(0, builder, loc)}); +} + +Value ScatterAccumulateElements(Value indices, Value updates, Value buffer, + OpBuilder builder, Location loc) { + auto buffer_type = mlir::cast(buffer.getType()); + auto updates_type = mlir::cast(updates.getType()); + int64_t maybe_contiguous_start = GetFirstIfIndicesAreContiguous(indices); + if (maybe_contiguous_start == 0 && buffer_type == updates_type) { + return AccumulateBuffers(buffer, updates, builder, loc); + } + // We cannot simply use a TensorScatterUpdate, as it does not accumulate with + // the old data; it is tricky to manually add the old data either, since there + // could be duplicates in the index. We follow the old bridge's approach by + // iterating through the indices. + auto per_slice_shape = llvm::to_vector<8>(buffer_type.getShape()); + per_slice_shape[0] = 1; + auto slice_sizes = GetR1Const(per_slice_shape, builder, loc); + llvm::SmallVector starts_in_update(buffer_type.getRank(), 0); + for (int64_t i = 0; i < updates_type.getDimSize(0); ++i) { + auto index = TF::SliceOp::create( + builder, loc, ArrayRef{GetSizeType(builder)}, + ArrayRef{indices, GetR1Const({i}, builder, loc), + GetR1Const({1}, builder, loc)}); + auto old_slice = + GetElement(index, buffer, builder, loc, /*keep_slice_shape=*/true); + starts_in_update[0] = i; + auto update_slice_starts = GetR1Const(starts_in_update, builder, loc); + Value slice = + TF::SliceOp::create( + builder, loc, ArrayRef{old_slice.getType()}, + ArrayRef{updates, update_slice_starts, slice_sizes}) + .getOutput(); + slice = AccumulateBuffers(old_slice, slice, builder, loc); + buffer = SetElement(index, buffer, slice, builder, loc); + } + return buffer; +} + +} // namespace collection_ops_util +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h new file mode 100644 index 0000000000000000000000000000000000000000..187a7d76b0251b0a35f697a68c42a7bce919e468 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h @@ -0,0 +1,109 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_COLLECTION_OPS_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_COLLECTION_OPS_UTIL_H_ + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TF { +namespace collection_ops_util { + +// This file includes utilities for decomposing collection ops (stack, tensor +// list, tensor array) in TF. We represent such a data structure as a buffer of +// shape [max_element_count, element_shape]. + +// Creates an i32 scalar tf.Const. +Value CreateScalarConst(int32_t value, OpBuilder builder, Location loc); + +// Creates an integer vector tf.Const. +Value GetR1Const(ArrayRef r1, OpBuilder builder, Location loc, + int bitwidth = 32); + +// Returns the type of the size tensor used to track a data structure's element +// count. It is a tensor<1xi32>, and we use R1 instead of a scalar because it is +// easier to concat it with other offsets. +TensorType GetSizeType(OpBuilder builder); + +// Reshapes a scalar value to match the size type tensor. +Value ReshapeScalarToSizeType(OpBuilder builder, Value scalar, Location loc); + +// Creates ops that represent the indices of the slice for an element in the +// buffer. Requires `index` to have tensor<1xi32> type. +Value GetIndicesForElement(Value index, Value buffer, OpBuilder builder, + Location loc); + +// Creates ops that slice the element out of a buffer at the given index. +// Requires `index` to have tensor<1xi32> type. +Value GetElement(Value index, Value buffer, OpBuilder builder, Location loc, + bool keep_slice_shape = false); + +// Creates ops that copy the buffer and update an element at the given index. +// Requires `index` to have tensor<1xi32> type. +Value SetElement(Value index, Value buffer, Value element, OpBuilder builder, + Location loc); + +// Creates the buffer for the data structure with given element shape, type and +// maximum size. +LogicalResult CreateInitBufferValue(ArrayRef element_shape, + int64_t max_size, Operation* op, + Type element_dtype, OpBuilder builder, + Value* buffer); + +// Same as above, but uses a Value as max_size and check if it is a constant. +LogicalResult CreateInitBufferValue(ArrayRef element_shape, + Value max_size, Operation* op, + Type element_dtype, OpBuilder builder, + Value* buffer); + +// Tries to infer the element type with full shape based its write accesses. +// `infer_from_user` should check if the provided op is an accessing op that +// could be used to infer the type. +std::optional GetElementTypeFromAccess( + Value collection, ModuleOp module, + llvm::function_ref(Operation*)> infer_from_op); + +// Creates a ReadVariableOp on a local variable. +Value ReadLocalVariable(Value local_var, OpBuilder builder, Location loc); + +// Creates an AssignVariableOp on a local variable. +TF::AssignVariableOp WriteLocalVariable(Value local_var, Value value, + OpBuilder builder, Location loc); + +// Adds two values, or creates a logical-or if they are boolean type. +Value AccumulateBuffers(Value a, Value b, OpBuilder builder, Location loc); + +// Gathers elements in buffer with the indices. +Value GatherElements(Value indices, Value buffer, OpBuilder builder, + Location loc); + +// Scatters elements into buffer, where each scattered element is accumulated +// with the old value in buffer. +Value ScatterAccumulateElements(Value indices, Value updates, Value buffer, + OpBuilder builder, Location loc); + +} // namespace collection_ops_util +} // namespace TF +} // namespace mlir +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_COLLECTION_OPS_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/colocate_tpu_copy_with_dynamic_shape.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/colocate_tpu_copy_with_dynamic_shape.cc new file mode 100644 index 0000000000000000000000000000000000000000..c686c4b1d8e475379128b2dc4e0270cc1ae0c5ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/colocate_tpu_copy_with_dynamic_shape.cc @@ -0,0 +1,159 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/Support/Casting.h" +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/SparseAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlowFramework.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/ControlFlowInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDevice[] = "device"; + +#define GEN_PASS_DEF_COLOCATETPUCOPYWITHDYNAMICSHAPEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ColocateTPUCopyWithDynamicShapePass + : public impl::ColocateTPUCopyWithDynamicShapePassBase< + ColocateTPUCopyWithDynamicShapePass> { + void runOnOperation() override; +}; + +class Device : public dataflow::AbstractSparseLattice { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(Device) + using AbstractSparseLattice::AbstractSparseLattice; + + ChangeResult meet(const dataflow::AbstractSparseLattice &other) override { + const auto *otherDevice = reinterpret_cast(&other); + if (otherDevice->device_) { + return SetDevice(otherDevice->device_); + } + return ChangeResult::NoChange; + } + + void print(raw_ostream &os) const override { + if (device_) { + os << "<" << device_ << ">"; + } else { + os << ""; + } + } + + ChangeResult SetDevice(mlir::StringAttr device) { + bool changed = (device != device_); + device_ = device; + return changed ? ChangeResult::Change : ChangeResult::NoChange; + } + + bool hasDevice() const { return !!device_; } + + mlir::StringAttr getDevice() const { return device_; } + + private: + mutable mlir::StringAttr device_; +}; + +class DeviceDataflowAnalysis + : public ::mlir::dataflow::SparseBackwardDataFlowAnalysis { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(DeviceDataflowAnalysis) + using ::mlir::dataflow::SparseBackwardDataFlowAnalysis< + Device>::SparseBackwardDataFlowAnalysis; + ~DeviceDataflowAnalysis() override = default; + + LogicalResult visitOperation(Operation *op, ArrayRef operands, + ArrayRef results) override { + if (llvm::isa(op) || + llvm::isa(op)) { + auto device = op->getAttrOfType(kDevice); + for (auto *operand : operands) + propagateIfChanged(operand, operand->SetDevice(device)); + } else { + // Propagate device through other ops. These ops might have their + // own device annotation, but that's fine. We only care about + // where the TPUExecute ops live. + StringAttr device; + for (const Device *d : results) { + if (d->hasDevice()) { + device = d->getDevice(); + break; + } + } + for (auto *operand : operands) + propagateIfChanged(operand, operand->SetDevice(device)); + } + return mlir::success(); + } + void visitBranchOperand(OpOperand &operand) override {} + void visitCallOperand(OpOperand &operand) override {} + void visitNonControlFlowArguments( + RegionSuccessor& successor, ArrayRef arguments) override {} + void setToExitState(Device *lattice) override {} +}; + +void ColocateTPUCopyWithDynamicShapePass::runOnOperation() { + Operation *module = getOperation(); + + SymbolTableCollection symbolTables; + + DataFlowSolver solver; + solver.load(); + solver.load(); + solver.load(symbolTables); + if (failed(solver.initializeAndRun(module))) return signalPassFailure(); + + module->walk([&](TF::TPUCopyWithDynamicShapeOp op) { + const Device *state; + for (auto result : op->getResults()) { + state = solver.lookupState(result); + if (state) break; + } + if (!state || !state->hasDevice()) { + return WalkResult::advance(); + } + op->setAttr(kDevice, state->getDevice()); + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> +CreateColocateTPUCopyWithDynamicShapePass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.cc new file mode 100644 index 0000000000000000000000000000000000000000..1405a9b78f1a24e9292f635877b5fd5ad05cec28 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.cc @@ -0,0 +1,134 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.h" + +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeRange.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.h" + +namespace mlir { +namespace TF { + +// Implements a TF specific policy on when constant folding is allowed. +// Policy: +// +// Find the total size of the operands and results, ignoring types with +// undefined bit widths and unknown shapes. +// Disable constant folding if operands size is greater than a certain threshold +// (`kOperandsSizeThreshold`). +// +// Otherwise, allow folding if: +// 1. size of results is less than a certain threshold +// (`kResultsSizeThreshold`), or +// 2. size of results is within a factor (`kSizeFactor`) of size of operands. +// TODO(b/157226221): Look into other heuristics for constant fold policy. +static bool IsFoldedByDefaultPolicy(Operation* inst) { + auto get_size = [&](TypeRange types) { + int64_t size = 0; + for (auto t : types) { + auto tensor_type = mlir::cast(t); + // Ignore types with undefined bit widths. + if (!tensor_type.getElementType().isIntOrFloat()) continue; + // Ignore types with dynamic shapes. + if (!tensor_type.hasStaticShape()) continue; + size += tensor_type.getNumElements() * + tensor_type.getElementType().getIntOrFloatBitWidth(); + } + return size; + }; + + int64_t results_size = get_size(inst->getResultTypes()); + int64_t operands_size = get_size(inst->getOperandTypes()); + + constexpr int kSizeFactor = 2; + constexpr int64_t kResultsSizeThreshold = (1 << 16); // 64 Kib = 8 KiB + constexpr int64_t kOperandsSizeThreshold = (1 << 30); // 1 Gib = 128 MiB + + return (operands_size <= kOperandsSizeThreshold) && + ((results_size <= kResultsSizeThreshold) || + (results_size <= kSizeFactor * operands_size)); +} + +LogicalResult ConstantFoldFallbackHook( + Operation* inst, ArrayRef operands, + SmallVectorImpl& results) { // NOLINT + if (!CanBeFolded(inst)) return failure(); + + // Determine if we should attempt to fold this operation by considering the + // size/size increase due to folding. + if (!IsFoldedByDefaultPolicy(inst)) return failure(); + + // If all the results are empty and has numerical element types, set results + // to empty elements attribute. This is restricted to the numerical element + // types as the DenseElementsAttr only supports numerical and string types. + // TODO(hinsu): Handle ops that have one of the results empty for constant + // propagation. + bool has_empty_numerical_results = + llvm::all_of(inst->getResultTypes(), [](Type ty) { + ShapedType shaped_ty = mlir::cast(ty); + Type element_ty = shaped_ty.getElementType(); + return shaped_ty.hasStaticShape() && shaped_ty.getNumElements() == 0 && + element_ty.isIntOrFloat(); + }); + if (has_empty_numerical_results && + // TODO(jpienaar): Remove this once some unmodeled op behavior is + // addressed. + inst->isRegistered()) { + for (Type ty : inst->getResultTypes()) { + auto shaped_ty = mlir::cast(ty); + results.push_back( + DenseElementsAttr::get(shaped_ty, llvm::ArrayRef())); + } + return success(); + } + + // Returns directly if any of the operands is not an elements attributes. + if (std::any_of(operands.begin(), operands.end(), [](Attribute attr) { + return !attr || !mlir::isa(attr); + })) + return failure(); + + SmallVector inputs; + inputs.reserve(operands.size()); + for (auto input : operands) { + inputs.push_back(mlir::cast(input)); + } + + SmallVector constants; + LogicalResult status = EvaluateOperation(inst, inputs, constants); + results.assign(constants.begin(), constants.end()); + return status; +} + +static bool init_hooks = ([] () { + TensorFlowDialect::RegisterConstantFoldHook(ConstantFoldFallbackHook); +}(), true); + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.h new file mode 100644 index 0000000000000000000000000000000000000000..a835fd06ae0ed5e9d9b135dc3321cdd17b5c362c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.h @@ -0,0 +1,35 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TF { + +LogicalResult ConstantFoldFallbackHook( + Operation *inst, ArrayRef operands, + SmallVectorImpl &results); // NOLINT + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..53ea66693067553950ff9933aa4ee1bce88e263b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.cc @@ -0,0 +1,204 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/fallback/op_kernel_runner.h" + +namespace mlir { +namespace TF { + +using tensorflow::tfrt_stub::FallbackState; +using tensorflow::tfrt_stub::OpKernelRunner; + +static bool IsOk(const absl::Status& s) { + if (s.ok()) return true; + VLOG(2) << s.message(); + return false; +} + +#define RETURN_FAILURE_IF_ERROR(expr) \ + if (!IsOk(expr)) { \ + return mlir::failure(); \ + } + +bool CanBeFolded(Operation* inst) { + // Instructions with side effects should not be constant folded to preserve + // the original semantics. Ops that have no side effect and zero results but + // could be folded should have a custom folder instead of relying on the + // TensorFlow folding hook. + if (inst == nullptr || inst->getNumResults() == 0 || + inst->hasTrait() || + inst->getNumRegions() != 0 || !isMemoryEffectFree(inst)) { + return false; + } + + // If any of the result types are variants, don't try to constant fold them. + // This creates opaque variant constants which lose information and would + // require "raising" later. + for (const Type type : inst->getResultTypes()) { + if (const TensorType tensor_type = mlir::dyn_cast(type)) { + if (mlir::isa(tensor_type.getElementType())) { + return false; + } + } + } + + // Operations that execute function calls shouldn't be constant folded. + if (llvm::isa(inst)) { + return false; + } + + return true; +} + +static const FallbackState& GetDefaultFallbackState() { + static const auto* const fallback_state = []() { + tensorflow::SessionOptions session_options; + tensorflow::FunctionDefLibrary fdef_lib; + auto fallback_state = + FallbackState::CreateWithCpuDevice(session_options, fdef_lib).value(); + return fallback_state.release(); + }(); + + return *fallback_state; +} + +static std::function)>* GetDefaultRunner() { + static auto* const default_runner = + new std::function)>( + [](const std::function& f) { f(); }); + return default_runner; +} + +LogicalResult EvaluateOperation(Operation* inst, + llvm::ArrayRef operands, + llvm::SmallVector& results) { + // If any operand is nullptr returns true for a failure. + // TODO(b/120678030): remove this constraint if we find operators can be + // evaluated with some unknown operands. + if (std::any_of(operands.begin(), operands.end(), + [](Attribute operand) { return !operand; })) { + VLOG(1) << "Can't evaluate since not all operands are constant."; + return failure(); + } + + // Builds TF operation and sets all the attributes. + std::string node_name = "unnamed"; + if (const StringAttr attr = inst->getAttrOfType("name")) { + node_name = std::string(attr.getValue()); + } + absl::StatusOr> node_def = + tensorflow::ConvertTFDialectOpToNodeDef( + inst, node_name.c_str(), /*ignore_unregistered_attrs=*/true); + RETURN_FAILURE_IF_ERROR(node_def.status()); + + const FallbackState& fallback_state = GetDefaultFallbackState(); + + // Explicitly set device to Host CPU instead of the device present in device + // attribute of the MLIR op. The assigned device might be remote, not + // available during compilation or compilation only device for on demand + // execution which may create a recursion if used for constant folding. + std::string host_cpu = tensorflow::DeviceNameUtils::FullName( + /*job=*/"localhost", /*replica=*/0, /*task=*/0, /*type=*/"CPU", /*id=*/0); + + absl::StatusOr runner = OpKernelRunner::Create( + node_def->get()->op(), node_def->get()->name(), host_cpu, operands.size(), + [&](tensorflow::AttrValueMap* attr_value_map) { + *attr_value_map = node_def->get()->attr(); + return absl::OkStatus(); + }, + fallback_state.device_manager(), + fallback_state.process_function_library_runtime()); + RETURN_FAILURE_IF_ERROR(runner.status()); + + VLOG(1) << "Start to evaluate node: " << node_def->get()->DebugString(); + + std::vector inputs; + + // Adds inputs to the TF operation. + for (const ElementsAttr& operand : operands) { + tensorflow::Tensor tensor; + RETURN_FAILURE_IF_ERROR(tensorflow::ConvertToTensor(operand, &tensor)); + inputs.push_back(std::move(tensor)); + } + + std::vector input_values; + for (tensorflow::Tensor& tensor : inputs) { + input_values.emplace_back(); + input_values.back().tensor = &tensor; + } + + tensorflow::OpKernelContext::Params params; + params.inputs = input_values; + params.device = runner->device(); + params.op_kernel = runner->op_kernel(); + + // Still use original device's resource_manager. + params.resource_manager = runner->resource_manager(); + params.input_alloc_attrs = runner->input_alloc_attrs(); + params.output_attr_array = runner->output_alloc_attrs().data(); + + // Following two parameters are used to support executing tf.data via + // fallback. + params.function_library = runner->function_library_runtime(); + params.runner = GetDefaultRunner(); + + // Executes the TF operation. + tensorflow::OpKernelContext op_kernel_context(¶ms); + runner->Run(&op_kernel_context); + RETURN_FAILURE_IF_ERROR(op_kernel_context.status()); + + // Converts the outputs to MLIR attributes. + Builder builder(inst->getContext()); + + for (int i = 0; i < op_kernel_context.num_outputs(); ++i) { + DCHECK(op_kernel_context.mutable_output(i)); + absl::StatusOr result_attr = tensorflow::ConvertTensor( + *op_kernel_context.mutable_output(i), &builder); + RETURN_FAILURE_IF_ERROR(result_attr.status()); + results.push_back(result_attr.value()); + } + + VLOG(1) << "Evaluate node " << node_name << " successfully!"; + + return success(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..0ceda2af639a25ec9f3c6ed75feb1ea5109caa86 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_fold_utils.h @@ -0,0 +1,38 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_UTILS_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Checks whether the given TF operation can be folded or not. +bool CanBeFolded(Operation* inst); + +// Evaluates the operation with given operand values. +LogicalResult EvaluateOperation(Operation* inst, + llvm::ArrayRef operands, + llvm::SmallVector& results); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_CONSTANT_FOLD_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_op_device_assignment.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_op_device_assignment.cc new file mode 100644 index 0000000000000000000000000000000000000000..7761af0bd67145ba1dcf201440777424b957d471 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/constant_op_device_assignment.cc @@ -0,0 +1,93 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass adds the device attribute to every tf.Const op based on the device +// attribute of the operations that read its result. If the result of a tf.Const +// op is read by operations placed on multiple devices, then the pass will +// replicate the tf.Const op once for each device. + +#include + +#include "mlir/IR/Builders.h" +#include "mlir/Pass/Pass.h" +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +constexpr const char *kDeviceAttr = "device"; + +#define GEN_PASS_DEF_CONSTANTOPDEVICEASSIGNMENTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ConstantOpDeviceAssignmentPass + : public impl::ConstantOpDeviceAssignmentPassBase< + ConstantOpDeviceAssignmentPass> { + void runOnOperation() override; +}; + +void ConstantOpDeviceAssignmentPass::runOnOperation() { + ModuleOp module = getOperation(); + + module.walk([&](TF::ConstOp op) { + // Keep the ConstOp if the op already have the device attribute. + if (StringAttr device_attr = op->getAttrOfType(kDeviceAttr)) { + return WalkResult::advance(); + } + OpBuilder builder(op); + llvm::StringMap cloned_op_by_device; + bool all_uses_replaced = true; + + for (mlir::OpOperand &use : + llvm::make_early_inc_range(op.getResult().getUses())) { + mlir::Operation *user_op = use.getOwner(); + StringAttr device_attr = user_op->getAttrOfType(kDeviceAttr); + if (!device_attr) { + all_uses_replaced = false; + continue; + } + // Cloned the ConstOp and set its device attribute to be the same as the + // device of the user operation. + if (cloned_op_by_device.find(device_attr.getValue()) == + cloned_op_by_device.end()) { + mlir::Operation *new_op = builder.clone(*op.getOperation()); + new_op->setAttr(kDeviceAttr, device_attr); + cloned_op_by_device[device_attr.getValue()] = new_op; + } + // Update the user operation to use the result of the cloned ConstOp. + mlir::Operation *new_op = cloned_op_by_device[device_attr.getValue()]; + user_op->setOperand(use.getOperandNumber(), new_op->getResult(0)); + } + // Erase the original ConstOp if all its uses have been replaced. + if (all_uses_replaced) { + op.erase(); + } + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> +CreateConstantOpDeviceAssignmentPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_control_to_data_outputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_control_to_data_outputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..7ed2a99cf668ce65ead726db8a1b76b23fffeb59 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_control_to_data_outputs.cc @@ -0,0 +1,673 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/EquivalenceClasses.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/SparseAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlowFramework.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_dataflow.h" +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_remaining_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" + +namespace mlir { +namespace tf_executor { +namespace { + +using TF::ResourceId; +using ResourceAndDevice = std::pair; +static constexpr ResourceId kUnknownResourceId = + TF::detail::ResourceAliasAnalysisInfo::kUnknownResourceId; +static constexpr ResourceId kInvalidResourceId = + TF::detail::ResourceAliasAnalysisInfo::kInvalidResourceId; +using OperationSetTy = SmallPtrSet; +using ResourceToOpsMapTy = DenseMap; +using DeviceMap = DenseMap; +constexpr int64_t kAnyDevice = 0; +constexpr ResourceAndDevice kInvalidResourceAndDevice{kInvalidResourceId, + kAnyDevice}; +constexpr ResourceAndDevice kUnknownResourceAndDevice{kUnknownResourceId, + kAnyDevice}; + +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_EXECUTORCONVERTCONTROLTODATAOUTPUTSPASS +#define GEN_PASS_DECL_EXECUTORCONVERTCONTROLTODATAOUTPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ConvertControlToDataOutputsPass + : public impl::ExecutorConvertControlToDataOutputsPassBase< + ConvertControlToDataOutputsPass> { + ConvertControlToDataOutputsPass() = default; + explicit ConvertControlToDataOutputsPass( + bool composite_tpuexecute_side_effects) + : ExecutorConvertControlToDataOutputsPassBase( + ExecutorConvertControlToDataOutputsPassOptions{ + composite_tpuexecute_side_effects}) {} + + void runOnOperation() override; +}; + +// Returns a vector of all tf.WhileOp(s) which use func as while body. If any of +// the uses is as a while condition, an empty vector is returned. +SmallVector GetWhileCallers(func::FuncOp func, + SymbolUserMap& symbol_map) { + SmallVector while_callers; + for (auto user : symbol_map.getUsers(func)) { + if (auto while_caller = dyn_cast(user)) { + // If used as while conditional anywhere, then skip optimizing this + // function. Return empty vector. + if (while_caller.cond_function() == func) return {}; + assert(while_caller.body_function() == func); + while_callers.push_back(while_caller); + } + } + return while_callers; +} + +bool IsResourceType(Type type) { + return mlir::isa(getElementTypeOrSelf(type)); +} + +bool OnlyOperatesOnCompositeDevices( + TF::TPUExecuteAndUpdateVariablesOp& op, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + const DataFlowSolver& solver) { + auto& alias_analysis = side_effect_analysis.GetAliasAnalysis(); + llvm::SmallSet read_array; + for (const Attribute& attr : op.getDeviceVarReadsIndices()) { + read_array.insert(mlir::cast(attr).getInt()); + } + llvm::SmallSet update_array; + for (const Attribute& attr : op.getDeviceVarUpdatesIndices()) { + update_array.insert(mlir::cast(attr).getInt()); + } + + for (auto& arg : op->getOpOperands()) { + Value v = arg.get(); + if (!IsResourceType(arg.get().getType())) continue; + if (alias_analysis.IsUnknownResource(v)) continue; + for (auto id : alias_analysis.GetResourceUniqueIds(v)) { + (void)id; + } + } + + for (auto& arg : op->getOpOperands()) { + if (!IsResourceType(arg.get().getType())) { + continue; + } + auto lattice = + solver.lookupState(arg.get())->getValue(); + bool is_read = read_array.contains(arg.getOperandNumber()); + bool is_update = update_array.contains(arg.getOperandNumber()); + // We want the resource operands that are on composite devices to be the + // exact same set as the resource operands that are read or updated. + if ((is_read || is_update) != lattice.is_on_composite_device) { + return false; + } + } + return true; +} + +// Populates `chain_resource_to_ops_map`, the map from all resources that need +// to be chained to the set of operations that access the resource, and +// `resource_equivalence_classes`. Resources are equivalent if they are accessed +// by a common op, and equivalent resources will be assigned to the same chain. +void CollectChainResources( + func::FuncOp func, ResourceToOpsMapTy& chain_resource_to_ops_map, + llvm::EquivalenceClasses& resource_equivalence_classes, + DeviceMap& devices, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + const DataFlowSolver& solver, bool composite_tpuexecute_side_effects) { + auto graph_op = cast(func.front().front()); + + // For each op in the graph, get the resources it uses and update the access + // information for them. + graph_op.walk([&](IslandOp island) { + // This pass assumes that all functions are suitable for export i.e., each + // function has a single tf_executor.graph op and all islands wrap the + // internal op perfectly. Hence this assertion should never fail. + assert(island.WrapsSingleOp()); + Operation& op = island.GetBody().front(); + + // If the op only operates on resources stored on devices that are + // "COMPOSITE", then this op is defined to work in parallel with other + // TPUExecute* ops. So we can make all ResourceIds device-specific below. + // (Even the per-op "resource ids", like ResourceEffects::TPUExecute.) + bool op_only_operates_on_composite_devices = false; + if (auto execute = llvm::dyn_cast(op)) { + if (OnlyOperatesOnCompositeDevices(execute, side_effect_analysis, + solver)) { + op_only_operates_on_composite_devices = true; + } + } + + auto device_attr = op.getAttrOfType(kDeviceAttr); + int64_t device_id; + if (!device_attr) { + device_id = kAnyDevice; + } else if (devices.find(device_attr) != devices.end()) { + device_id = devices[device_attr]; + } else { + device_id = 1 + devices.size(); + devices[device_attr] = device_id; + } + + auto& alias_analysis = side_effect_analysis.GetAliasAnalysis(); + + ResourceAndDevice prev_resource_and_device = kInvalidResourceAndDevice; + for (auto resource_id_read_only_pair : + side_effect_analysis.GetResourceIds(&op)) { + auto resource_id = resource_id_read_only_pair.first; + // If alias analysis knows about a resource (as evidenced by the fact that + // GetValuesForResourceId isn't empty), and dataflow tells us that this + // stems from a function argument that was annotated as + // "tf._composite_device", then we can treat this resource as + // device-specific (see below). + bool resource_is_on_composite_device = false; + for (Value value : alias_analysis.GetValuesForResourceId(resource_id)) { + auto lattice = solver.lookupState(value); + if (lattice) { + resource_is_on_composite_device |= + lattice->getValue().is_on_composite_device; + } + } + + // A device-specific resource identifier creates an edge only between ops + // on the same device, thus preventing ops on different devices from + // blocking each other, even if they access the same resource. + ResourceAndDevice resource_and_device; + if (composite_tpuexecute_side_effects && + (op_only_operates_on_composite_devices || + resource_is_on_composite_device)) { + resource_and_device = std::make_pair(resource_id, device_id); + } else { + resource_and_device = std::make_pair(resource_id, kAnyDevice); + } + + // If the resource was allocated by an op with `UniqueResourceAllocation` + // trait, then we don't need to chain resource ops accessing this resource + // between iterations: Every iteration will create a new independent + // resource. This enables more parallelism across iterations. + if (!side_effect_analysis.IsUniqueResourceAllocationId( + resource_and_device.first)) { + chain_resource_to_ops_map[resource_and_device].insert(&op); + if (prev_resource_and_device != kInvalidResourceAndDevice) { + // Merge class of current ID with class of previous ID since both + // resources are accessed by `op`. + resource_equivalence_classes.unionSets(prev_resource_and_device, + resource_and_device); + } else { + resource_equivalence_classes.insert(resource_and_device); + } + prev_resource_and_device = resource_and_device; + } + } + }); +} + +// tf.NoOp islands are used to combine multiple control dependencies into one. +// These islands have a single tf.NoOp inside them and consume multiple control +// outputs to generate a single control output. +// +// For example, +// ``` +// %merged_control = "tf_executor.island"(%control_a, %control_b) ({ +// "tf.NoOp"() : () -> () +// "tf_executor.yield"() : () -> () +// }) : (!tf_executor.control, !tf_executor.control) -> (!tf_executor.control) +// ``` +// +// `%merged_control` is a NoOp control barrier in this case. +// +// Checks if the value `control` is a NoOp control barrier. +bool IsNoOpControlBarrier(Value control) { + if (!mlir::isa(control.getType())) return false; + + auto control_island = dyn_cast_or_null(control.getDefiningOp()); + if (!control_island) return false; + + // All islands perfectly wrap a single op is an invariant of this pass and + // is checked at the very beginning of the pass. + assert(control_island.WrapsSingleOp()); + return control_island.getOutputs().empty() && + isa(control_island.GetBody().front()); +} + +// Remove all control outputs of the function. Traverses NoOp control barrier +// chains from FetchOp to all NoOp control barriers. Returns true +// iff at least one control output is deleted. The ops_connected_to_fetch +// set is populated with all operations that had a direct (or indirect, through +// Identity ops) control connection to the fetch. That set will contain nullptr +// if an arg is connected to the fetch. +bool RemoveAllControlOutputs( + func::FuncOp func, SmallPtrSet* ops_connected_to_fetch) { + auto graph_op = cast(func.front().front()); + + FetchOp fetch = graph_op.GetFetch(); + // Return early if no control outputs exist. + if (fetch.getNumOperands() == graph_op->getNumResults()) return false; + + std::queue control_barrier_worklist; + for (Value control_input : + fetch.getFetches().drop_front(graph_op->getNumResults())) { + ops_connected_to_fetch->insert(control_input.getDefiningOp()); + if (IsNoOpControlBarrier(control_input)) + control_barrier_worklist.push(control_input); + } + + // Erase all control outputs at the end from fetch. + fetch.getFetchesMutable().erase( + graph_op.getNumResults(), + fetch.getNumOperands() - graph_op.getNumResults()); + + // Iterate the worklist to remove all NoOp control barriers at the end of the + // function body that are used to merge two or more control dependencies. + while (!control_barrier_worklist.empty()) { + Value control_barrier = control_barrier_worklist.front(); + control_barrier_worklist.pop(); + + // We can only erase control barriers whose uses have been erased as well. + if (!control_barrier.use_empty()) continue; + + // Only values defined by IslandOp were inserted in the worklist. + IslandOp current_island = cast(control_barrier.getDefiningOp()); + + for (auto control_input : current_island.getControlInputs()) { + ops_connected_to_fetch->insert(control_input.getDefiningOp()); + if (IsNoOpControlBarrier(control_input)) + control_barrier_worklist.push(control_input); + } + current_island.erase(); + ops_connected_to_fetch->erase(current_island); + } + return true; +} + +// Appends function arguments with `num_resources` number of arguments of +// requested type. +void AppendFunctionArguments(func::FuncOp func, int num_resources, + ShapedType chaining_data_type) { + for (int i = 0; i < num_resources; ++i) { + func.getRegion().addArgument(chaining_data_type, func.getLoc()); + } + + FunctionType ftype = + FunctionType::get(func.getContext(), func.getBody().getArgumentTypes(), + func.getFunctionType().getResults()); + func.setType(ftype); +} + +// Appends function results with `num_resources` number of results of requested +// type. +void AppendFunctionResults(func::FuncOp func, int num_resources, + ShapedType chaining_data_type) { + Block& block = func.front(); + auto graph_op = cast(block.front()); + // Note that func result types are same as the result types of + // GraphOp in the function `func`. + assert(std::equal(func->getResultTypes().begin(), + func->getResultTypes().end(), + graph_op->getResultTypes().begin())); + auto new_result_types = + llvm::to_vector<4>(func.getFunctionType().getResults()); + for (int i = 0; i < num_resources; ++i) { + new_result_types.push_back(chaining_data_type); + } + FunctionType ftype = FunctionType::get( + func.getContext(), func.getArgumentTypes(), new_result_types); + func.setType(ftype); + + // Rewrite GraphOp to have same number of results as the + // function. + OpBuilder builder(graph_op); + auto new_graph_op = + GraphOp::create(builder, graph_op.getLoc(), new_result_types); + new_graph_op.getRegion().takeBody(graph_op.getRegion()); + graph_op->replaceAllUsesWith( + new_graph_op->getResults().drop_back(num_resources)); + graph_op.erase(); + func::ReturnOp return_op = cast(block.getTerminator()); + int num_old_arguments = return_op.getNumOperands(); + return_op->insertOperands( + num_old_arguments, + new_graph_op.getResults().slice(num_old_arguments, num_resources)); +} + +// Creates a wrapper island enclosing the `sub_op` dependent on +// `control_inputs`. +IslandOp CreateIsland(Operation* sub_op, ValueRange control_inputs, + OpBuilder builder) { + assert(sub_op); + auto control_type = ControlType::get(builder.getContext()); + auto island = + IslandOp::create(builder, sub_op->getLoc(), sub_op->getResultTypes(), + control_type, control_inputs); + island.getBody().push_back(new Block); + Block* block = &island.getBody().back(); + builder.setInsertionPointToEnd(block); + sub_op->replaceAllUsesWith(island.getOutputs()); + sub_op->moveBefore(block, block->begin()); + YieldOp::create(builder, sub_op->getLoc(), sub_op->getResults()); + return island; +} + +// Adds control dependencies from/to chain arguments/results. It adds two +// identity ops, chain_src and chain_sink, per resource equivalence class. +// Using the resource to operations map, it adds (1) a control dependency +// from chain_src to all the operations that read/write to a resource of the +// equivalence class, and (2) a control dependency from all the operations that +// read/write to a resource of the class to the chain_sink operation. +void ChainResourceOps( + func::FuncOp func, ResourceToOpsMapTy& chain_resource_to_ops_map, + llvm::EquivalenceClasses& resource_equivalence_classes, + SmallPtrSet ops_connected_to_fetch, int num_old_outputs) { + assert(num_old_outputs + resource_equivalence_classes.getNumClasses() == + func.getNumArguments()); + auto graph_op = cast(func.front().front()); + + auto fetch = graph_op.GetFetch(); + OpBuilder builder_chain_src(fetch); + builder_chain_src.setInsertionPointToStart(fetch->getBlock()); + + OpBuilder builder_chain_sink(fetch); + int chain_index = num_old_outputs; + + // Iterate over all equivalence classes. + for (auto class_iter = resource_equivalence_classes.begin(); + class_iter != resource_equivalence_classes.end(); ++class_iter) { + // Only visit one element per class, the leader. + if (!(*class_iter)->isLeader()) continue; + + // Create chain source and sink identity islands for current equivalence + // class. + auto chain_arg = func.getArgument(chain_index++); + auto src_identity = TF::IdentityOp::create( + builder_chain_src, chain_arg.getLoc(), chain_arg.getType(), chain_arg); + auto chain_src_island = CreateIsland(src_identity, {}, builder_chain_src); + + auto sink_identity = TF::IdentityOp::create( + builder_chain_sink, chain_arg.getLoc(), chain_arg.getType(), chain_arg); + auto chain_sink_island = + CreateIsland(sink_identity, {}, builder_chain_sink); + + // Add the chain sink data output to fetch. These might stay empty. + fetch.getFetchesMutable().append(chain_sink_island.getOutputs().front()); + + // Iterate over all members of the current equivalence class (represented + // by `class_iter`). Keep track of ops that have already been processed. + llvm::SmallDenseSet processed_ops; + for (auto member_iter = + resource_equivalence_classes.member_begin(**class_iter); + member_iter != resource_equivalence_classes.member_end(); + ++member_iter) { + ResourceAndDevice resource_and_device = *member_iter; + auto map_iter = chain_resource_to_ops_map.find(resource_and_device); + if (map_iter == chain_resource_to_ops_map.end()) continue; + OperationSetTy& resource_ops = map_iter->getSecond(); + + // Add dependencies between all ops that access current resource and chain + // source and sink. + for (Operation* op : resource_ops) { + if (processed_ops.contains(op)) continue; + + IslandOp wrapper = op->getParentOfType(); + assert(wrapper); + wrapper.getControlInputsMutable().append(chain_src_island.getControl()); + if (ops_connected_to_fetch.contains(wrapper)) { + chain_sink_island.getControlInputsMutable().append( + wrapper.getControl()); + } + processed_ops.insert(op); + } + } + } + VLOG(2) << "Added " << resource_equivalence_classes.getNumClasses() + << " chains for " << chain_resource_to_ops_map.size() << " resources"; +} + +// Generate a dummy constant island of requested type. +IslandOp GetDummyConstant(OpBuilder builder, ShapedType const_type, + Location loc) { + DenseIntElementsAttr val = DenseIntElementsAttr::get(const_type, 1); + auto const_op = TF::ConstOp::create(builder, loc, val); + auto const_island = CreateIsland(const_op, {}, builder); + return const_island; +} + +// Rewrites the while op with extra chaining operands and results. Uses a +// dummy constant of requested type as argument to all the new chaining +// operands. +TF::WhileOp RewriteWhileOp(TF::WhileOp while_op, int num_resource_inputs, + ShapedType const_type) { + IslandOp while_wrapper = while_op->getParentOfType(); + assert(while_wrapper && "While op is expected to be wrapped in a IslandOp"); + + // Get the dummy constant. + OpBuilder builder(while_wrapper); + auto loc = NameLoc::get( + builder.getStringAttr("chain_control_outputs@" + while_op.getBody())); + IslandOp const_wrapper = GetDummyConstant(builder, const_type, loc); + + // Get new operand and result types. + auto new_operands = llvm::to_vector<4>(while_op->getOperands()); + auto new_result_types = llvm::to_vector<4>(while_op->getResultTypes()); + Value const_output = const_wrapper.getOutputs()[0]; + for (int i = 0; i < num_resource_inputs; ++i) { + new_operands.push_back(const_output); + new_result_types.push_back(const_output.getType()); + } + + // Replace old while op with new while op. + auto new_while_op = + TF::WhileOp::create(builder, while_op.getLoc(), new_result_types, + new_operands, while_op->getAttrs()); + auto new_while_wrapper = + CreateIsland(new_while_op, while_wrapper.getControlInputs(), builder); + for (auto result : while_wrapper.getOutputs()) { + result.replaceAllUsesWith( + new_while_wrapper.getOutputs()[result.getResultNumber()]); + } + while_wrapper.getControl().replaceAllUsesWith(new_while_wrapper.getControl()); + while_wrapper.erase(); + return new_while_op; +} + +// Converts the control outputs of the while body to data outputs, thus +// removing control barrier at the end of while loop body. +void ConvertControlToDataOutputs( + func::FuncOp while_body, SmallVectorImpl& while_callers, + OperationSetTy& recompute_analysis_for_funcs, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + const DataFlowSolver& solver, bool composite_tpuexecute_side_effects) { + if (while_callers.empty()) return; + + // Collect access information for each resource in the while body that needs + // to be chained, along with equivalence classes (resources in one class will + // use the same chain). + ResourceToOpsMapTy chain_resource_to_ops_map; + llvm::EquivalenceClasses resource_equivalence_classes; + DeviceMap devices; + CollectChainResources( + while_body, chain_resource_to_ops_map, resource_equivalence_classes, + devices, side_effect_analysis, solver, composite_tpuexecute_side_effects); + + // Check for presence of unknown side-effecting ops within the while loop + // body. These ops act as barriers and the optimization would not yield much + // inter iteration parallelism for this while loop body. So return with + // warning. + if (chain_resource_to_ops_map.count(kUnknownResourceAndDevice) > 0) { + std::set blocking_ops; + for (Operation* op : chain_resource_to_ops_map[kUnknownResourceAndDevice]) { + std::string op_name = op->getName().getStringRef().str(); + if (blocking_ops.insert(op_name).second) { + LOG(WARNING) + << "[`tf-executor-convert-control-to-data-outputs` disabled] " + "Op type '" + << op_name + << "' has unknown side effects and blocks inter iteration " + "parallelism for the while loop. Consider modeling side " + "effects of this op."; + } + } + return; + } + + // First remove all control outputs of while loop body. + SmallPtrSet ops_connected_to_fetch; + bool changed = RemoveAllControlOutputs(while_body, &ops_connected_to_fetch); + + // If there was no control output to be removed, return early. + if (!changed) return; + + int num_chains = resource_equivalence_classes.getNumClasses(); + RankedTensorType chaining_data_type = + RankedTensorType::get({}, OpBuilder(while_body).getI32Type()); + // Create new while body + int num_old_outputs = while_body.getNumResults(); + AppendFunctionArguments(while_body, num_chains, chaining_data_type); + AppendFunctionResults(while_body, num_chains, chaining_data_type); + + // Insert identity ops with control dep + ChainResourceOps(while_body, chain_resource_to_ops_map, + resource_equivalence_classes, ops_connected_to_fetch, + num_old_outputs); + // Modify all the while ops referencing the body function and the + // corresponding while condition functions. Note that each while condition + // needs to be modified only once. + OperationSetTy visited; + for (TF::WhileOp while_op : while_callers) { + // If the while callers are modified as part of the optimization, then the + // side effect analysis of their parent functions are invalidated. They + // need to be recomputed. + recompute_analysis_for_funcs.insert( + while_op->getParentOfType()); + func::FuncOp while_cond = while_op.cond_function(); + // Rewrite while op with extra chaining arguments and results. + while_op = RewriteWhileOp(while_op, num_chains, chaining_data_type); + bool first_visit = visited.insert(while_cond).second; + if (!first_visit) continue; + // Modify while condition function with extra chaining arguments. + AppendFunctionArguments(while_cond, num_chains, chaining_data_type); + } +} + +void ConvertControlToDataOutputsPass::runOnOperation() { + ModuleOp module = getOperation(); + + DataFlowSolver solver; + solver.load(); + solver.load(); + TF::LoadIsCompositeDataflowAnalysis(solver); + if (failed(solver.initializeAndRun(module))) return signalPassFailure(); + + // This pass assumes that all functions are suitable for export i.e., each + // function has a single tf_executor.graph op and all islands wrap the + // internal op perfectly. Verify that in the beginning once. + if (failed(tensorflow::VerifyExportSuitable(module))) { + signalPassFailure(); + return; + } + TF::SideEffectAnalysis side_effect_analysis(module); + + SymbolTableCollection table; + SymbolUserMap symbol_map(table, module); + llvm::SmallDenseMap> + while_body_func_to_while_ops; + + // Get all the while body functions and the corresponding while ops first + // because the symbol user map is invalidated once we start deleting while + // ops. + for (auto func : module.getOps()) { + if (func.isExternal()) continue; + SmallVector while_callers = GetWhileCallers(func, symbol_map); + if (while_callers.empty()) continue; + while_body_func_to_while_ops[func] = while_callers; + // TODO(b/295892728): verify while body sanity. This pass expects a 1:1 + // correspondence between function results and tf_executor.graph results. + } + // Keep track of functions whose side effect analysis is invalidated because + // of modifications to that function. + OperationSetTy recompute_analysis_for_funcs; + + for (auto& entry : while_body_func_to_while_ops) { + func::FuncOp while_body = entry.getFirst(); + SmallVector& while_callers = entry.getSecond(); + if (recompute_analysis_for_funcs.contains(while_body)) { + // TODO(b/202540801): Recomputing side effect analysis for the entire + // module is wasteful. It would be better to just recompute analysis for + // specific functions but the current side effect analysis interface + // does not allow that. + side_effect_analysis = TF::SideEffectAnalysis(module); + } + ConvertControlToDataOutputs( + while_body, while_callers, recompute_analysis_for_funcs, + side_effect_analysis.GetAnalysisForFunc(while_body), solver, + composite_tpuexecute_side_effects_); + } +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorConvertControlToDataOutputsPass() { + return std::make_unique(); +} + +std::unique_ptr> +CreateTFExecutorConvertControlToDataOutputsPass( + bool composite_tpuexecute_side_effects) { + return std::make_unique( + composite_tpuexecute_side_effects); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_launch_func_to_tf_call.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_launch_func_to_tf_call.cc new file mode 100644 index 0000000000000000000000000000000000000000..411786bf21c81409aac63247c2d9df2575f60508 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_launch_func_to_tf_call.cc @@ -0,0 +1,75 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +#define GEN_PASS_DEF_CONVERTLAUNCHFUNCTOTFCALLPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Rewrites tf_device::LaunchFuncOp into TF::PartitionedCallOp. +struct ConvertLaunchFuncToTFCallPass + : public impl::ConvertLaunchFuncToTFCallPassBase< + ConvertLaunchFuncToTFCallPass> { + void runOnOperation() override; +}; + +void ConvertLaunchFuncToTFCallPass::runOnOperation() { + ModuleOp module = getOperation(); + module.walk([&](tf_device::LaunchFuncOp launch) { + OpBuilder builder(launch); + auto call_op = TF::PartitionedCallOp::create( + builder, module.getLoc(), launch.getResultTypes(), launch.getOperands(), + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, + SymbolRefAttr::get(builder.getContext(), launch.getFunc()), + /*config=*/builder.getStringAttr(""), + /*config_proto=*/builder.getStringAttr(""), + /*executor_type=*/builder.getStringAttr("")); + call_op->setAttr("device", launch->getAttrOfType("device")); + launch.replaceAllUsesWith(call_op); + launch.erase(); + }); +} + +} // namespace + +std::unique_ptr> CreateConvertLaunchFuncToTFCallPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_session_initializer_to_function.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_session_initializer_to_function.cc new file mode 100644 index 0000000000000000000000000000000000000000..5774ddfd15631a15660a61f5ef2921a412888ff7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_session_initializer_to_function.cc @@ -0,0 +1,100 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +#define GEN_PASS_DEF_CONVERTSESSIONINITIALIZERTOFUNCTIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +struct ConvertSessionInitializerToFunctionPass + : public impl::ConvertSessionInitializerToFunctionPassBase< + ConvertSessionInitializerToFunctionPass> { + void runOnOperation() override; +}; + +void ConvertSessionInitializerToFunctionPass::runOnOperation() { + ModuleOp module = getOperation(); + auto session_initializer = tf_saved_model::GetSessionInitializerOp(module); + if (!session_initializer) return; + + OpBuilder builder(session_initializer); + const char *name = "session_initializer"; + + // In the (unlikely) case of there already being a session initializer + // function, bail out. + if (SymbolTable::lookupSymbolIn(module, name)) { + module->emitWarning("session_initializer function already exists"); + session_initializer.erase(); + return; + } + + auto init = func::FuncOp::create( + builder, module.getLoc(), name, + FunctionType::get(module.getContext(), /*inputs=*/{}, /*results=*/{})); + + // Make savedmodel verification happy. + init->setAttr("tf_saved_model.exported_names", + builder.getStrArrayAttr({name})); + + builder.setInsertionPointToStart(init.addEntryBlock()); + + for (func::FuncOp func : tf_saved_model::GetInitializerFunctions(module)) { + if (func.getNumArguments() != 0) { + session_initializer->emitWarning( + "encountered session initializers with arguments"); + continue; + } + + // Since we're now calling this function, savedmodel verification + // needs it to be private. + func.setVisibility(SymbolTable::Visibility::Private); + func->removeAttr("tf_saved_model.exported_names"); + + ArrayRef args; + func::CallOp::create(builder, session_initializer.getLoc(), + func.getFunctionType().getResults(), func.getSymName(), + args); + } + func::ReturnOp::create(builder, session_initializer.getLoc()); + + session_initializer.erase(); +} + +} // namespace + +std::unique_ptr> +CreateConvertSessionInitializerToFunctionPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_tf_control_flow_to_scf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_tf_control_flow_to_scf.cc new file mode 100644 index 0000000000000000000000000000000000000000..765131fa512dcd0e893b16d42508f2a590d31987 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_tf_control_flow_to_scf.cc @@ -0,0 +1,213 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +/// Move the ops of `source_block` into `destination_block`, keeping the later's +/// block arguments' type as `block_arguments_type`. +static void moveBlock(Block* source_block, Block* destination_block, + TypeRange block_arguments_type, + PatternRewriter& rewriter) { + // If `destination_block` isn't empty, erase its terminator to ensure that it + // never contains two terminator-like ops after merging. + if (!destination_block->empty()) + rewriter.eraseOp(destination_block->getTerminator()); + + destination_block->addArguments( + block_arguments_type, + SmallVector(block_arguments_type.size(), + source_block->getParent()->getLoc())); + rewriter.mergeBlocks(source_block, destination_block, + destination_block->getArguments()); +} + +/// Convert the `tf.IfRegion` op to the `scf.if` op. +class ConvertIfRegionOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(IfRegionOp op, + PatternRewriter& rewriter) const override { + // Creates the `then` or `else` region of the `scf.if` op. Note that + // `tf_then_or_else_region` is the `then` or `else` region of the + // `tf.IfRegion` op and `scf_then_or_else_region` is the `then` or `else` + // region of the new `scf.if` op. Further, `tf_if_region_return_type` is the + // list of return types of the `tf.IfRegion` op. + auto createScfThenOrElse = [](Region& tf_then_or_else_region, + Region& scf_then_or_else_region, + TypeRange tf_if_region_return_type, + PatternRewriter& rewriter) { + // Move the first block of `tf_then_or_else_region` into the first block + // of `scf_then_or_else_region` and do not add any arguments to the block. + moveBlock(&tf_then_or_else_region.front(), + &scf_then_or_else_region.front(), TypeRange(), rewriter); + + // Replace the current terminator (a `tf.Yield` op) with an `scf.yield` + // op. The input of the `scf.yield` op is a list of results of `tf.Cast` + // ops, each of which casts an operand of the current terminator to the + // corresponding result type of the `tf.IfRegion` op. + Operation* current_terminator = + scf_then_or_else_region.front().getTerminator(); + rewriter.setInsertionPoint(current_terminator); + SmallVector scf_yield_input; + for (auto it : llvm::zip(tf_if_region_return_type, + current_terminator->getOperands())) { + scf_yield_input.push_back( + CastOp::create(rewriter, current_terminator->getLoc(), + std::get<0>(it), std::get<1>(it))); + } + + rewriter.replaceOpWithNewOp(current_terminator, + scf_yield_input); + }; + + Location loc = op.getLoc(); + + // The condition of an `scf.if` op is a 1-bit signless integer. Whereas, the + // condition of the `tf.IfRegion` op is a 0-D tensor of 1-bit signless + // integers. Thus, we use the `tensor.extract` op to compute the condition + // of `scf.if` from that of `tf.IfRegion`. + auto scf_if_condition = + tensor::ExtractOp::create(rewriter, loc, op.getCond()); + + TypeRange tf_if_region_return_type = op.getResultTypes(); + + // Create the `scf.if` op. + auto scf_if_op = + scf::IfOp::create(rewriter, loc, tf_if_region_return_type, + scf_if_condition, /*withElseRegion=*/true); + + Region& then_region = op.getThenBranch(); + Region& else_region = op.getElseBranch(); + + // Create the `then` and `else` regions of the `scf.if` op. + createScfThenOrElse(then_region, scf_if_op.getThenRegion(), + tf_if_region_return_type, rewriter); + createScfThenOrElse(else_region, scf_if_op.getElseRegion(), + tf_if_region_return_type, rewriter); + + // Replace the `tf.IfRegion` op with the results of the `scf.if` op. + rewriter.replaceOp(op, scf_if_op.getResults()); + return success(); + } +}; + +/// Convert the `tf.WhileRegion` op to the `scf.while` op. +class ConvertWhileRegionOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(WhileRegionOp op, + PatternRewriter& rewriter) const override { + // Creates the `before` or `after` region of the `scf.while` op. Note that + // `tf_cond_or_body_region` is the `cond` or `body` region of the + // `tf.WhileRegion` op. `scf_before_or_after_region` is the `before` or + // `after` region of the new `scf.while` op. `scf_block_arguments_type` is + // the type of arguments that need to be in the first block of + // `scf_before_or_after_region`. + auto createScfCondOrBody = + [](Region& tf_cond_or_body_region, Region& scf_before_or_after_region, + TypeRange scf_block_arguments_type, PatternRewriter& rewriter) { + // Move the first block of `tf_cond_or_body_region` into the first + // block of `scf_before_or_after_region` and keep the later's + // arguments' type as `scf_block_arguments_type`. + moveBlock(&tf_cond_or_body_region.front(), + &scf_before_or_after_region.front(), + scf_block_arguments_type, rewriter); + + Operation* cond_or_body_terminator = + scf_before_or_after_region.front().getTerminator(); + rewriter.setInsertionPoint(cond_or_body_terminator); + return cond_or_body_terminator; + }; + + ValueRange opInput = op.getInput(); + TypeRange scf_block_arguments_type = opInput.getType(); + + // Create the `scf.while` op. + auto scf_while_op = scf::WhileOp::create(rewriter, op.getLoc(), + op.getResultTypes(), opInput); + + // Create the `before` block of the `scf.while` op (with an `scf.condition` + // op as the terminator). Note that the arguments' type of this block is + // kept as `opInput`'s type. Note that the input of an `scf.condition` op is + // a 1-bit signless integer. But, the condition of the `tf.WhileRegion` op + // is a 0-D tensor of 1-bit signless integers. Thus, we use the + // `tensor.extract` op to compute the input of `scf.condition`. + rewriter.createBlock(&scf_while_op.getBefore()); + Operation* cond_terminator = + createScfCondOrBody(op.getCond(), scf_while_op.getBefore(), + scf_block_arguments_type, rewriter); + auto scf_condition_input = tensor::ExtractOp::create( + rewriter, cond_terminator->getLoc(), cond_terminator->getOperand(0)); + rewriter.replaceOpWithNewOp( + cond_terminator, scf_condition_input.getResult(), + scf_while_op.getBefore().front().getArguments()); + + // Create the `after` block of the `scf.while` op (with an `scf.yield` op as + // the terminator). Note that the arguments' type of this block is kept as + // `opInput`'s type. + rewriter.createBlock(&scf_while_op.getAfter()); + Operation* body_terminator = + createScfCondOrBody(op.getBody(), scf_while_op.getAfter(), + scf_block_arguments_type, rewriter); + rewriter.replaceOpWithNewOp(body_terminator, + body_terminator->getOperands()); + + // Replace the `tf.WhileRegion` op with the `scf.while` op. + rewriter.replaceOp(op, scf_while_op.getResults()); + + return success(); + } +}; + +} // end anonymous namespace + +void populateTfControlFlowToScfPatterns(MLIRContext* context, + RewritePatternSet* patterns) { + patterns->add(context); +} + +#define GEN_PASS_DEF_CONVERTTFCONTROLFLOWTOSCFPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ConvertTfControlFlowToScf + : public impl::ConvertTfControlFlowToScfPassBase< + ConvertTfControlFlowToScf> { + void runOnOperation() override { + RewritePatternSet patterns(&getContext()); + populateTfControlFlowToScfPatterns(&getContext(), &patterns); + (void)applyPatternsGreedily(getOperation(), std::move(patterns)); + } +}; + +std::unique_ptr> createConvertTfControlFlowToScfPass() { + return std::make_unique(); +} + +} // namespace TF +} // end namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_to_legacy_compile_and_replicate_attributes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_to_legacy_compile_and_replicate_attributes.cc new file mode 100644 index 0000000000000000000000000000000000000000..5ef4b261e5eab7f676b16db9f6abe1acb9a5b7dd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/convert_to_legacy_compile_and_replicate_attributes.cc @@ -0,0 +1,74 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass converts unified compilation and replication +// attributes into legacy attributes. For example, _replication_info=X +// and _xla_compile_device_type=TPU should be replaced with _tpu_replicate=X. +// This ensures the unified attributes not get exposed outside of the MLIR +// bridge with V1 pipeline in some cases. + +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Debug.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +#define GEN_PASS_DEF_CONVERTTOLEGACYCOMPILEANDREPLICATEATTRIBUTESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ConvertToLegacyCompileAndReplicateAttributesPass + : public impl::ConvertToLegacyCompileAndReplicateAttributesPassBase< + ConvertToLegacyCompileAndReplicateAttributesPass> { + void runOnOperation() override; +}; + +LogicalResult ConvertToLegacyAttributes(func::FuncOp func_op) { + auto result = func_op->walk([&](mlir::Operation* op) { + if (failed(TF::HasValidCompilationAndReplicationAttributes(*op))) + return WalkResult::interrupt(); + if (op->hasAttr(TF::kReplicationInfoAttr)) { + op->setAttr(TF::kTpuReplicateAttr, op->getAttr(TF::kReplicationInfoAttr)); + op->removeAttr(TF::kReplicationInfoAttr); + op->removeAttr(TF::kCompileDeviceTypeAttr); + } + return mlir::WalkResult::advance(); + }); + return failure(result.wasInterrupted()); +} + +void ConvertToLegacyCompileAndReplicateAttributesPass::runOnOperation() { + func::FuncOp func_op = getOperation(); + if (failed(ConvertToLegacyAttributes(func_op))) return signalPassFailure(); +} + +} // namespace + +std::unique_ptr> +CreateConvertToLegacyCompileAndReplicateAttributesPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decode_attributes_hook.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decode_attributes_hook.cc new file mode 100644 index 0000000000000000000000000000000000000000..a0c660895472aa208841d321158382e2b923085e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decode_attributes_hook.cc @@ -0,0 +1,33 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "llvm/ADT/ArrayRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/constant_fold.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/logging.h" + +namespace mlir { +namespace { + +} // anonymous namespace +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_optionals.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_optionals.cc new file mode 100644 index 0000000000000000000000000000000000000000..9bc6709d148c658f7a3f81f5565e74d50d12f019 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_optionals.cc @@ -0,0 +1,301 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Rewrite/FrozenRewritePatternSet.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_DECOMPOSEOPTIONALSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct DecomposeOptionalsPass + : public impl::DecomposeOptionalsPassBase { + void runOnOperation() override; +}; + +class HandleOptionalFrom : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::OptionalFromValueOp op, + PatternRewriter& rewriter) const override { + Value value = nullptr; + for (auto v : op.getComponents()) { + value = v; + } + if (!value) return failure(); + rewriter.replaceOpWithNewOp(op, value.getType(), value); + return success(); + } +}; + +class HandleOptionalGet : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::OptionalGetValueOp op, + PatternRewriter& rewriter) const override { + auto input = op.getOptional(); + auto elementType = getElementTypeOrSelf(input.getType()); + if (isa(elementType)) { + // We can only replace OptionalGetValue after the inputs have been + // replaced. + return failure(); + } + rewriter.replaceOpWithNewOp(op, op.getResult(0).getType(), + input); + return success(); + } +}; + +class HandleOptionalNone : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::OptionalNoneOp op, + PatternRewriter& rewriter) const override { + Type elementType = getElementTypeOrSelf(op->getResult(0)); + Type newType = nullptr; + if (auto variant = dyn_cast(elementType)) { + ArrayRef sub = variant.getSubtypes(); + if (sub.size() == 1) { + auto inner = sub[0]; + if (!isa(inner)) { + newType = inner; + } + } + } + if (!newType) { + return failure(); + } + rewriter.replaceOpWithNewOp(op, newType, + op->getOperands()); + return success(); + } +}; + +class HandleCall : public OpInterfaceRewritePattern { + // Optional-agnostic pattern that propagates types across the program. + using OpInterfaceRewritePattern::OpInterfaceRewritePattern; + + LogicalResult matchAndRewrite(CallOpInterface call, + PatternRewriter& rewriter) const override { + // Adjust the function arguments to match the types of the caller. + // So e.g. + // + // func.func @f(%x : foo) { + // } + // ... + // func.call @f(%x : bar) + // + // will be transformed to + // + // func.func @f(%x : bar) { + // ... + // } + // ... + // func.call @f(%x : bar) + + CallInterfaceCallable callable = call.getCallableForCallee(); + mlir::SymbolRefAttr sym = callable.dyn_cast(); + auto symbol = + mlir::SymbolTable::lookupNearestSymbolFrom(call.getOperation(), sym); + if (!symbol) return failure(); + auto f = llvm::dyn_cast(symbol); + + if (call.getOperation()->getNumOperands() != + f.getBody().front().getNumArguments()) { + return failure(); // RemoteCall et al + } + + rewriter.startOpModification(f); + bool changed = false; + for (auto [call_arg, body_arg] : + llvm::zip(call.getOperation()->getOperands(), + f.getBody().front().getArguments())) { + if (call_arg.getType() != body_arg.getType()) { + body_arg.setType(call_arg.getType()); + changed = true; + } + } + if (changed) { + rewriter.finalizeOpModification(f); + return success(); + } else { + rewriter.cancelOpModification(f); + return failure(); + } + } +}; + +class HandleIf : public OpRewritePattern { + // Optional-agnostic pattern that propagates types across the program. + using OpRewritePattern::OpRewritePattern; + + LogicalResult adjustBranch(TF::IfOp ifop, func::FuncOp branch, + PatternRewriter& rewriter) const { + bool changed = false; + rewriter.startOpModification(branch); + for (auto [call_arg, body_arg] : + llvm::zip(llvm::drop_begin(ifop.getOperation()->getOperands()), + branch.getBody().front().getArguments())) { + if (call_arg.getType() != body_arg.getType()) { + body_arg.setType(call_arg.getType()); + changed = true; + } + } + if (changed) { + rewriter.finalizeOpModification(branch); + return success(); + } else { + rewriter.cancelOpModification(branch); + return failure(); + } + } + + LogicalResult matchAndRewrite(TF::IfOp ifop, + PatternRewriter& rewriter) const override { + bool success = + succeeded(adjustBranch(ifop, ifop.then_function(), rewriter)) || + succeeded(adjustBranch(ifop, ifop.else_function(), rewriter)); + return LogicalResult::success(success); + } +}; + +class HandleFunc : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(func::FuncOp f, + PatternRewriter& rewriter) const override { + // Adjust the function type to match the block args. So this e.g. transforms + // + // func.func @f(%x : foo) -> foo { + // ^main(%x : bar): + // yield %x : bar + // } + // + // to + // + // func.func @f(%x : bar) -> bar { + // ... + // } + auto ret = f.getBody().front().getTerminator(); + std::vector argument_types; + for (auto arg : f.getBody().front().getArguments()) { + argument_types.push_back(arg.getType()); + } + std::vector return_types; + for (auto ret_op : ret->getOperands()) { + return_types.push_back(ret_op.getType()); + } + auto newType = + FunctionType::get(rewriter.getContext(), argument_types, return_types); + if (f.getFunctionType() == newType) { + return failure(); + } + rewriter.modifyOpInPlace(f, [&] { f.setType(newType); }); + + // Adjust the type of the return values callers of the function to + // match the "func.return" within the function. + // + // So this would transform + // func.func @f(...) -> ... { + // yield %x : foo + // } + // ... + // func.call @f(...) -> bar + // + // to + // + // func.func @f(...) -> ... { + // yield %x : foo + // } + // ... + // func.call f(...) -> foo + auto symbol_uses = f.getSymbolUses(f->getParentOp()); + if (!symbol_uses.has_value()) { + return failure(); + } + for (auto use : *symbol_uses) { + Operation* caller = use.getUser(); + bool changed = false; + rewriter.startOpModification(caller); + for (auto [result, type] : + llvm::zip(caller->getResults(), return_types)) { + if (result.getType() != type) { + result.setType(type); + changed = true; + } + } + if (changed) { + rewriter.finalizeOpModification(caller); + } else { + rewriter.cancelOpModification(caller); + } + } + return success(); + } +}; + +void DecomposeOptionalsPass::runOnOperation() { + mlir::ModuleOp module = getOperation(); + + RewritePatternSet pattern_list(&getContext()); + pattern_list.add(&getContext()); + pattern_list.add(&getContext()); + pattern_list.add(&getContext()); + pattern_list.add(&getContext()); + pattern_list.add(&getContext()); + pattern_list.add(&getContext()); + FrozenRewritePatternSet patterns(std::move(pattern_list)); + + if (failed(applyPatternsGreedily(module, patterns))) { + signalPassFailure(); + } +} +} // namespace + +std::unique_ptr> CreateDecomposeOptionalsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_reduce_dataset.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_reduce_dataset.cc new file mode 100644 index 0000000000000000000000000000000000000000..900089b7d87531b896b23ad802562fb23ea5618e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_reduce_dataset.cc @@ -0,0 +1,338 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +using mlir::func::FuncOp; + +namespace mlir { +namespace TF { + +namespace { + +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_DECOMPOSEREDUCEDATASETPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct DecomposeReduceDatasetPass + : public impl::DecomposeReduceDatasetPassBase { + void getDependentDialects(DialectRegistry& registry) const override { + registry.insert(); + } + + void runOnOperation() override; +}; + +// Create the AnonymousIterator for `reduce_dataset` with `dataset_types` using +// `builder`. +AnonymousIteratorV3Op CreateIterator(OpBuilder builder, + llvm::ArrayRef dataset_types, + ReduceDatasetOp reduce_dataset) { + llvm::SmallVector shape_attrs; + llvm::SmallVector type_attrs; + for (Type type : dataset_types) { + shape_attrs.push_back( + TF::ShapeAttr::get(builder.getContext(), mlir::cast(type))); + type_attrs.push_back(TypeAttr::get(getElementTypeOrSelf(type))); + } + + auto anonymous_iterator = AnonymousIteratorV3Op::create( + builder, reduce_dataset.getLoc(), + RankedTensorType::get({}, builder.getType()), + /*output_types=*/builder.getArrayAttr(type_attrs), + /*shape_types=*/builder.getArrayAttr(shape_attrs)); + MakeIteratorOp::create(builder, reduce_dataset.getLoc(), + reduce_dataset.getInputDataset(), + anonymous_iterator.getResult()); + return anonymous_iterator; +} + +// Create a WhileRegionOp turning `reduce_dataset` into a dataset iteration with +// reduce_fn call. +WhileRegionOp CreateDatasetWhile(OpBuilder builder, + ReduceDatasetOp reduce_dataset) { + auto const_true = TF::ConstOp::create( + builder, reduce_dataset.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get(/*shape=*/{}, builder.getI1Type()), true)); + + SmallVector while_input_values; + SmallVector while_input_types; + while_input_values.push_back(const_true.getResult()); + while_input_types.push_back(const_true.getResult().getType()); + for (int i = 1; i < reduce_dataset.getNumOperands(); ++i) { + while_input_values.push_back(reduce_dataset.getOperand(i)); + while_input_types.push_back(reduce_dataset.getOperand(i).getType()); + } + + auto dataset_while = + TF::WhileRegionOp::create(builder, reduce_dataset.getLoc(), + while_input_types, /*input=*/while_input_values, + /*parallel_iterations=*/10, false, + /*shape_invariant=*/false); + + // `_lower_using_switch_merge` is the default for While ops created + // in TensorFlow and allows lowering to V1 control flow for loop + // parallelization. + dataset_while->setAttr("_lower_using_switch_merge", + builder.getBoolAttr(true)); + + return dataset_while; +} + +// Populate the cond of `dataset_while`. The cond body just returns the +// condition of whether to continue to next iteration. +void PopulateDatasetWhileCond(OpBuilder builder, WhileRegionOp dataset_while, + Location loc) { + auto& cond_region = dataset_while.getCond(); + Block* cond_block = builder.createBlock(&cond_region); + auto while_input_types = dataset_while.getOperandTypes(); + cond_block->addArguments( + while_input_types, SmallVector(while_input_types.size(), loc)); + YieldOp::create(builder, loc, cond_block->getArgument(0)); +} + +// Create an IfRegionOp with a predicate from `optional_has_value`. If true, it +// uses `get_next` to get the next value and calls `reduce_func`. `body_args` +// is used as pass through of state values for else branch. `dataset_types` is +// used for constructing the CallOp for `reduce_func`. +IfRegionOp CreateOptionalDatasetIf( + OpBuilder builder, ReduceDatasetOp reduce_dataset, FuncOp reduce_func, + IteratorGetNextAsOptionalOp get_next, OptionalHasValueOp optional_has_value, + ArrayRef body_args, ArrayRef dataset_types) { + const Location loc = reduce_dataset.getLoc(); + // If returns are the state variables. + SmallVector if_return_types; + const int state_size = + reduce_dataset->getAttrOfType("Tstate").size(); + for (int i = 1; i < state_size + 1; i++) { + if_return_types.push_back(reduce_dataset.getOperand(i).getType()); + } + + auto dataset_if = TF::IfRegionOp::create( + builder, loc, if_return_types, optional_has_value.getResult(), false, + /*_then_func_name=*/nullptr, + /*_else_func_name=*/nullptr); + // `_lower_using_switch_merge` allows lowering to V1 control flow for loop + // parallelization. + dataset_if->setAttr("_lower_using_switch_merge", builder.getBoolAttr(true)); + // Empty else branch, if there is no more data, do nothing. + auto& else_branch = dataset_if.getElseBranch(); + else_branch.push_back(new Block); + builder.setInsertionPointToEnd(&else_branch.front()); + // Return only the state variables from the body arguments. + SmallVector else_returns; + for (int i = 1; i < state_size + 1; i++) { + else_returns.push_back(body_args[i]); + } + TF::YieldOp::create(builder, loc, + /*operands=*/else_returns); + + // Then branch gets the data and calls the reduce_function. + auto& then_branch = dataset_if.getThenBranch(); + then_branch.push_back(new Block); + builder.setInsertionPointToEnd(&then_branch.front()); + // Add iterator operational data access inside if. + auto get_value = TF::OptionalGetValueOp::create(builder, loc, dataset_types, + get_next.getResult()); + SmallVector reduce_fn_args; + + // Function arguments are state values, dataset values, and then passthrough + // arguments. + // First argument to body is the while loop condition and state values start + // at index=1. + for (int i = 1; i < state_size + 1; ++i) { + reduce_fn_args.push_back(body_args[i]); + } + for (Value value : get_value.getResults()) { + reduce_fn_args.push_back(value); + } + for (int i = state_size + 1; i < body_args.size(); ++i) { + reduce_fn_args.push_back(body_args[i]); + } + + auto reduce_call = + mlir::func::CallOp::create(builder, loc, reduce_func, reduce_fn_args); + + // Both the device attribute and compile_device_type attribute should be + // propagated to the reduce function call. + reduce_call->setAttr(kDeviceAttr, + reduce_dataset->getAttrOfType(kDeviceAttr)); + reduce_call->setAttr( + TF::kCompileDeviceTypeAttr, + reduce_dataset->getAttrOfType(TF::kCompileDeviceTypeAttr)); + + SmallVector if_returns; + + TF::YieldOp::create(builder, loc, + /*operands=*/reduce_call.getResults()); + return dataset_if; +} + +// Populates WhileRegionOp body which is replacing `reduce_dataset`. Iterates +// `anonymous_iterator` with `dataset_types` and optional calls `reduce_func`. +void PopulateDatasetWhileBody(OpBuilder builder, ReduceDatasetOp reduce_dataset, + FuncOp reduce_func, WhileRegionOp dataset_while, + AnonymousIteratorV3Op anonymous_iterator, + ArrayRef dataset_types) { + const Location loc = reduce_dataset.getLoc(); + auto while_input_types = dataset_while.getOperandTypes(); + auto& body_region = dataset_while.getBody(); + Block* body_block = builder.createBlock(&body_region); + auto body_arguments = body_block->addArguments( + while_input_types, SmallVector(while_input_types.size(), loc)); + auto get_next = IteratorGetNextAsOptionalOp::create( + builder, loc, RankedTensorType::get({}, builder.getType()), + anonymous_iterator.getResult(), anonymous_iterator.getOutputTypes(), + anonymous_iterator.getOutputShapes()); + auto optional_has_value = OptionalHasValueOp::create( + builder, loc, RankedTensorType::get({}, builder.getI1Type()), + get_next.getResult()); + + SmallVector body_args; + for (Value value : body_arguments) { + body_args.push_back(value); + } + + IfRegionOp dataset_if = + CreateOptionalDatasetIf(builder, reduce_dataset, reduce_func, get_next, + optional_has_value, body_args, dataset_types); + + builder.setInsertionPointToEnd(body_block); + // The body returns consist of the loop condition (whether the next iterator + // has a value), the state returned by the IfRegionOp, and the pass through + // values. + SmallVector body_returns; + body_returns.push_back(optional_has_value.getResult()); + + const int state_size = + reduce_dataset->getAttrOfType("Tstate").size(); + for (int i = 0; i < state_size; ++i) { + body_returns.push_back(dataset_if.getResult(i)); + } + // Copy the arguments but skip the states and the loop condition + // which are updated in while body. + for (int i = state_size + 1; i < body_args.size(); ++i) { + body_returns.push_back(body_args[i]); + } + TF::YieldOp::create(builder, loc, + /*operands=*/body_returns); +} + +// Decomposes any ReduceDatasetOps in `function` into a dataset iteration and a +// call to the reduce function in the ReduceDatasetOp. +LogicalResult DecomposeReduceDatasetInFunction(FuncOp function) { + if (!llvm::hasSingleElement(function)) + return function.emitOpError("Expecting a single block function"); + + auto decompose_result = function.walk([&](ReduceDatasetOp reduce_dataset) { + if (!reduce_dataset->hasAttrOfType(TF::kCompileDeviceTypeAttr)) + return WalkResult::advance(); + OpBuilder builder(reduce_dataset); + Location loc = reduce_dataset.getLoc(); + + // Get reduce function signature for dataset iteration types. + // Note: lookupSymbol is a linear lookup which means the overall + // complexity = # ReduceDataset ops x # of functions in module. + func::FuncOp reduce_func = + function->getParentOfType().lookupSymbol( + reduce_dataset.getF()); + + // The reduce function arguments consist of three part in this order: + // 1. Reduction state inputs. + // 2. Dataset inputs. + // 3. Captures inputs. + // The number of dataset inputs can be indirectly determined to be + // total_number_of_inputs - state_inputs - captured_inputs. + auto func_inputs = reduce_func.getFunctionType().getInputs(); + const int func_input_size = func_inputs.size(); + const int argument_size = + reduce_dataset->getAttrOfType("Targuments").size(); + const int state_size = + reduce_dataset->getAttrOfType("Tstate").size(); + const int dataset_input_size = func_input_size - state_size - argument_size; + + SmallVector dataset_types; + for (int i = 0; i < dataset_input_size; ++i) { + dataset_types.push_back(func_inputs[state_size + i]); + } + + // Create dataset iterator and iterate dataset in while loop which calls + // reduce_fn. + AnonymousIteratorV3Op anonymous_iterator = + CreateIterator(builder, dataset_types, reduce_dataset); + WhileRegionOp dataset_while = CreateDatasetWhile(builder, reduce_dataset); + PopulateDatasetWhileCond(builder, dataset_while, loc); + PopulateDatasetWhileBody(builder, reduce_dataset, reduce_func, + dataset_while, anonymous_iterator, dataset_types); + + // Updates usage and erases rewritten reduce_dataset op based on the number + // of state variables. + for (int i = 0; i < state_size; ++i) { + reduce_dataset.getResult(i).replaceAllUsesWith( + dataset_while.getResult(i + 1)); + } + reduce_dataset.erase(); + + return WalkResult::advance(); + }); + + return failure(decompose_result.wasInterrupted()); +} + +void DecomposeReduceDatasetPass::runOnOperation() { + if (failed(DecomposeReduceDatasetInFunction(getOperation()))) { + return signalPassFailure(); + } +} + +} // anonymous namespace + +std::unique_ptr> +CreateDecomposeReduceDatasetPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..5bf3c36fd8dcd2b90b4bdaf65ec1deb251d15ed5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.cc @@ -0,0 +1,206 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.h" + +#include +#include + +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h" +#include "tensorflow/core/framework/rng_alg.h" + +namespace mlir { +namespace TF { + +namespace { + +// Returns subtype of `resource` if present. Otherwise an unranked tensor type +// of `element_type` is returned. +static Type GetResourceSubtypeOrDefault(Value resource, Type element_type) { + auto resource_type = llvm::cast( + llvm::cast(resource.getType()).getElementType()); + if (resource_type.getSubtypes().size() == 1) + return resource_type.getSubtypes().front(); + + return UnrankedTensorType::get(element_type); +} + +static bool HasResourceSubtype(Value resource) { + return llvm::cast( + llvm::cast(resource.getType()).getElementType()) + .getSubtypes() + .size() == 1; +} + +static Type GetResourceSubtype(Value resource) { + return llvm::cast( + llvm::cast(resource.getType()).getElementType()) + .getSubtypes() + .front(); +} + +// Decompose tf.RngReadAndSkip. +// +// For Philox, the resource variable holds a tensor<3xi64> with the state: +// [counter_lo, counter_hi, key] +// +// RngReadAndSkip increments the 128 bit counter value by 256 * delta and +// returns the original state value. +// +// For Threefry, the resource variable holds a tensor<2xi64> with the state: +// [counter, key] +// +// RngReadAndSkip increments the 64 bit counter value by 256 * delta and +// returns a tensor<3xi64> value [counter, key, 0]. +class DecomposeRngReadAndSkipOp : public RewritePattern { + public: + explicit DecomposeRngReadAndSkipOp(MLIRContext *context) + : RewritePattern(RngReadAndSkipOp::getOperationName(), 1, context, + { + AddV2Op::getOperationName(), + AssignVariableOp::getOperationName(), + CastOp::getOperationName(), + ConstOp::getOperationName(), + LessOp::getOperationName(), + MulOp::getOperationName(), + PadOp::getOperationName(), + PackOp::getOperationName(), + ReadVariableOp::getOperationName(), + SelectV2Op::getOperationName(), + UnpackOp::getOperationName(), + }) {} + + LogicalResult matchAndRewrite(Operation *op, + PatternRewriter &rewriter) const override { + auto rng_op = cast(op); + + DenseIntElementsAttr alg_constant; + if (!matchPattern(rng_op.getAlg(), m_Constant(&alg_constant))) { + return rewriter.notifyMatchFailure( + op, "unable to determine algorithm statically"); + } + + if (alg_constant.getNumElements() != 1) { + return rewriter.notifyMatchFailure(op, "expected alg to be a scalar"); + } + + uint64_t alg_value = ((*alg_constant.value_begin()).getZExtValue()); + tensorflow::ConcreteRngAlgorithm alg; + if (tensorflow::RNG_ALG_PHILOX == alg_value) { + alg = tensorflow::ConcreteRngAlgorithm::RNG_ALG_PHILOX; + } else if (tensorflow::RNG_ALG_THREEFRY == alg_value) { + alg = tensorflow::ConcreteRngAlgorithm::RNG_ALG_THREEFRY; + } else if (tensorflow::RNG_ALG_AUTO_SELECT == alg_value) { + // For AUTO_SELECT, we'll manage the counter as if it's for Philox. + alg = tensorflow::ConcreteRngAlgorithm::RNG_ALG_PHILOX; + } else { + return rewriter.notifyMatchFailure(op, "unsupported alg"); + } + + Type state_element_type = rewriter.getI64Type(); + RankedTensorType op_type = RankedTensorType::get( + {tensorflow::RNG_MAX_COUNTER_SIZE + tensorflow::RNG_KEY_SIZE}, + state_element_type); + if (op_type != rng_op.getType()) { + return rewriter.notifyMatchFailure(op, "unexpected op type"); + } + + if (!HasResourceSubtype(rng_op.getResource())) { + return rewriter.notifyMatchFailure(op, "missing resource subtype"); + } + + int counter_size = tensorflow::GetCounterSize(alg); + int state_size = counter_size + tensorflow::RNG_KEY_SIZE; + RankedTensorType res_type = + RankedTensorType::get({state_size}, state_element_type); + if (res_type != GetResourceSubtype(rng_op.getResource())) { + return rewriter.notifyMatchFailure(op, "unexpected resource subtype"); + } + + Location loc = op->getLoc(); + + // Read the state value from the resource. + Value state = + ReadVariableOp::create(rewriter, loc, res_type, rng_op.getResource()); + + // Extract the key and counter from the state. + RankedTensorType word_type = RankedTensorType::get({}, state_element_type); + auto unpacked = UnpackOp::create( + rewriter, loc, SmallVector(state_size, word_type), state, 0); + Value key = unpacked.getResult(counter_size); + + SmallVector counter; + for (int i = 0; i < counter_size; ++i) { + counter.push_back(unpacked.getResult(i)); + } + + // Set the increment to 256 * delta. + Type u64 = rewriter.getIntegerType(64, /*isSigned=*/false); + RankedTensorType u64_scalar = RankedTensorType::get({}, u64); + Value step_size = ConstOp::create(rewriter, loc, GetScalarOfType(u64, 256)); + Value increment = + MulOp::create(rewriter, loc, u64_scalar, step_size, rng_op.getDelta()); + + // Increment the counter. + SmallVector pack_args; + RankedTensorType word_u64_type = RankedTensorType::get({}, u64); + Value zero_u64 = ConstOp::create(rewriter, loc, GetScalarOfType(u64, 0)); + Value one_u64 = ConstOp::create(rewriter, loc, GetScalarOfType(u64, 1)); + for (int i = 0; i < counter_size; ++i) { + Value word = counter[i]; + Value word_u64 = CastOp::create(rewriter, loc, word_u64_type, word); + Value new_word_u64 = AddV2Op::create(rewriter, loc, word_u64, increment); + Value new_word = CastOp::create(rewriter, loc, word_type, new_word_u64); + pack_args.push_back(new_word); + + Value overflow = LessOp::create(rewriter, loc, new_word_u64, word_u64); + increment = + SelectV2Op::create(rewriter, loc, overflow, one_u64, zero_u64); + } + + // Save the new state value to the resource. + pack_args.push_back(key); + Value new_state = PackOp::create(rewriter, loc, res_type, pack_args); + AssignVariableOp::create(rewriter, loc, rng_op.getResource(), new_state); + + // Pad the original state as necessary to fill the output shape. + int pad = tensorflow::RNG_MAX_COUNTER_SIZE - counter_size; + Type i64 = rewriter.getI64Type(); + RankedTensorType paddings_ty = RankedTensorType::get({1, 2}, i64); + std::vector paddings_values = {0, pad}; + Value paddings = ConstOp::create( + rewriter, loc, DenseIntElementsAttr::get(paddings_ty, paddings_values)); + Value output = PadOp::create(rewriter, loc, op_type, state, paddings); + + rewriter.replaceOp(op, output); + return success(); + } +}; + +#include "tensorflow/compiler/mlir/tensorflow/transforms/generated_decompose_resource_ops.inc" +} // namespace + +void PopulateDecomposeResourceOpsPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + patterns->add(context); + populateWithGenerated(*patterns); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..d7794500fa98b62b457f6736e4aef018101a804c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.h @@ -0,0 +1,37 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_DECOMPOSE_RESOURCE_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_DECOMPOSE_RESOURCE_OPS_H_ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Populates rewrite patterns that decompose composite resource operations into +// primitive ones like ReadVariableOp, AssignVariableOp and other computations +// to facilitate transformations like resource op lifting. +// NOTE: These patterns do not support `use_locking=true` for a lot of resource +// operations. So decomposition may not be correct outside of backends like XLA, +// which automatically locks all resource variables. +void PopulateDecomposeResourceOpsPatterns(MLIRContext *context, + RewritePatternSet *patterns); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_DECOMPOSE_RESOURCE_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..09560fbd462446b0f229f6c859ca5de7b2c49327 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.td @@ -0,0 +1,638 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/IR/OpBase.td" +include "mlir/Dialect/Func/IR/FuncOps.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" +include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td" + +def NonScalarRankedType : Type>, HasRankPred]>, + "Non scalar type">; + +def HasSingleElement: Constraint>; + +def EmptyList: NativeCodeCall<"llvm::SmallVector{}">; + +// Creates a tf.ReadVariable op that reads a resource `$2` that has the same +// element type as `$1`. The op created will use location of `$0`. +def CreateTFReadVariableOp : NativeCodeCall< + "TF::ReadVariableOp::create($_builder, " + " $0.getLoc()," + " GetResourceSubtypeOrDefault(" + " $2, llvm::cast($1.getType()).getElementType())," + " $2)" + >; + +def CheckHasResourceSubtype : Constraint>; + +def CreateConstBoolAttrFalse : NativeCodeCall<"$_builder.getBoolAttr(false)">; + +def CreateTensorScatterAddOp : NativeCodeCall< + "TF::TensorScatterAddOp::create($_builder, " + "$0.getLoc(), $0.getType(), $0, $1, $2, $_builder.getStringAttr(\"\"))">; + +def CreateTensorScatterUpdateOp : NativeCodeCall< + "TF::TensorScatterUpdateOp::create($_builder, " + "$0.getLoc(), $0.getType(), $0, $1, $2, $_builder.getStringAttr(\"\"))">; + +def CreateTFReadVariableOpFromResourceHandle : NativeCodeCall< + "TF::ReadVariableOp::create($_builder, " + "$0.getLoc(), GetResourceSubtype($1), $1)">; + +def CreateTFSelectOp: NativeCodeCall< + "TF::SelectOp::create($_builder, " + "$0.getLoc(), $3.getType(), $1, $2, $3)">; + +def ConstAttrIfThenElse: NativeCodeCall< + "($0).getValue() ? ($1) : ($2)">; + +// Convert clamp(lo, x, hi) to clipbyvalue(x, lo, hi). +def Clamp: NativeCodeCall< + "TF::ClipByValueOp::create($_builder, " + " $0.getLoc()," + " $2.getType(), $2, $1, $3)">; + +def CopyXlaOutsideCompilationAttr: NativeCodeCallVoid< + "CopyXlaOutsideCompilationAttributesAdaptor($0, $1)">; + +def DecomposeAssignAddVariableOp : + Pat< + (TF_AssignAddVariableOp:$src_op $resource, $value), + (TF_AssignVariableOp + $resource, + (TF_AddV2Op + (CreateTFReadVariableOp $src_op, $value, $resource), + $value + ), + (CreateConstBoolAttrFalse) + ) + >; + +def DecomposeAssignSubVariableOp : + Pat< + (TF_AssignSubVariableOp:$src_op $resource, $value), + (TF_AssignVariableOp + $resource, + (TF_SubOp + (CreateTFReadVariableOp $src_op, $value, $resource), + $value + ), + (CreateConstBoolAttrFalse) + ) + >; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +def DecomposeResourceApplyGradientDescentOp : + Pat< + (TF_ResourceApplyGradientDescentOp:$src_op $resource, $alpha, $delta, BoolAttr:$_), + (TF_AssignVariableOp + $resource, + (TF_SubOp + (CreateTFReadVariableOp $src_op, $alpha, $resource), + (TF_MulOp $alpha, $delta) + ), + (CreateConstBoolAttrFalse) + ) + >; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// accum = accum * momentum + grad; +// var -= accum * lr +def DecomposeResourceApplyMomentumOpNonNesterov : + Pattern< + (TF_ResourceApplyMomentumOp:$src_op + $var_resource, $accum_resource, $lr, $grad, $momentum, + BoolAttr:$_, ConstBoolAttrFalse:$use_nesterov + ), + [(TF_AddV2Op:$accum_new + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + $momentum + ), + $grad + ), + (TF_AssignVariableOp $accum_resource, $accum_new, (CreateConstBoolAttrFalse)), + (TF_AssignSubVariableOp $var_resource, (TF_MulOp $accum_new, $lr)) + ] + >; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// accum = accum * momentum + grad; +// var -= grad * lr + accum * momentum * lr +def DecomposeResourceApplyMomentumOpNesterov : + Pattern< + (TF_ResourceApplyMomentumOp:$src_op + $var_resource, $accum_resource, $lr, $grad, $momentum, + BoolAttr:$_, ConstBoolAttrTrue:$use_nesterov + ), + [(TF_AddV2Op:$accum_new + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + $momentum + ), + $grad + ), + (TF_AssignVariableOp $accum_resource, $accum_new, (CreateConstBoolAttrFalse)), + (TF_AssignSubVariableOp + $var_resource, + (TF_AddV2Op + (TF_MulOp $grad, $lr), + (TF_MulOp $accum_new, (TF_MulOp $momentum, $lr)) + ) + ) + ] + >; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// accum = accum * momentum - lr * grad +// var += accum +def DecomposeResourceApplyKerasMomentumOpNonNesterov : + Pattern< + (TF_ResourceApplyKerasMomentumOp:$src_op + $var_resource, $accum_resource, $lr, $grad, $momentum, + BoolAttr:$_, ConstBoolAttrFalse:$use_nesterov + ), + [(TF_SubOp:$accum_new + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + $momentum + ), + (TF_MulOp $grad, $lr) + ), + (TF_AssignVariableOp $accum_resource, $accum_new, (CreateConstBoolAttrFalse)), + (TF_AssignAddVariableOp $var_resource, $accum_new) + ] + >; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// accum = accum * momentum - lr * grad +// var += accum * momentum - lr * grad +def DecomposeResourceApplyKerasMomentumOpNesterov : + Pattern< + (TF_ResourceApplyKerasMomentumOp:$src_op + $var_resource, $accum_resource, $lr, $grad, $momentum, + BoolAttr:$_, ConstBoolAttrTrue:$use_nesterov + ), + [(TF_SubOp:$accum_new + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + $momentum + ), + (TF_MulOp:$grad_lr $grad, $lr) + ), + (TF_AssignVariableOp $accum_resource, $accum_new, (CreateConstBoolAttrFalse)), + (TF_AssignAddVariableOp + $var_resource, + (TF_SubOp + (TF_MulOp $accum_new, $momentum), + $grad_lr + ) + ) + ] + >; + +// Pattern to Decompose ResourceApplyAdagradV2. +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// accum <- accum + grad * grad +// variable <- variable - lr * grad / (sqrt(accum) + epsilon) +def DecomposeResourceApplyAdagradV2 : + Pattern< + (TF_ResourceApplyAdagradV2Op:$src_op + $var_resource, $accum_resource, $lr, $epsilon, $grad, BoolAttr:$_, + ConstBoolAttrTrue:$update_slots), + [ + (TF_AddV2Op:$new_accum + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + (TF_MulOp $grad, $grad) + ), + (TF_AssignSubVariableOp + $var_resource, + (TF_DivOp + (TF_MulOp $lr, $grad), + (TF_AddV2Op (TF_SqrtOp $new_accum), $epsilon) + ) + ), + (TF_AssignVariableOp $accum_resource, $new_accum, (CreateConstBoolAttrFalse)), + ] + >; + +// ResourceApplyAdagrad op can be canonicalized to ResourceApplyAdagradV2 with +// zero epsilon and then decomposed using DecomposeResourceApplyAdagradV2 +// pattern. +def DecomposeResourceApplyAdagrad : + Pattern< + (TF_ResourceApplyAdagradOp $var_resource, $accum_resource, $lr, $grad, + $use_locking, $update_slots), + [ + (TF_ConstOp:$zero_epsilon (GetScalarOfType<0> $grad)), + (TF_ResourceApplyAdagradV2Op $var_resource, $accum_resource, $lr, + $zero_epsilon, $grad, $use_locking, $update_slots + ) + ]>; + + +// Pattern to Decompose ResourceApplyAdam without Nesterov momentum. +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// alpha <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t) +// m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t +// v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t +// variable <- variable - alpha * m_t / (sqrt(v_t) + epsilon) +def DecomposeResourceApplyAdamNonNesterov : + Pattern< + (TF_ResourceApplyAdamOp:$src_op + $var_resource, $m_resource, $v_resource, $beta1_power, $beta2_power, $lr, + $beta1, $beta2, $epsilon, $grad, BoolAttr:$_, + ConstBoolAttrFalse:$use_nesterov), + [ + (TF_ConstOp:$one (GetScalarOfType<1> $grad)), + (TF_MulOp:$alpha + $lr, + (TF_DivOp + (TF_SqrtOp (TF_SubOp $one, $beta2_power)), + (TF_SubOp $one, $beta1_power) + ) + ), + (TF_AddV2Op:$new_m + (TF_MulOp $beta1, (CreateTFReadVariableOp $src_op, $grad, $m_resource)), + (TF_MulOp (TF_SubOp $one, $beta1), $grad) + ), + (TF_AddV2Op:$new_v + (TF_MulOp $beta2, (CreateTFReadVariableOp $src_op, $grad, $v_resource)), + (TF_MulOp + (TF_SubOp $one, $beta2), + (TF_SquareOp $grad) + ) + ), + (TF_AssignSubVariableOp + $var_resource, + (TF_DivOp + (TF_MulOp $alpha, $new_m), + (TF_AddV2Op (TF_SqrtOp $new_v), $epsilon) + ) + ), + (TF_AssignVariableOp $m_resource, $new_m, (CreateConstBoolAttrFalse)), + (TF_AssignVariableOp $v_resource, $new_v, (CreateConstBoolAttrFalse)) + ] + >; + +// Pattern to Decompose ResourceApplyAdam with Nesterov momentum. +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// alpha <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t) +// m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t +// v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t +// variable <- variable - (alpha * (m_t * beta1 + (1 - beta1) * g_t) / +// (sqrt(v_t) + epsilon)) +def DecomposeResourceApplyAdamNesterov : + Pattern< + (TF_ResourceApplyAdamOp:$src_op + $var_resource, $m_resource, $v_resource, $beta1_power, $beta2_power, $lr, + $beta1, $beta2, $epsilon, $grad, BoolAttr:$_, + ConstBoolAttrTrue:$use_nesterov), + [ + (TF_ConstOp:$one (GetScalarOfType<1> $grad)), + (TF_MulOp:$alpha + $lr, + (TF_DivOp + (TF_SqrtOp (TF_SubOp $one, $beta2_power)), + (TF_SubOp $one, $beta1_power) + ) + ), + (TF_AddV2Op:$new_m + (TF_MulOp $beta1, (CreateTFReadVariableOp $src_op, $grad, $m_resource)), + (TF_MulOp (TF_SubOp $one, $beta1), $grad) + ), + (TF_AddV2Op:$new_v + (TF_MulOp $beta2, (CreateTFReadVariableOp $src_op, $grad, $v_resource)), + (TF_MulOp + (TF_SubOp $one, $beta2), + (TF_SquareOp $grad) + ) + ), + (TF_AssignSubVariableOp + $var_resource, + (TF_DivOp + (TF_MulOp + $alpha, + (TF_AddV2Op + (TF_MulOp $new_m, $beta1), + (TF_MulOp (TF_SubOp $one, $beta1), $grad) + ) + ), + (TF_AddV2Op (TF_SqrtOp $new_v), $epsilon) + ) + ), + (TF_AssignVariableOp $m_resource, $new_m, (CreateConstBoolAttrFalse)), + (TF_AssignVariableOp $v_resource, $new_v, (CreateConstBoolAttrFalse)) + ] + >; + +// Pattern to decompose tf.ResourceGather into tf.ReadVariable and tf.GatherV2. +def DecomposeResourceGather : Pat< + (TF_ResourceGatherOp:$old_result + $resource, $indices, $batch_dims, $validate_indices), + (TF_GatherV2Op:$dest + (CreateTFReadVariableOp $old_result, $old_result, $resource), + $indices, + (TF_ConstOp $batch_dims), // axis + $batch_dims + ), [], [(CopyXlaOutsideCompilationAttr $old_result, $dest)]>; + +// Pattern to decompose tf.ResourceScatterAdd into tf.ReadVariable, +// tf.TensorScatterAdd, and tf.AssignVariable. +def DecomposeResourceScatterAdd : Pat< + (TF_ResourceScatterAddOp:$src_op $resource, $indices, $updates), + (TF_AssignVariableOp + $resource, + (CreateTensorScatterAddOp + (CreateTFReadVariableOp $src_op, $updates, $resource), + (TF_ExpandDimsOp $indices, + (TF_ConstOp (GetScalarOfType<-1> $indices))), + $updates + ), + (CreateConstBoolAttrFalse) + )>; + +// Pattern to decompose tf.ResourceScatterUpdate into tf.ReadVariable, +// tf.TensorScatterUpdate, and tf.AssignVariable. +def DecomposeResourceScatterUpdate : Pat< + (TF_ResourceScatterUpdateOp:$src_op $resource, $indices, $updates), + (TF_AssignVariableOp + $resource, + (CreateTensorScatterUpdateOp + (CreateTFReadVariableOp $src_op, $updates, $resource), + (TF_ExpandDimsOp $indices, + (TF_ConstOp (GetScalarOfType<-1> $indices))), + $updates + ), + (CreateConstBoolAttrFalse) + )>; + +// Pattern to decompose tf.VariableShape into tf.ReadVariable and tf.Shape. +def DecomposeVariableShape : Pat< + (TF_VariableShapeOp:$src_op $resource), + (TF_ShapeOp (CreateTFReadVariableOpFromResourceHandle $src_op, $resource)), + [(CheckHasResourceSubtype $resource)]>; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// ms <- rho * ms_{t-1} + (1-rho) * grad * grad +// mg = grad * (one - rho) + mg * rho; +// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon) +// +def DecomposeResourceApplyCenteredRMSProp : + Pattern< + (TF_ResourceApplyCenteredRMSPropOp:$src_op + $var_resource, $mg_resource, $ms_resource, $mom_resource, $lr, $rho, $momentum, $epsilon, + $grad, $use_locking + ), + [(TF_ConstOp:$one (GetScalarOfType<1> $grad)), + (CreateTFReadVariableOp $src_op, $grad, $ms_resource), + (TF_AddV2Op:$ms_new + (TF_MulOp + (TF_MulOp $grad, $grad), + (TF_SubOp $one, $rho) + ), + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $ms_resource), + $rho + ) + ), + (TF_AssignVariableOp $ms_resource, $ms_new, (CreateConstBoolAttrFalse)), + // mg = grad * (one - rho) + mg * rho; + (TF_AddV2Op:$mg_new + (TF_MulOp + $grad, + (TF_SubOp $one, $rho) + ), + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $mg_resource), + $rho + ) + ), + (TF_AssignVariableOp $mg_resource, $mg_new, (CreateConstBoolAttrFalse)), + // mom <- momentum * mom_{t-1} + lr * grad / sqrt((ms - mg * mg) + epsilon) + (TF_AddV2Op:$mom_new + (TF_MulOp $momentum, + (CreateTFReadVariableOp $src_op, $grad, $mom_resource)), + (TF_DivOp + (TF_MulOp $lr, $grad), + (TF_SqrtOp + (TF_AddV2Op + (TF_SubOp $ms_new, (TF_SquareOp $mg_new)), + $epsilon + ) + ) + ) + ), + (TF_AssignVariableOp $mom_resource, $mom_new, (CreateConstBoolAttrFalse)), + // var <- var - mom + (TF_AssignSubVariableOp $var_resource, $mom_new) + ] + >; + +// Same as DecomposeResourceApplyFtrlV2, with l2_shrinkage set to zero. +// TODO(kramm): Move this pattern to canonicalize.td? +def DecomposeResourceApplyFtrl : Pat< + (TF_ResourceApplyFtrlOp $var, $accum, $linear, $grad, $lr, $l1, $l2, + $lr_power, $use_locking, $multiply_linear_by_lr), + (TF_ResourceApplyFtrlV2Op $var, $accum, $linear, $grad, $lr, $l1, $l2, + (TF_ConstOp (GetScalarOfType<0> $lr_power)), $lr_power, $use_locking, $multiply_linear_by_lr)>; + +// TODO(kramm): Remove use_locking. The old bridge doesn't use this, either. +def DecomposeResourceApplyFtrlV2: Pattern< + (TF_ResourceApplyFtrlV2Op:$src_op $var, $accum, $linear, $grad, $lr, $l1, $l2, + $l2_shrinkage, $lr_power, $use_locking, $multiply_linear_by_lr), + [// grad_with_shrinkage = grad + 2 * l2_shrinkage * var + (TF_ConstOp:$zero (GetScalarOfType<0> $lr)), + (TF_ConstOp:$one (GetScalarOfType<1> $lr)), + (TF_ConstOp:$two (GetScalarOfType<2> $lr)), + (TF_AddV2Op:$grad_with_shrinkage + $grad, + (TF_MulOp + $two, + (TF_MulOp $l2_shrinkage, (CreateTFReadVariableOp $src_op, $grad, $var)) + ) + ), + // accum_new = accum + grad * grad + (TF_AddV2Op:$accum_new + (CreateTFReadVariableOp $src_op, $grad, $accum), + (TF_SquareOp $grad) + ), + // if multiply_linear_by_lr: + // linear += grad_with_shrinkage * lr - (accum_new^(-lr_power) - accum^(-lr_power)) * var + // else: + // linear += grad_with_shrinkage - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var + (TF_AssignAddVariableOp $linear, + (ConstAttrIfThenElse + $multiply_linear_by_lr, + (TF_SubOp + (TF_MulOp + $grad_with_shrinkage, + $lr), + (TF_MulOp + (TF_SubOp:$sub + (TF_PowOp $accum_new, (TF_NegOp $lr_power)), + (TF_PowOp (CreateTFReadVariableOp $src_op, $grad, $accum), (TF_NegOp $lr_power)) + ), + (CreateTFReadVariableOp $src_op, $grad, $var) + ) + ), + (TF_SubOp + $grad_with_shrinkage, + (TF_MulOp + (TF_DivOp + $sub, + $lr + ), + (CreateTFReadVariableOp $src_op, $grad, $var) + ) + ) + ) + ), + // quadratic = (new_accum^(-lr_power) / (multiply_linear_by_lr ? 1 : lr)) + // + 2 * l2 * (multiply_linear_by_lr ? lr : 1) + (TF_AddV2Op:$quadratic + (TF_DivOp + (TF_PowOp + $accum_new, + (TF_NegOp $lr_power) + ), + (ConstAttrIfThenElse $multiply_linear_by_lr, $one, $lr) + ), + (TF_MulOp + (TF_MulOp $two, $l2), + (ConstAttrIfThenElse $multiply_linear_by_lr, $lr, $one)) + ), + // linear_clipped_minus_linear = + // (multiply_linear_by_lr ? clamp(-l1 * lr, linear, l1 * lr) + // clamp(-l1, linear, l1)) - linear + (TF_SubOp:$linear_clipped_minus_linear + (ConstAttrIfThenElse + $multiply_linear_by_lr, + (Clamp $src_op, + (TF_NegOp (TF_MulOp $l1, $lr)), + (CreateTFReadVariableOp $src_op, $l1, $linear), + (TF_MulOp $l1, $lr) + ), + (Clamp $src_op, + (TF_NegOp $l1), + (CreateTFReadVariableOp $src_op, $l1, $linear), + $l1 + ) + ), + (CreateTFReadVariableOp $src_op, $l1, $linear) + ), + // var = linear_clipped_minus_linear / quadratic + (TF_AssignVariableOp $var, + (TF_DivOp + $linear_clipped_minus_linear, + $quadratic + ), + (CreateConstBoolAttrFalse) + ), + // accum = accum_new + (TF_AssignVariableOp $accum, $accum_new, (CreateConstBoolAttrFalse)), + ]>; + +// This decomposition is only correct inside XLA as it ignores use_locking +// attribute. +// ms <- rho * ms_{t-1} + (1-rho) * grad * grad +// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) +// var <- var - mom +def DecomposeResourceApplyRMSProp : + Pattern< + (TF_ResourceApplyRMSPropOp:$src_op + $var_resource, $ms_resource, $mom_resource, $lr, $rho, $momentum, $epsilon, + $grad, $use_locking + ), + [(TF_ConstOp:$one (GetScalarOfType<1> $grad)), + (CreateTFReadVariableOp $src_op, $grad, $ms_resource), + // ms <- rho * ms_{t-1} + (1-rho) * grad * grad + (TF_AddV2Op:$ms_new + (TF_MulOp + (CreateTFReadVariableOp $src_op, $grad, $ms_resource), + $rho + ), + (TF_MulOp + (TF_SquareOp $grad), + (TF_SubOp $one, $rho) + ) + ), + (TF_AssignVariableOp $ms_resource, $ms_new, (CreateConstBoolAttrFalse)), + // mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) + (TF_AddV2Op:$mom_new + (TF_MulOp $momentum, + (CreateTFReadVariableOp $src_op, $grad, $mom_resource)), + (TF_DivOp + (TF_MulOp $lr, $grad), + (TF_SqrtOp + (TF_AddV2Op $ms_new, $epsilon) + ) + ) + ), + (TF_AssignVariableOp $mom_resource, $mom_new, (CreateConstBoolAttrFalse)), + // var <- var - mom + (TF_AssignSubVariableOp $var_resource, $mom_new) + ] + >; + +def DecomposeResourceApplyProximalAdagrad : + Pattern< + (TF_ResourceApplyProximalAdagradOp:$src_op + $var_resource, $accum_resource, $lr, $l1, $l2, $grad, + $use_locking + ), + [(TF_ConstOp:$one (GetScalarOfType<1> $grad)), + (TF_ConstOp:$zero (GetScalarOfType<0> $grad)), + (TF_AddV2Op:$accum_new + (CreateTFReadVariableOp $src_op, $grad, $accum_resource), + (TF_SquareOp $grad)), + (TF_MulOp:$adagrad_lr $lr, (TF_RsqrtOp $accum_new)), + (TF_SubOp:$prox_var + (CreateTFReadVariableOp $src_op, $grad, $var_resource), + (TF_MulOp $grad, $adagrad_lr)), + (TF_MulOp:$l1_gt_zero (TF_SignOp $prox_var), + (TF_MaximumOp + (TF_SubOp (TF_AbsOp $prox_var), (TF_MulOp $adagrad_lr, $l1)), $zero)), + (TF_SelectV2Op:$var_numerator (TF_GreaterOp $l1, $zero), + $l1_gt_zero, $prox_var), + (TF_DivOp:$var_new + $var_numerator, (TF_AddOp $one, (TF_MulOp $adagrad_lr, $l2))), + (TF_AssignVariableOp $var_resource, $var_new, (CreateConstBoolAttrFalse)), + (TF_AssignVariableOp $accum_resource, $accum_new, (CreateConstBoolAttrFalse)) + ] + >; + +def DecomposeCollectiveReduceV2 : + Pattern< + (TF_CollectiveReduceV2Op:$src_op + $input, $group_size, $group_key, $instance_key, $ordering_token, + $merge_op, $final_op, $communication_hint, $timeout_seconds, + $is_stateless, $max_subdivs_per_device + ), + [(TF_CollectiveReduceV2Op + $input, $group_size, $group_key, $instance_key, (EmptyList), + $merge_op, $final_op, $communication_hint, $timeout_seconds, + $is_stateless, $max_subdivs_per_device + )], + [(HasSingleElement $ordering_token)] + >; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..8582377f8bed6cf21a4ff4018922c4c545a85ff4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops_pass.cc @@ -0,0 +1,205 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/decompose_resource_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFDevice { +namespace { + +constexpr char kBadDecompositionMessage[] = + "Resource ops decomposition did not converge"; + +// Decomposing resource ops should not take more than a few iterations (2-3) to +// converge as only a few patterns create new resource ops that can be further +// decomposed. The rest of the iterations are enough to clean up any dead ops +// created by decomposition. +constexpr int kMaxIterations = 20; + +// Populates `reachable_functions` with all functions that can be reached from +// device cluster ops. +void PopulateClusterReachableFunctions( + ModuleOp module, SmallPtrSetImpl& reachable_functions) { + SymbolTableCollection table; + SymbolUserMap symbol_map(table, module); + + // Create map from caller to set of all callee(s). + llvm::DenseMap> caller_callee_map; + + // Use worklist to populate the set of reachable functions. + std::queue function_worklist; + + // Iterates over all functions within the module to (1) create caller-callee + // map, and (2) initialize function worklist with functions referenced from + // device cluster ops. + for (auto func : module.getOps()) { + for (auto user : symbol_map.getUsers(func)) { + // Populate caller-callee map. + if (func::FuncOp caller = user->getParentOfType()) + caller_callee_map[caller].insert(func); + // Initialize function worklist with functions refrerenced in device + // cluster. + if (auto cluster = user->getParentOfType()) { + if (reachable_functions.insert(func).second) + function_worklist.push(func); + } + } + } + + // Uses worklist algorithm to insert all functions reachable from device + // cluster ops. + while (!function_worklist.empty()) { + func::FuncOp caller = function_worklist.front(); + function_worklist.pop(); + for (auto callee : caller_callee_map[caller]) { + if (reachable_functions.insert(callee).second) + function_worklist.push(callee); + } + } +} + +// Applies patterns locally on ops within `cluster` until convergence or +// `max_iterations` are reached. Returns failure if resource ops decomposition +// does not converge after `max_iterations`. +// TODO(prakalps): This can be useful to a lot of other passes in bridge. +// Extract out as a separate utility. +LogicalResult ApplyPatternsLocallyUntilConverged( + Operation* op_with_regions, FrozenRewritePatternSet& patterns, + int max_iterations) { + bool changed = true; + int iteration = 0; + while (changed && (iteration++ < max_iterations)) { + changed = false; + auto walk_result = + op_with_regions->walk([&patterns, &changed](Operation* operation) { + GreedyRewriteConfig config; + config.setStrictness(mlir::GreedyRewriteStrictness::ExistingOps); + bool op_erased; + if (failed(applyOpPatternsGreedily(operation, patterns, config, + &op_erased))) + return WalkResult::interrupt(); + changed |= op_erased; + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return failure(); + } + // Return failure is `op_with_region` was modified changed in last iteration. + return success(!changed); +} + +// Applies patterns in only device clusters and functions reachable from such +// clusters. Returns failure if it fails to converge in `max_iterations`. +// TODO(prakalps): This can be useful to a lot of other passes in bridge. +// Extract out as a separate utility. +LogicalResult ApplyPatternsInClusterAndReachableFunctions( + ModuleOp module, FrozenRewritePatternSet& patterns, int max_iterations) { + SmallPtrSet reachable_functions; + PopulateClusterReachableFunctions(module, reachable_functions); + + // Apply patterns to reachable functions. + for (Operation* op : reachable_functions) { + assert(isa(op)); + if (failed(applyPatternsGreedily(op, patterns))) { + return op->emitError() << kBadDecompositionMessage; + } + } + + // Apply patterns to device cluster ops. + // Note: This module search for cluster ops is a bit wasteful as we could have + // collected many cluster ops when we were populating reachable functions. But + // we would still need to do a walk to find all clusters that do not + // reference any function. + for (func::FuncOp func : module.getOps()) { + // If we have already applied patterns to a function then we can skip + // applying patterns to any device clusters it contains. + if (reachable_functions.contains(func)) continue; + + auto walk_result = func.walk([&](tf_device::ClusterOp cluster) { + // Cluster ops are not isolated from above so we cannot use + // `applyPatternsGreedily` utility. Instead we apply patterns + // locally on each op within the cluster until convergence. + if (failed(ApplyPatternsLocallyUntilConverged(cluster, patterns, + max_iterations))) { + cluster.emitError() << kBadDecompositionMessage; + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return failure(); + } + + return success(); +} + +#define GEN_PASS_DEF_DECOMPOSERESOURCEOPSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct DecomposeResourceOpsPass + : public impl::DecomposeResourceOpsPassBase { + void runOnOperation() override { + // Add lowering patterns to the list. + RewritePatternSet patterns(&getContext()); + TF::PopulateDecomposeResourceOpsPatterns(&getContext(), &patterns); + + if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) { + getOperation().emitError() << kBadDecompositionMessage; + signalPassFailure(); + } + } +}; + +#define GEN_PASS_DEF_DECOMPOSERESOURCEOPSINCLUSTERPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct DecomposeResourceOpsInClusterPass + : public impl::DecomposeResourceOpsInClusterPassBase< + DecomposeResourceOpsInClusterPass> { + void runOnOperation() override { + // Add lowering patterns to the list. + RewritePatternSet patterns(&getContext()); + TF::PopulateDecomposeResourceOpsPatterns(&getContext(), &patterns); + FrozenRewritePatternSet frozen_patterns(std::move(patterns)); + + if (failed(ApplyPatternsInClusterAndReachableFunctions( + getOperation(), frozen_patterns, kMaxIterations))) + signalPassFailure(); + } +}; + +} // namespace + +std::unique_ptr> CreateDecomposeResourceOpsPass() { + return std::make_unique(); +} + +std::unique_ptr> +CreateDecomposeResourceOpsInClusterPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc new file mode 100644 index 0000000000000000000000000000000000000000..31d171d553e24690d1896d2147e2bda2e816b9bf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc @@ -0,0 +1,68 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +#define GEN_PASS_DEF_DEDUPBOUNDINPUTBINDINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +class DedupBoundInputBindingPass + : public impl::DedupBoundInputBindingPassBase { + void runOnOperation() final; +}; + +void DedupBoundInputBindingPass::runOnOperation() { + func::FuncOp func = getOperation(); + if (!mlir::tf_saved_model::IsExported(func)) return; + llvm::SmallDenseMap unique_bound_inputs; + llvm::BitVector arg_indices_to_erase(func.getNumArguments()); + for (unsigned i = 0, e = func.getNumArguments(); i < e; i++) { + auto attr = func.getArgAttrOfType( + i, "tf_saved_model.bound_input"); + if (!attr) continue; + auto inserted = unique_bound_inputs.insert(std::make_pair(attr, i)); + if (inserted.second) continue; + auto duplicate_arg = func.getArgument(i); + auto original_arg = func.getArgument(unique_bound_inputs[attr]); + duplicate_arg.replaceAllUsesWith(original_arg); + arg_indices_to_erase.set(i); + } + if (failed(func.eraseArguments(arg_indices_to_erase))) { + return signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> +CreateDedupBoundInputBindingPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_attribute_to_launch.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_attribute_to_launch.cc new file mode 100644 index 0000000000000000000000000000000000000000..274234124f29cd8e7c38500b7ea848b131536cc9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_attribute_to_launch.cc @@ -0,0 +1,80 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_DEVICEATTRIBUTETOLAUNCHPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct DeviceAttributeToLaunch + : public impl::DeviceAttributeToLaunchPassBase { + void runOnOperation() override; +}; + +void WrapOpInLaunch(Operation* op, llvm::StringRef device) { + OpBuilder builder(op); + + auto launch_op = tf_device::LaunchOp::create( + builder, op->getLoc(), builder.getStringAttr(device), + /*result_types=*/op->getResultTypes()); + op->replaceAllUsesWith(launch_op); + + launch_op.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&launch_op.GetBody()); + auto* return_op = + tf_device::ReturnOp::create(builder, op->getLoc(), op->getResults()) + .getOperation(); + MLIRContext* context = launch_op.getContext(); + op->removeAttr(StringAttr::get(context, kDeviceAttr)); + op->moveBefore(return_op); +} + +void DeviceAttributeToLaunch::runOnOperation() { + const Dialect* tf_dialect = getContext().getLoadedDialect("tf"); + + getOperation().walk([&](Operation* op) { + if (op->getDialect() != tf_dialect) return WalkResult::advance(); + if (auto device = op->getAttrOfType(kDeviceAttr)) { + if (!device.getValue().empty()) WrapOpInLaunch(op, device.getValue()); + } + return WalkResult::advance(); + }); +} + +} // anonymous namespace + +std::unique_ptr> +CreateDeviceAttributeToLaunchPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_index_selector.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_index_selector.cc new file mode 100644 index 0000000000000000000000000000000000000000..9450c7c065dfbc0a57ba8e54f17579ffb6539eea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/device_index_selector.cc @@ -0,0 +1,88 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Converts DeviceIndex to constant device. + +#include + +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_DEVICEINDEXSELECTORPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Folds the DeviceIndex op to a constant value. The DeviceIndex return the +// index of the device the op should run on. The user can use this to provide +// different op specializations. E.g., +// +// ```mlir +// %1 = "tf.DeviceIndex"() +// {device = "", device_names = ["CPU", "GPU"]} : () -> tensor +// %4 = "tf.Case"(%1, %arg0, %arg1) +// {branches = [@foo, @baz], output_shapes = [#tf_type.shape<>]} : +// (tensor, tensor, tensor) -> tensor +// ``` +// +// Shows an example where there are 2 different functions which could be +// executed to produce the same values but with different functions optimized +// for CPU or GPU. +struct DeviceIndexSelector + : public impl::DeviceIndexSelectorPassBase { + void runOnOperation() override; +}; + +} // namespace + +void DeviceIndexSelector::runOnOperation() { + func::FuncOp func = getOperation(); + // Convert all the DeviceIndex ops to constant values. + func.getBody().walk([](TF::DeviceIndexOp op) { + // This just selects the default in all cases where DeviceIndex feeds into + // tf.Case. This could be enhanced to have some sort of policy in the + // future. + OpBuilder b(op); + RankedTensorType type = RankedTensorType::get({}, b.getIntegerType(32)); + int index = op.getDeviceNames().size(); + for (auto use : op.getOperation()->getUsers()) { + // Skip if it doesn't feed into case. Alternatively this could always + // return the CPU device index if it exists. + if (!isa(use)) return; + } + DenseElementsAttr attr = + DenseElementsAttr::get(type, b.getI32IntegerAttr(index)); + auto constant = arith::ConstantOp::create(b, op.getLoc(), type, attr); + op.replaceAllUsesWith(constant.getOperation()); + op.erase(); + }); +} + +// Creates an instance of the TensorFlow DeviceIndex selector pass. +std::unique_ptr> CreateDeviceIndexSelectorPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/drop_while_shape_invariant.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/drop_while_shape_invariant.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6a8288e3b10b94bc803c8560c242e786990395e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/drop_while_shape_invariant.cc @@ -0,0 +1,74 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TF { + +namespace { + +constexpr char kShapeInvariantAttr[] = "shape_invariant"; + +#define GEN_PASS_DEF_DROPWHILESHAPEINVARIANTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class DropWhileShapeInvariantPass + : public impl::DropWhileShapeInvariantPassBase< + DropWhileShapeInvariantPass> { + void runOnOperation() override; +}; + +#define GEN_PASS_DEF_DROPWHILESHAPEINVARIANTINDEVICECLUSTERPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class DropWhileShapeInvariantInDeviceClusterPass + : public impl::DropWhileShapeInvariantInDeviceClusterPassBase< + DropWhileShapeInvariantInDeviceClusterPass> { + void runOnOperation() override; +}; + +void DropWhileShapeInvariantAttr(Operation* op) { + if (llvm::isa(op)) + op->removeAttr(kShapeInvariantAttr); +} +void DropWhileShapeInvariantPass::runOnOperation() { + getOperation().walk([](Operation* op) { DropWhileShapeInvariantAttr(op); }); +} + +void DropWhileShapeInvariantInDeviceClusterPass::runOnOperation() { + getOperation().walk([](tf_device::ClusterOp cluster) { + cluster.walk([](Operation* op) { DropWhileShapeInvariantAttr(op); }); + }); +} +} // namespace + +std::unique_ptr> +CreateDropWhileShapeInvariantPass() { + return std::make_unique(); +} + +std::unique_ptr> +CreateDropWhileShapeInvariantInDeviceClusterPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.cc new file mode 100644 index 0000000000000000000000000000000000000000..7ddd4c1e0ebac8ad649b0165772ea3d6b2dad41f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.cc @@ -0,0 +1,886 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/einsum.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Regex.h" +#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h" // from @llvm-project +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Traits.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h" +#include "tensorflow/core/util/matmul_bcast.h" + +namespace mlir { +namespace TF { + +namespace { + +// Creates ConstOp for int32_t value. +ConstOp createI32ConstOp(int32_t value, Location loc, + PatternRewriter* rewriter) { + auto int_attr = IntegerAttr::get(rewriter->getIntegerType(32), value); + return ConstOp::create(*rewriter, loc, int_attr); +} + +// Creates ConstantOp for array of int32_t. +arith::ConstantOp createI32ConstantOp(llvm::ArrayRef values, + Location loc, PatternRewriter* rewriter) { + auto values_type = RankedTensorType::get( + {static_cast(values.size())}, rewriter->getIntegerType(32)); + auto constant_attr = rewriter->getI32TensorAttr(values); + return arith::ConstantOp::create(*rewriter, loc, values_type, constant_attr); +} + +// Creates ConstantOp for array of int64_t. +arith::ConstantOp createI64ConstantOp(llvm::ArrayRef values, + Location loc, PatternRewriter* rewriter) { + auto values_type = RankedTensorType::get( + {static_cast(values.size())}, rewriter->getIntegerType(64)); + auto constant_attr = rewriter->getI64TensorAttr(values); + return arith::ConstantOp::create(*rewriter, loc, values_type, constant_attr); +} + +// Function to create a tf.SumOp to sum the element in 'value' reduced along the +// 'redux_axes'. +TF::SumOp createSumOp(Value value, Location loc, + llvm::ArrayRef redux_axes, + PatternRewriter* rewriter) { + Value redux_op = createI32ConstantOp(redux_axes, loc, rewriter); + + auto value_type = mlir::cast(value.getType()); + auto shape = value_type.getShape(); + llvm::SmallVector sum_shape; + for (int i = 0; i < shape.size(); ++i) { + if (std::find(redux_axes.begin(), redux_axes.end(), i) == + redux_axes.end()) { + sum_shape.push_back(shape[i]); + } + } + return TF::SumOp::create( + *rewriter, loc, + RankedTensorType::get(sum_shape, value_type.getElementType()), value, + redux_op); +} + +TF::TransposeOp createTransposeOp(Value value, Location loc, + llvm::ArrayRef permutation, + PatternRewriter* rewriter) { + auto perm_op = createI32ConstantOp(permutation, loc, rewriter); + auto value_type = mlir::cast(value.getType()); + auto shape = value_type.getShape(); + SmallVector transposed_shape(shape.begin(), shape.end()); + for (int i = 0, end = shape.size(); i < end; ++i) { + transposed_shape[i] = shape[permutation[i]]; + } + auto transposed_type = + RankedTensorType::get(transposed_shape, value_type.getElementType()); + return TF::TransposeOp::create(*rewriter, loc, transposed_type, value, + perm_op); +} + +TF::ReshapeOp createReshapeOp(Value value, ArrayRef shape, + Type element_type, Location loc, + PatternRewriter* rewriter) { + auto shape_tensor = createI64ConstantOp( + tensorflow::ConvertMlirShapeToTF(shape), loc, rewriter); + Type resultType = RankedTensorType::get(shape, element_type); + return TF::ReshapeOp::create(*rewriter, loc, resultType, /*tensor=*/value, + /*shape=*/shape_tensor); +} + +// Creates ReshapeOp with runtime calcuation of required shape to support +// dynamic shapes. The shape is calculated by Shape and UnsortedSegmentProd op. +// `reshape_segids` and `num_reshape_segids` for UnsortedSegmentProd is +// calculated in `reshapeForBatchMatmul`. +TF::ReshapeOp createReshapeOpForDynamic(Value value, ArrayRef shape, + ArrayRef reshape_segids, + int32_t num_reshape_segids, + Location loc, + PatternRewriter* rewriter) { + // Build ShapeOp + auto input_shape = + TF::ShapeOp::create(*rewriter, loc, value, rewriter->getBoolAttr(true)); + + // Build UnsortedSegmentProdOp + Type segProdresultType = + RankedTensorType::get(num_reshape_segids, rewriter->getIntegerType(32)); + auto segids_tensor = createI32ConstantOp(reshape_segids, loc, rewriter); + auto num_reshape_segids_tensor = + createI32ConstOp(num_reshape_segids, loc, rewriter); + auto segprod = TF::UnsortedSegmentProdOp::create( + *rewriter, loc, segProdresultType, input_shape->getResults()[0], + segids_tensor, num_reshape_segids_tensor); + + // Build ReshapeOp with the result of UnsortedSegmentProdOp. + Type out_tensor_type = + RankedTensorType::get(shape, getElementTypeOrSelf(value.getType())); + return TF::ReshapeOp::create(*rewriter, loc, out_tensor_type, + /*tensor=*/value, + /*shape=*/segprod->getResults()[0]); +} + +struct EinsumDimensionNumbers { + // Each field contains the list of dimensions appearing only in the specifed + // arguments of the einsum op with natural ordering. For example `rhs_out` + // contains the dimensions appearing in the RHS and the OUTPUT of the einsum + // but not in the LHS. + std::vector lhs; + std::vector rhs; + std::vector> lhs_rhs; + std::vector> lhs_out; + std::vector> rhs_out; + std::vector> lhs_rhs_out; +}; + +TF::ReshapeOp createOutputReshapeOpForDynamic( + Value value, ArrayRef shape, Value org_lhs, Value org_rhs, + EinsumDimensionNumbers& dnums, Location loc, PatternRewriter* rewriter) { + BoolAttr true_attr = rewriter->getBoolAttr(true); + // Build ShapeOp + auto shape_lhs = TF::ShapeOp::create(*rewriter, loc, org_lhs, true_attr); + auto shape_rhs = TF::ShapeOp::create(*rewriter, loc, org_rhs, true_attr); + + std::vector bl_index; // Indexes of B0,...,Bn and L0,...,Ln + bl_index.reserve(dnums.lhs_rhs_out.size() + dnums.lhs_out.size()); + for (auto i : dnums.lhs_rhs_out) { + bl_index.push_back(std::get<0>(i)); + } + for (auto i : dnums.lhs_out) { + bl_index.push_back(std::get<0>(i)); + } + std::vector r_index; // Indexes of R0,...,Rn + r_index.reserve(dnums.rhs_out.size()); + for (auto i : dnums.rhs_out) { + r_index.push_back(std::get<0>(i)); + } + + auto lhs_index_tensor = createI32ConstantOp(bl_index, loc, rewriter); + auto gather_lhs = TF::GatherOp::create( + *rewriter, loc, + RankedTensorType::get({static_cast(bl_index.size())}, + rewriter->getIntegerType(32)), + shape_lhs->getResults()[0], lhs_index_tensor->getResults()[0], true_attr); + auto rhs_index_tensor = createI32ConstantOp(r_index, loc, rewriter); + auto gather_rhs = TF::GatherOp::create( + *rewriter, loc, + RankedTensorType::get({static_cast(r_index.size())}, + rewriter->getIntegerType(32)), + shape_rhs->getResults()[0], rhs_index_tensor->getResults()[0], true_attr); + Value zero_value = createI32ConstOp(0, loc, rewriter); + auto concat_out_shape = TF::ConcatOp::create( + *rewriter, loc, + RankedTensorType::get({static_cast(bl_index.size()) + + static_cast(r_index.size())}, + rewriter->getIntegerType(32)), + zero_value, + ArrayRef( + {gather_lhs->getResults()[0], gather_rhs->getResults()[0]})); + + // Build ReshapeOp with the calculated output shape. + Type out_type = + RankedTensorType::get(shape, getElementTypeOrSelf(value.getType())); + return TF::ReshapeOp::create(*rewriter, loc, out_type, + /*tensor=*/value, + /*shape=*/concat_out_shape->getResults()[0]); +} + +std::optional> EquationToMap( + llvm::StringRef equation) { + llvm::SmallDenseMap map; + for (int64_t i = 0; i < equation.size(); ++i) { + if (!llvm::isAlpha(equation[i])) { + // Unsupported character in the equation. + return std::nullopt; + } + if (map.count(equation[i])) { + // Duplicate character in the equation. + return std::nullopt; + } + map.try_emplace(equation[i], i); + } + return map; +} + +std::optional> GetAvailableLabels( + llvm::StringRef lhs, llvm::StringRef rhs, int* lhs_named_label_count, + int* rhs_named_label_count) { + llvm::SetVector labels; + for (int i = 0; i < 26; ++i) { + labels.insert('a' + i); + labels.insert('A' + i); + } + + auto is_start_of_ellipsis = [](StringRef equation, int start_index) { + if (equation.size() < (start_index + 3)) return false; + + if (equation.substr(start_index, 3) != "...") return false; + return true; + }; + + int lhs_count = 0; + const int lhs_size = lhs.size(); + for (int i = 0; i < lhs_size; ++i) { + const char label = lhs[i]; + if (llvm::isAlpha(label)) { + labels.remove(label); + ++lhs_count; + } else if (label == '.') { + if (!is_start_of_ellipsis(lhs, i)) return std::nullopt; + i += 2; + } else { + // Unsupported character in the equation. + return std::nullopt; + } + } + *lhs_named_label_count = lhs_count; + + int rhs_count = 0; + const int rhs_size = rhs.size(); + for (int i = 0; i < rhs_size; ++i) { + const char label = rhs[i]; + if (llvm::isAlpha(label)) { + labels.remove(label); + ++rhs_count; + } else if (label == '.') { + if (!is_start_of_ellipsis(rhs, i)) return std::nullopt; + i += 2; + } else { + // Unsupported character in the equation. + return std::nullopt; + } + } + + *rhs_named_label_count = rhs_count; + return labels; +} + +// Generate new unnamed labels for the expression. +// For example, if we have GenerateLabels(2, {'b', 'c', 'd'}) for "...xy" +// We will have "dcxy" for the ellipsis expression since it's rank 4, +// we will have dcbxy if it's rank 5. +std::string GenerateLabels(int count, + const llvm::SetVector& available_labels) { + std::string new_labels(count, 0); + for (int i = 0; i < count; ++i) { + new_labels[count - 1 - i] = available_labels[i]; + } + + return new_labels; +} + +std::tuple FlattenEllipsis( + llvm::StringRef lhs, int lhs_named_label_count, llvm::StringRef rhs, + int rhs_named_label_count, llvm::StringRef out, RankedTensorType lhs_ty, + RankedTensorType rhs_ty, const llvm::SetVector& available_labels) { + std::string new_labels; + std::string new_lhs; + for (int i = 0; i < lhs.size(); ++i) { + const char label = lhs[i]; + if (llvm::isAlpha(label)) { + new_lhs.push_back(label); + } else { + // Encounter ellipsis: generate unnamed labels then insert to the new + // labels. + new_labels = GenerateLabels(lhs_ty.getRank() - lhs_named_label_count, + available_labels); + new_lhs.append(new_labels); + i += 2; + } + } + + std::string new_rhs, new_rhs_labels; + for (int i = 0; i < rhs.size(); ++i) { + const char label = rhs[i]; + if (llvm::isAlpha(label)) { + new_rhs.push_back(label); + } else { + // Encounter ellipsis: generate unnamed labels then insert to the new + // labels. + new_rhs_labels = GenerateLabels(rhs_ty.getRank() - rhs_named_label_count, + available_labels); + new_rhs.append(new_rhs_labels); + i += 2; + if (new_rhs_labels.size() > new_labels.size()) { + new_labels = new_rhs_labels; + } + } + } + + // Deal with the output next. + std::string new_output; + for (int i = 0; i < out.size(); ++i) { + const char label = out[i]; + if (llvm::isAlpha(label)) { + new_output.push_back(label); + } else { + // Encounter ellipsis: we will just insert the generated labels to the new + // output label. + new_output.append(new_labels); + i += 2; + } + } + + return std::make_tuple(new_lhs, new_rhs, new_output); +} + +// vectors/maps to map the dimensions of lhs with output in unary einsum op +std::optional GetEinsumDimensionNumbersUnary( + llvm::StringRef equation, RankedTensorType lhs_ty) { + llvm::StringRef lhs; + llvm::StringRef out; + std::tie(lhs, out) = equation.split("->"); + if (lhs.empty() || out.empty()) return std::nullopt; + + // Try to flatten the "..." if possible. + int lhs_named_label, rhs_named_label; + + // following rhs and rhs_ty variables are non-functional here only created to + // comply with the existing API + llvm::StringRef rhs; + RankedTensorType rhs_ty; + + auto available_labels = + GetAvailableLabels(lhs, rhs, &lhs_named_label, &rhs_named_label); + if (!available_labels.has_value()) return std::nullopt; + + auto flattended_labels = + FlattenEllipsis(lhs, lhs_named_label, rhs, rhs_named_label, out, lhs_ty, + rhs_ty, available_labels.value()); + + lhs = std::get<0>(flattended_labels); + out = std::get<2>(flattended_labels); + + auto lhs_map_or = EquationToMap(lhs); + if (!lhs_map_or.has_value()) return std::nullopt; + auto lhs_map = lhs_map_or.value(); + + auto out_map_or = EquationToMap(out); + if (!out_map_or.has_value()) return std::nullopt; + auto out_map = out_map_or.value(); + + EinsumDimensionNumbers dnums; + for (int64_t i = 0; i < lhs.size(); ++i) { + auto out_index = out_map.find(lhs[i]); + if (out_index == out_map.end()) { + dnums.lhs.emplace_back(i); + } else { + dnums.lhs_out.emplace_back(i, out_index->second); + } + } + + for (int64_t i = 0; i < out.size(); ++i) { + auto lhs_index = lhs_map.find(out[i]); + if (lhs_index == lhs_map.end()) { + // out only isn't supported + return std::nullopt; + } + } + return dnums; +} + +std::optional GetEinsumDimensionNumbers( + llvm::StringRef equation, RankedTensorType lhs_ty, + RankedTensorType rhs_ty) { + llvm::StringRef lhs_rhs; + llvm::StringRef out; + std::tie(lhs_rhs, out) = equation.split("->"); + if (lhs_rhs.empty() || out.empty()) return std::nullopt; + + llvm::StringRef lhs; + llvm::StringRef rhs; + std::tie(lhs, rhs) = lhs_rhs.split(','); + if (lhs.empty() || rhs.empty()) return std::nullopt; + + // Try to flatten the "..." if possible. + int lhs_named_label, rhs_named_label; + auto available_labels = + GetAvailableLabels(lhs, rhs, &lhs_named_label, &rhs_named_label); + if (!available_labels.has_value()) return std::nullopt; + + auto flattended_labels = + FlattenEllipsis(lhs, lhs_named_label, rhs, rhs_named_label, out, lhs_ty, + rhs_ty, available_labels.value()); + + lhs = std::get<0>(flattended_labels); + rhs = std::get<1>(flattended_labels); + out = std::get<2>(flattended_labels); + + auto lhs_map_or = EquationToMap(lhs); + if (!lhs_map_or.has_value()) return std::nullopt; + auto lhs_map = lhs_map_or.value(); + + auto rhs_map_or = EquationToMap(rhs); + if (!rhs_map_or.has_value()) return std::nullopt; + auto rhs_map = rhs_map_or.value(); + + auto out_map_or = EquationToMap(out); + if (!out_map_or.has_value()) return std::nullopt; + auto out_map = out_map_or.value(); + + EinsumDimensionNumbers dnums; + for (int64_t i = 0, e = lhs.size(); i < e; ++i) { + auto rhs_index = rhs_map.find(lhs[i]); + auto out_index = out_map.find(lhs[i]); + if (rhs_index == rhs_map.end() && out_index == out_map.end()) { + dnums.lhs.emplace_back(i); + } else if (rhs_index == rhs_map.end()) { + dnums.lhs_out.emplace_back(i, out_index->second); + } else if (out_index == out_map.end()) { + dnums.lhs_rhs.emplace_back(i, rhs_index->second); + } else { + dnums.lhs_rhs_out.emplace_back(i, rhs_index->second, out_index->second); + } + } + for (int64_t i = 0, e = rhs.size(); i < e; ++i) { + auto lhs_index = lhs_map.find(rhs[i]); + auto out_index = out_map.find(rhs[i]); + if (lhs_index == lhs_map.end()) { + if (out_index == out_map.end()) { + dnums.rhs.emplace_back(i); + } else { + dnums.rhs_out.emplace_back(i, out_index->second); + } + } + } + for (int64_t i = 0, e = out.size(); i < e; ++i) { + auto lhs_index = lhs_map.find(out[i]); + auto rhs_index = rhs_map.find(out[i]); + if (lhs_index == lhs_map.end() && rhs_index == rhs_map.end()) { + // out only isn't supported + return std::nullopt; + } + } + return dnums; +} + +// Function to replace a unary einsum op, that can undergo simple transpose, to +// an explicit transpose op. +LogicalResult rewriteToReduceSumAndTranspose(TF::EinsumOp op, + EinsumDimensionNumbers dnums, + PatternRewriter& rewriter) { + auto inputs = op.getInputs(); + Value lhs = inputs.front(); + + // Having indices in dnums.lhs list indicates that the ranks of the input and + // output to the unary einsum are not equal making it non-candidate for simple + // transpose. + bool needs_reduce_sum = false; + if (!dnums.lhs.empty()) { + needs_reduce_sum = true; + llvm::SmallVector reduce_idcs(dnums.lhs.size()); + for (int64_t i = 0; i < dnums.lhs.size(); ++i) { + reduce_idcs[i] = dnums.lhs[i]; + } + + lhs = createSumOp(lhs, lhs.getLoc(), reduce_idcs, &rewriter); + } + + llvm::SmallVector lhs_transpose; + lhs_transpose.reserve(dnums.lhs_out.size()); + + llvm::SmallDenseMap out_lhs_map(dnums.lhs_out.size()); + for (int64_t i = 0; i < dnums.lhs_out.size(); ++i) { + out_lhs_map[std::get<1>(dnums.lhs_out[i])] = std::get<0>(dnums.lhs_out[i]); + } + + bool needs_transpose = false; + for (int64_t i = 0; i < dnums.lhs_out.size(); ++i) { + if (std::get<0>(dnums.lhs_out[i]) > + mlir::cast(lhs.getType()).getRank() - 1) { + continue; + } + + if (std::get<0>(dnums.lhs_out[i]) != std::get<1>(dnums.lhs_out[i])) { + needs_transpose = true; + } + lhs_transpose.push_back(out_lhs_map[i]); + } + + if (!needs_reduce_sum && !needs_transpose) { + return rewriter.notifyMatchFailure( + op, "unary einsum equation does not require transpose"); + } else if (needs_reduce_sum && !needs_transpose) { + rewriter.replaceOp(op, lhs); + return success(); + } + + lhs = createTransposeOp(lhs, lhs.getLoc(), lhs_transpose, &rewriter); + rewriter.replaceOp(op, lhs); + return success(); +} + +std::vector inverseTransposeVector( + llvm::ArrayRef input, llvm::ArrayRef permutation) { + std::vector output(input.size()); + for (int64_t i = 0; i < input.size(); ++i) { + output[permutation[i]] = input[i]; + } + return output; +} + +// Computes the transpositions required to convert dnums to one supported by +// tf.BatchMatmulV2 and returns the new set of dimension numbers with them. +// Transposed LHS shape will be B0,...,Bn,L0,...,Ln,C0,...,Cn and, +// transposed RHS shape will be B0,...,Bn,C0,...,Cn,R0,...,Rn respectively. +LogicalResult transposeForBatchMatmul( + const Location& loc, EinsumDimensionNumbers& dnums, Value* lhs, Value* rhs, + std::vector* out_inverse_transpose, PatternRewriter* rewriter) { + std::vector lhs_transpose; + std::vector rhs_transpose; + std::vector out_transpose; + lhs_transpose.reserve(dnums.lhs_rhs_out.size() + dnums.lhs_out.size() + + dnums.lhs_rhs.size()); + rhs_transpose.reserve(dnums.lhs_rhs_out.size() + dnums.rhs_out.size() + + dnums.lhs_rhs.size()); + out_transpose.reserve(dnums.lhs_rhs_out.size() + dnums.lhs_out.size() + + dnums.rhs_out.size()); + // Generate transpose matrix for B0,...,Bn + for (int64_t i = 0, e = dnums.lhs_rhs_out.size(); i < e; ++i) { + lhs_transpose.push_back(std::get<0>(dnums.lhs_rhs_out[i])); + rhs_transpose.push_back(std::get<1>(dnums.lhs_rhs_out[i])); + out_transpose.push_back(std::get<2>(dnums.lhs_rhs_out[i])); + dnums.lhs_rhs_out[i] = std::make_tuple(i, i, i); + } + + // Generate transpose matrix for L0,...,Ln + for (int64_t i = 0, e = dnums.lhs_out.size(); i < e; ++i) { + lhs_transpose.push_back(std::get<0>(dnums.lhs_out[i])); + out_transpose.push_back(std::get<1>(dnums.lhs_out[i])); + dnums.lhs_out[i] = + std::make_tuple(lhs_transpose.size() - 1, out_transpose.size() - 1); + } + // Generate transpose matrix for C0,...,Cn + for (int64_t i = 0, e = dnums.lhs_rhs.size(); i < e; ++i) { + lhs_transpose.push_back(std::get<0>(dnums.lhs_rhs[i])); + rhs_transpose.push_back(std::get<1>(dnums.lhs_rhs[i])); + dnums.lhs_rhs[i] = + std::make_tuple(lhs_transpose.size() - 1, rhs_transpose.size() - 1); + } + for (int64_t i = 0, e = dnums.rhs_out.size(); i < e; ++i) { + rhs_transpose.push_back(std::get<0>(dnums.rhs_out[i])); + out_transpose.push_back(std::get<1>(dnums.rhs_out[i])); + dnums.rhs_out[i] = + std::make_tuple(rhs_transpose.size() - 1, out_transpose.size() - 1); + } + + out_inverse_transpose->resize(out_transpose.size()); + for (int64_t i = 0, e = out_transpose.size(); i < e; ++i) { + out_inverse_transpose->at(out_transpose[i]) = i; + } + + *lhs = createTransposeOp(*lhs, loc, lhs_transpose, rewriter); + *rhs = createTransposeOp(*rhs, loc, rhs_transpose, rewriter); + return success(); +} + +template +inline int64_t ProdShapeWithIndexInTuple( + ArrayRef shape, + const std::vector>& index_tuples) { + int64_t prod_shape = 1; + for (auto index_tuple : index_tuples) { + const int64_t shape_i = shape[std::get(index_tuple)]; + if (ShapedType::isDynamic(shape_i)) return ShapedType::kDynamic; + prod_shape *= shape_i; + } + return prod_shape; +} + +// Reshapes LHS and RHS to have B0,...,Bn,L,C and B0,...,Bn,C,R shape +// respectively while assuming that the initial shape for them is +// B0,...,Bn,L0,...,Ln,C0,...,Cn and B0,...,Bn,C0,...,Cn,R0,...,Rn respectively. +LogicalResult reshapeForBatchMatmul(const Location& loc, + EinsumDimensionNumbers& dnums, Value* lhs, + Value* rhs, + SmallVectorImpl* out_shape, + PatternRewriter* rewriter) { + RankedTensorType lhs_type = mlir::cast(lhs->getType()); + RankedTensorType rhs_type = mlir::cast(rhs->getType()); + + int32_t num_lhs_reshape_segids = 0; + int32_t num_rhs_reshape_segids = 0; + std::vector lhs_reshape_segids; + int lhs_rank = + dnums.lhs_rhs_out.size() + dnums.lhs_out.size() + dnums.lhs_rhs.size(); + lhs_reshape_segids.resize(lhs_rank); + std::vector rhs_reshape_segids; + int rhs_rank = + dnums.lhs_rhs_out.size() + dnums.rhs_out.size() + dnums.lhs_rhs.size(); + rhs_reshape_segids.resize(rhs_rank); + + // Labels exist in all lhs, rhs and output are the batch labels B0,...,Bn. + std::vector lhs_shape; + std::vector rhs_shape; + lhs_shape.reserve(dnums.lhs_rhs_out.size() + dnums.lhs_out.size() + 1); + rhs_shape.reserve(dnums.lhs_rhs_out.size() + 2); + for (auto i : dnums.lhs_rhs_out) { + const int64_t b1 = lhs_type.getShape()[std::get<0>(i)]; + lhs_shape.push_back(b1); + const int64_t b2 = rhs_type.getShape()[std::get<1>(i)]; + rhs_shape.push_back(b2); + + lhs_reshape_segids.at(std::get<0>(i)) = num_lhs_reshape_segids++; + rhs_reshape_segids.at(std::get<1>(i)) = num_rhs_reshape_segids++; + } + if (!OpTrait::util::getBroadcastedShape(lhs_shape, rhs_shape, *out_shape)) { + return failure(); + } + + // Calculates dimension for the label L from L0,...,Ln in lhs. + if (dnums.lhs_out.empty()) { + lhs_shape.push_back(1); + out_shape->push_back(1); + dnums.lhs_out.emplace_back(lhs_shape.size() - 1, out_shape->size() - 1); + ++num_lhs_reshape_segids; + } else if (dnums.lhs_rhs_out.empty()) { + // If there is not batch labels B0,...,Bn, it is safe to use L0,...,Ln as + // the batch labels in lhs, the rhs will be broadcasted. + for (auto i : dnums.lhs_out) { + const int64_t b = lhs_type.getShape()[std::get<0>(i)]; + lhs_shape.push_back(b); + out_shape->push_back(b); + + lhs_reshape_segids.at(std::get<0>(i)) = num_lhs_reshape_segids++; + } + } else { + const int64_t lhs_out_size = + ProdShapeWithIndexInTuple<0>(lhs_type.getShape(), dnums.lhs_out); + lhs_shape.push_back(lhs_out_size); + out_shape->push_back(lhs_out_size); + + for (auto i : dnums.lhs_out) { + lhs_reshape_segids.at(std::get<0>(i)) = num_lhs_reshape_segids; + } + ++num_lhs_reshape_segids; + } + + // Calculates dimension for the common label C from labels C0,...,Cn that + // exist in both lhs and rhs. + const int64_t lhs_size = + ProdShapeWithIndexInTuple<0>(lhs_type.getShape(), dnums.lhs_rhs); + const int64_t rhs_size = + ProdShapeWithIndexInTuple<1>(rhs_type.getShape(), dnums.lhs_rhs); + lhs_shape.push_back(lhs_size); + rhs_shape.push_back(rhs_size); + + for (auto i : dnums.lhs_rhs) { + lhs_reshape_segids.at(std::get<0>(i)) = num_lhs_reshape_segids; + rhs_reshape_segids.at(std::get<1>(i)) = num_rhs_reshape_segids; + } + ++num_lhs_reshape_segids; + ++num_rhs_reshape_segids; + + // Calculates dimension for the label R from R0,...,Rn in rhs. + const int64_t rhs_out_size = + ProdShapeWithIndexInTuple<0>(rhs_type.getShape(), dnums.rhs_out); + rhs_shape.push_back(rhs_out_size); + out_shape->push_back(rhs_out_size); + + for (auto i : dnums.rhs_out) { + rhs_reshape_segids.at(std::get<0>(i)) = num_rhs_reshape_segids; + } + ++num_rhs_reshape_segids; + + // If LHS requires reshapes. + if (lhs_rank != num_lhs_reshape_segids) { + if (succeeded(VerifyShapeOfReshapeOp(lhs_shape))) { + *lhs = createReshapeOp(*lhs, lhs_shape, lhs_type.getElementType(), loc, + rewriter); + } else { + // Check if lhs LHS shape can be calculated with SegmentProd. It requires + // to have at least 1 common index in lhs_out and lhs_rhs. + if (dnums.lhs_out.empty() || dnums.lhs_rhs.empty()) return failure(); + *lhs = createReshapeOpForDynamic(*lhs, lhs_shape, lhs_reshape_segids, + num_lhs_reshape_segids, loc, rewriter); + } + } + // If RHS requires reshapes. + if (rhs_rank != num_rhs_reshape_segids) { + if (succeeded(VerifyShapeOfReshapeOp(rhs_shape))) { + *rhs = createReshapeOp(*rhs, rhs_shape, rhs_type.getElementType(), loc, + rewriter); + } else { + // Check if lhs RHS shape can be calculated with SegmentProd. It requires + // to have at least 1 common index in rhs_out and lhs_rhs. + if (dnums.rhs_out.empty() || dnums.lhs_rhs.empty()) return failure(); + *rhs = createReshapeOpForDynamic(*rhs, rhs_shape, rhs_reshape_segids, + num_rhs_reshape_segids, loc, rewriter); + } + } + + dnums.lhs_rhs.assign( + {std::make_tuple(dnums.lhs_rhs_out.size() + dnums.lhs_out.size(), + dnums.lhs_rhs_out.size())}); + dnums.rhs_out.assign( + {std::make_tuple(dnums.lhs_rhs_out.size() + dnums.lhs_out.size(), + dnums.lhs_rhs_out.size() + dnums.lhs_out.size())}); + return success(); +} + +LogicalResult rewriteToBatchMatmul(TF::EinsumOp op, + EinsumDimensionNumbers dnums, + PatternRewriter& rewriter) { + if (!dnums.lhs.empty() || !dnums.rhs.empty()) return failure(); + + auto inputs = op.getInputs(); + if (inputs.size() != 2) return failure(); + Value lhs = inputs.front(); + Value rhs = inputs.back(); + // Back original values for the later output shape calculation in + // `createOutputReshapeOpForDynamic`. + Value original_lhs = lhs; + Value original_rhs = rhs; + EinsumDimensionNumbers original_dnums = dnums; + + RankedTensorType original_type = + mlir::dyn_cast_or_null(op.getResult().getType()); + if (!original_type) return failure(); + + std::vector out_transpose; + if (failed(transposeForBatchMatmul(op.getLoc(), dnums, &lhs, &rhs, + &out_transpose, &rewriter))) + return failure(); + + llvm::SmallVector matmul_shape; + if (failed(reshapeForBatchMatmul(op.getLoc(), dnums, &lhs, &rhs, + &matmul_shape, &rewriter))) + return failure(); + + std::vector reshape_shape = + inverseTransposeVector(original_type.getShape(), out_transpose); + + auto matmul_type = + RankedTensorType::get(matmul_shape, original_type.getElementType()); + Value out = TF::BatchMatMulV2Op::create(rewriter, op.getLoc(), matmul_type, + lhs, rhs, rewriter.getBoolAttr(false), + rewriter.getBoolAttr(false)); + + bool out_reshape_need = (reshape_shape.size() != matmul_shape.size() || + original_type.getRank() != matmul_shape.size()); + // Always add reshape for concrete output shapes. + if (succeeded(VerifyShapeOfReshapeOp(reshape_shape))) { + out = createReshapeOp(out, reshape_shape, original_type.getElementType(), + op.getLoc(), &rewriter); + } else if (out_reshape_need) { + out = createOutputReshapeOpForDynamic(out, reshape_shape, original_lhs, + original_rhs, original_dnums, + op.getLoc(), &rewriter); + } + out = createTransposeOp(out, op.getLoc(), out_transpose, &rewriter); + + rewriter.replaceOp(op, out); + return success(); +} + +LogicalResult matchAndRewriteUnaryEinsumOp(TF::EinsumOp op, + PatternRewriter& rewriter) { + if (op->getNumOperands() != 1) { + return rewriter.notifyMatchFailure( + op, "Function only supports unary einsum op"); + } + RankedTensorType lhs = + mlir::dyn_cast_or_null(op.getOperand(0).getType()); + if (!lhs) { + return failure(); + } + // unary einsum op is only supported to the case where the operation can be + // replaced using reduce_sum and/or transpose + if (const auto dnums_or = + GetEinsumDimensionNumbersUnary(op.getEquation(), lhs)) { + return rewriteToReduceSumAndTranspose(op, dnums_or.value(), rewriter); + } + + return rewriter.notifyMatchFailure(op, "unsupported einsum lowering"); +} + +#define GEN_PASS_DEF_TRANSFORMEINSUMPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Transform Einsum to other TF Ops for the supported variants. +struct TransformEinsumPass + : public impl::TransformEinsumPassBase { + void runOnOperation() override; +}; + +void TransformEinsumPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + auto func = getOperation(); + + patterns.add(&getContext()); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +LogicalResult ConvertTFEinsumOp::matchAndRewrite( + TF::EinsumOp op, PatternRewriter& rewriter) const { + if (op->getNumOperands() == 1) { + return matchAndRewriteUnaryEinsumOp(op, rewriter); + } + + RankedTensorType lhs = + mlir::dyn_cast_or_null(op.getOperand(0).getType()); + RankedTensorType rhs = + mlir::dyn_cast_or_null(op.getOperand(1).getType()); + if (!lhs || !rhs) { + return failure(); + } + + // TODO(b/162328998) Better support Einsum with dynamic input. Currently, + // one dynamic dimension is always supported. If there are two or more + // dynamic dimensions, it is supported if they only exist in a single + // component among: L0,...,Ln R0,...,Rn or C0,...,Cn. + if (const auto dnums_or = + GetEinsumDimensionNumbers(op.getEquation(), lhs, rhs)) + return rewriteToBatchMatmul(op, dnums_or.value(), rewriter); + return rewriter.notifyMatchFailure(op, "unsupported einsum lowering"); +} + +std::unique_ptr> CreateTransformEinsumPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.h new file mode 100644 index 0000000000000000000000000000000000000000..36425668a876fb9873a0d42b5c5b9d6108b2021f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/einsum.h @@ -0,0 +1,55 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// This pass identifies patterns for certain Einsum Ops and replaces them +// with other equivalent TF Ops. + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_EINSUM_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_EINSUM_H_ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/matmul_bcast.h" + +namespace mlir { +namespace TF { + +// TF.Einsum provides fully general tensor contractions. For a few select +// cases, we can convert this op to other TF Ops, which in later passes +// properly convert to TF Lite ops. +struct ConvertTFEinsumOp : public OpRewritePattern { + public: + explicit ConvertTFEinsumOp(MLIRContext* context) + : OpRewritePattern(context) {} + + LogicalResult matchAndRewrite(TF::EinsumOp op, + PatternRewriter& rewriter) const override; +}; + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_EINSUM_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_island_coarsening.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_island_coarsening.cc new file mode 100644 index 0000000000000000000000000000000000000000..60ff6516d5284887b37d0e72b42dc74e011cea10 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_island_coarsening.cc @@ -0,0 +1,513 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass takes TensorFlow executor dialect IslandOps and +// merges them. Note, this currently does not handle TensorFlow V1 style control +// flow/frames or side effecting ops yet. + +#include +#include + +#include "absl/log/check.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/core/platform/logging.h" + +namespace mlir { +namespace tf_executor { + +namespace { + +//===----------------------------------------------------------------------===// +// Analysis +//===----------------------------------------------------------------------===// + +// This structure represents a merged island. It includes all of the islands +// that can be merged together and the point of insertion of the merged island. +struct MergedIsland { + // Construct a new island from the given root. + explicit MergedIsland(IslandOp root) : insert_point(root) { + islands.push_back(root); + } + + // The insertion point anchor of the merged island, or where the merged island + // will be inserted when created. + Operation* const insert_point; + + // The set of islands that are to be merged together. + SmallVector islands; +}; + +// This structure contains all of the merge decisions for islands within a +// graph. We compute which islands to merge first, so that we don't need to +// recursively mutate the IR (resulting in quadratic behavior when moving +// operations). A rough sketch of the coarsening algorithm is shown below: +// +// // The algorithm iterates until a fixpoint is reached, i.e. when no more +// // islands can be merged. +// while (changed) { +// // In the first phase we try to merge islands with their nearest consumer +// // iff the consumer is another island. +// // Note: A consumer is an operation that consumes one of our outputs. +// changed |= tryMergedIslandsIntoNearestConsumer(); +// +// // In the second phase we try to merge islands with their nearest producer +// // of a value they consume, iff the producer is another island. +// // Note: A producer is an operation that produces one of our inputs. +// changed |= tryMergedIslandsIntoNearestProducer(); +// } +// +class CoarseningAnalysis { + public: + // Compute the coarsening analysis over the given graph. + explicit CoarseningAnalysis(GraphOp graph); + + // Returns a list of all of the mergable islands found in the graph. + iterator_range< + llvm::filter_iterator::const_iterator, + function_ref>> + GetMergableIslands() const { + return llvm::make_filter_range( + merged_islands_, function_ref( + [](const MergedIsland& merged_island) { + return merged_island.islands.size() > 1; + })); + } + + private: + // Attempt to find an island group that produces a value consumed by one of + // the islands (or operation therein) within the given `merged_island`. If no + // candidate can be found, returns nullptr. + MergedIsland* GetOperandCandidateToMergeWith(GraphOp graph, + MergedIsland& merged_island); + + // Attempt to find an island group that consumes a result, either control or + // data, from one of the islands in the given `merged_island`. If no candidate + // can be found, returns nullptr. + MergedIsland* GetResultCandidateToMergeWith(GraphOp graph, + MergedIsland& merged_island); + + // All of the merged islands in the graph. + SmallVector merged_islands_; + // A mapping from an island operation to the current merged island group it + // is a part of. + DenseMap island_to_merged_island_; +}; + +CoarseningAnalysis::CoarseningAnalysis(GraphOp graph) { + // As an initial step, construct a merged island for each island in the + // graph. + for (IslandOp island : graph.SingleBlock::getBody()->getOps()) + merged_islands_.push_back(MergedIsland(island)); + + // Record the mapping from the island to the merge group as a secondary step, + // as we are taking the address of the islands here and the push_back step + // above may invalidate previously inserted islands mid-loop. + for (MergedIsland& island : merged_islands_) + island_to_merged_island_.try_emplace(island.insert_point, &island); + + // This functor merges the given `old_merged_island` into the + // `new_merged_island`. `merge_in_front` is whether the old island should be + // merged into the front of the new island, or the back. + auto merge_islands = [&](MergedIsland& old_merged_island, + MergedIsland& new_merged_island, + bool merge_in_front) { + for (IslandOp island : old_merged_island.islands) + island_to_merged_island_[island] = &new_merged_island; + + auto insert_point = merge_in_front ? new_merged_island.islands.begin() + : new_merged_island.islands.end(); + new_merged_island.islands.insert(insert_point, + old_merged_island.islands.begin(), + old_merged_island.islands.end()); + old_merged_island.islands.clear(); + }; + + // Iterate over all of the island groups attempting to merge as many islands + // groups as possible. + bool updated = false; + do { + updated = false; + + // Attempt to merge an island into an island consuming one of its results. + for (MergedIsland& merged_island : llvm::reverse(merged_islands_)) { + if (merged_island.islands.empty()) continue; + + MergedIsland* candidate = + GetResultCandidateToMergeWith(graph, merged_island); + if (candidate) { + merge_islands(merged_island, *candidate, /*merge_in_front=*/true); + updated = true; + } + } + + // Attempt to merge an island into an island producing one of its operands. + for (MergedIsland& merged_island : merged_islands_) { + if (merged_island.islands.empty()) continue; + + MergedIsland* candidate = + GetOperandCandidateToMergeWith(graph, merged_island); + if (candidate) { + merge_islands(merged_island, *candidate, /*merge_in_front=*/false); + updated = true; + } + } + } while (updated); +} + +MergedIsland* CoarseningAnalysis::GetOperandCandidateToMergeWith( + GraphOp graph, MergedIsland& merged_island) { + // The candidate operation to consider merging the current island group with. + Operation* candidate = nullptr; + // The island group of the current candidate if it is an IslandOp, nullptr + // otherwise. + MergedIsland* candidate_island = nullptr; + + // Given an input operation, try to replace the current candidate operation + // with it. + auto try_update_current_candidate = [&](Operation* rhs) { + MergedIsland* rhs_island = nullptr; + // Check if this is an island operation we can merge with. + auto rhs_it = island_to_merged_island_.find(rhs); + if (rhs_it != island_to_merged_island_.end()) { + rhs_island = rhs_it->second; + + // Ignore islands that are already a part of the current island group. + if (rhs_island == &merged_island) return; + + rhs = rhs_island->insert_point; + } + if (!candidate || candidate->isBeforeInBlock(rhs)) { + candidate = rhs; + candidate_island = rhs_island; + } + }; + + // Check island control operands. + for (IslandOp island : merged_island.islands) { + for (Value input : island.getControlInputs()) { + Operation* def = input.getDefiningOp(); + DCHECK_EQ(def->getParentOp(), graph); + try_update_current_candidate(def); + } + + // Check island data operands. + island.walk([&](Operation* op) { + for (Value input : op->getOperands()) { + Operation* def = input.getDefiningOp(); + if (!def || def->getParentOp() != graph) continue; + + try_update_current_candidate(def); + } + }); + } + + return candidate_island; +} + +MergedIsland* CoarseningAnalysis::GetResultCandidateToMergeWith( + GraphOp graph, MergedIsland& merged_island) { + // The candidate operation to consider merging the current island group with. + Operation* candidate = nullptr; + // The island group of the current candidate if it is an IslandOp, nullptr + // otherwise. + MergedIsland* candidate_island = nullptr; + + // Given an input operation, try to replace the current candidate operation + // with it. + auto try_update_current_candidate = [&](Operation* rhs) { + MergedIsland* rhs_island = nullptr; + + // Check if this is an island operation we can merge with. + auto rhs_it = island_to_merged_island_.find(rhs); + if (rhs_it != island_to_merged_island_.end()) { + rhs_island = rhs_it->second; + + // Ignore islands that are already a part of the current island group. + if (rhs_island == &merged_island) return; + + rhs = rhs_island->insert_point; + } + if (!candidate || rhs->isBeforeInBlock(candidate)) { + candidate = rhs; + candidate_island = rhs_island; + } + }; + + // Check island control results. + for (IslandOp island : merged_island.islands) { + for (Operation* user : island.getControl().getUsers()) { + DCHECK_EQ(user->getParentOp(), graph); + try_update_current_candidate(user); + } + + // Check island data results. + Block& graph_body = llvm::cast(graph).GetBody(); + for (Value result : island.getOutputs()) { + for (Operation* user : result.getUsers()) { + Operation* def = graph_body.findAncestorOpInBlock(*user); + DCHECK_NE(def, nullptr); + try_update_current_candidate(def); + } + } + } + + return candidate_island; +} + +//===----------------------------------------------------------------------===// +// Transformation +//===----------------------------------------------------------------------===// + +// IslandResult is a helper struct holding an islands result and associated +// inner op result. +struct IslandResult { + IslandResult(Value inner_op_result, Value island_result) + : inner_op_result(inner_op_result), island_result(island_result) {} + + Value inner_op_result; + Value island_result; +}; + +// This structure is used to gather the new operands and result of an island +// during merging. +struct IslandOperandsAndResults { + llvm::SmallSetVector operands; + llvm::SmallVector results; +}; + +// Collects the results for the new island by going through each data result of +// the islands being merged. Unused results outside of the merged island to be +// formed are pruned. If the child island inner ops consume the parent island +// control result, the child island inner ops will have that respective control +// input pruned. Results of the parent island that are consumed by the child +// island are replaced by the respective inner ops result from the parent +// island. +void GetNewIslandResultsAndForwardResults( + const MergedIsland& merged_island, + llvm::SmallVector& results) { + results.clear(); + + // Collect all of the blocks within each of the island operations, these will + // be used to detect when an operation has a use within one of the merged + // islands. + llvm::SmallPtrSet islandBlocks; + for (IslandOp island : merged_island.islands) + island->walk([&](Block* block) { islandBlocks.insert(block); }); + + for (IslandOp island : merged_island.islands) { + for (auto ret_vals : + llvm::zip(island.GetYield().getOperands(), island.getOutputs())) { + bool result_captured = false; + Value inner_op_result = std::get<0>(ret_vals); + Value island_result = std::get<1>(ret_vals); + for (auto& use : llvm::make_early_inc_range(island_result.getUses())) { + if (islandBlocks.count(use.getOwner()->getBlock())) { + // If the use is within our island group, forward the result from + // inner op. + use.set(inner_op_result); + } else if (!result_captured) { + results.emplace_back(inner_op_result, island_result); + result_captured = true; + } + } + } + } +} + +// Creates the new merged island. +IslandOp CreateNewIsland(const MergedIsland& merged_island, + llvm::ArrayRef operands, + llvm::ArrayRef results) { + // Collect types from results. + llvm::SmallVector result_types; + result_types.reserve(results.size()); + for (const auto& result : results) + result_types.push_back(result.inner_op_result.getType()); + + // IslandOps always have a control result. + result_types.push_back( + ControlType::get(merged_island.insert_point->getContext())); + + OpBuilder builder(merged_island.insert_point); + auto new_island = IslandOp::create( + builder, merged_island.insert_point->getLoc(), result_types, operands); + new_island.getBody().push_back(new Block); + return new_island; +} + +// Creates respective YieldOp for the new merged island. +YieldOp CreateNewIslandYieldOp(IslandOp new_island, + llvm::MutableArrayRef results) { + llvm::SmallVector yield_operands; + yield_operands.reserve(results.size()); + + for (auto [old_result, new_island] : + llvm::zip(results, new_island.getOutputs())) { + // Replace original island result with new island result. + old_result.island_result.replaceAllUsesWith(new_island); + + // Add associated inner op result to operands of the YieldOp. + yield_operands.push_back(old_result.inner_op_result); + } + + // Create YieldOp for the new island. + OpBuilder builder(&new_island.GetBody(), new_island.GetBody().end()); + return YieldOp::create(builder, new_island.getLoc(), yield_operands); +} + +// Moves inner ops (excluding last op/YieldOp) from islands being merged into +// the new merged island. +void MoveInnerOpsToNewIsland(const MergedIsland& merged_island, + Operation* new_yield_op) { + Block* block = new_yield_op->getBlock(); + + auto move_inner_ops = [block, new_yield_op](IslandOp island) { + auto& island_body = island.GetBody().getOperations(); + block->getOperations().splice(new_yield_op->getIterator(), island_body, + island_body.begin(), + std::prev(island_body.end())); + }; + for (IslandOp island : merged_island.islands) move_inner_ops(island); +} + +// Merges the islands within the given island group. +// `island_operands_and_results` is passed in as scrach storage for the duration +// of this function. +void MergeIslands(const MergedIsland& merged_island, + IslandOperandsAndResults& island_operands_and_results) { + // Collect operands for the new merged island. + island_operands_and_results.operands.clear(); + for (IslandOp island : merged_island.islands) + island_operands_and_results.operands.insert(island.operand_begin(), + island.operand_end()); + for (IslandOp island : merged_island.islands) + island_operands_and_results.operands.remove(island.getControl()); + + // Collect results for the new merged island. + GetNewIslandResultsAndForwardResults(merged_island, + island_operands_and_results.results); + + // Create the new merged island. + IslandOp new_island = CreateNewIsland( + merged_island, island_operands_and_results.operands.getArrayRef(), + island_operands_and_results.results); + + // Create associated YieldOp for the new merged island. + YieldOp new_yield_op = + CreateNewIslandYieldOp(new_island, island_operands_and_results.results); + + // Move inner ops from original islands into the new island. + MoveInnerOpsToNewIsland(merged_island, new_yield_op.getOperation()); + + // Update control inputs to point to the new merged island. + for (IslandOp island : merged_island.islands) + island.getControl().replaceAllUsesWith(new_island.getControl()); + for (IslandOp island : merged_island.islands) island->erase(); +} + +// Takes the inputs to tf_executor.fetch, make a new island that just yields +// them, and replace the fetch's input operands with the new yielded values. +// +// This allows our def-use based island coarsening algorithm to merge +// islands that independently feed into a fetch. +void InsertDummyIslandForFetch(FetchOp fetch) { + llvm::SmallVector data_fetches; + llvm::SmallVector data_types; + llvm::SmallVector control_fetches; + data_fetches.reserve(fetch.getFetches().size()); + data_types.reserve(data_fetches.capacity()); + control_fetches.reserve(data_fetches.capacity()); + + for (auto value : fetch.getFetches()) { + if (mlir::isa(value.getType())) { + control_fetches.push_back(value); + } else { + data_fetches.push_back(value); + data_types.push_back(value.getType()); + } + } + OpBuilder builder(fetch); + auto island = + IslandOp::create(builder, fetch.getLoc(), data_types, + /*control=*/ControlType::get(fetch.getContext()), + /*controlInputs=*/control_fetches); + island.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&island.GetBody()); + YieldOp::create(builder, fetch.getLoc(), data_fetches); + const int fetch_control_idx = data_fetches.size(); + for (int i = 0, e = fetch.getNumOperands(); i < e; i++) { + // The fetch could have multiple control operands (all at the end of its + // operand list). We replace them all with the island's single control + // operand. + if (i <= fetch_control_idx) { + fetch.setOperand(i, island.getResult(i)); + } else { + fetch.getOperation()->eraseOperand(fetch.getNumOperands() - 1); + } + } +} + +//===----------------------------------------------------------------------===// +// Pass Entry Point +//===----------------------------------------------------------------------===// + +#define GEN_PASS_DEF_EXECUTORISLANDCOARSENINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ExecutorIslandCoarseningPass + : public impl::ExecutorIslandCoarseningPassBase< + ExecutorIslandCoarseningPass> { + void runOnOperation() override; +}; + +void ExecutorIslandCoarseningPass::runOnOperation() { + // Temporary datastructure to keep operands and results for each island. + // We define it here to grow and reuse the storage for the duration of the + // pass. + IslandOperandsAndResults island_operands_and_results; + + getOperation().walk([&](GraphOp graph) { + InsertDummyIslandForFetch(graph.GetFetch()); + + // Compute an analysis that decides which islands should be merged together, + // and merge any island groups it finds. + CoarseningAnalysis analysis(graph); + for (const MergedIsland& island : analysis.GetMergableIslands()) + MergeIslands(island, island_operands_and_results); + }); +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorIslandCoarseningPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_inline_tpu_island.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_inline_tpu_island.cc new file mode 100644 index 0000000000000000000000000000000000000000..60c0da92c09a81ed7f956648d4ec65fdc5e39cf1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_inline_tpu_island.cc @@ -0,0 +1,106 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Inliner.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +#define DEBUG_TYPE "tf-executor-tpu-v1-island-inlining" + +namespace mlir { +namespace tf_executor { + +namespace { +constexpr llvm::StringRef kNestedModule = "_tpu_v1_compat_outlined"; + +#define GEN_PASS_DEF_EXECUTORTPUV1ISLANDINLININGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ExecutorTPUV1IslandInliningPass + : public impl::ExecutorTPUV1IslandInliningPassBase< + ExecutorTPUV1IslandInliningPass> { + void runOnOperation() override; +}; + +void ExecutorTPUV1IslandInliningPass::runOnOperation() { + SymbolTable symbol_table(getOperation()); + Operation *nested_module = symbol_table.lookup(kNestedModule); + if (!nested_module) return; + + InlinerInterface inliner(&getContext()); + InlinerConfig config; + auto walk_result = getOperation().walk([&](TF::PartitionedCallOp call_op) { + if (!call_op.getF().getRootReference().getValue().starts_with( + kNestedModule)) + return WalkResult::advance(); + // This is a call we need to inline! + LLVM_DEBUG(llvm::dbgs() + << "Found call to inline: " << *call_op.getOperation() << "\n"); + + auto call_interface = cast(call_op.getOperation()); + auto called_func = + dyn_cast_or_null(call_interface.resolveCallable()); + + if (failed(inlineCall(inliner, config.getCloneCallback(), call_interface, + cast(called_func.getOperation()), + called_func.getCallableRegion(), + /* shouldCloneInlinedRegion = */ false))) { + call_op.emitOpError() << "Failed to inline\n"; + return WalkResult::interrupt(); + } + called_func.erase(); + call_op.erase(); + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return signalPassFailure(); + // Move all remaining nested functions back into the parent module. + Block &nested_block = nested_module->getRegion(0).front(); + for (func::FuncOp func_op : + llvm::make_early_inc_range(nested_block.getOps())) { + if (!symbol_table.lookupSymbolIn(getOperation(), func_op.getName())) { + nested_block.getOperations().remove(func_op.getOperation()); + symbol_table.insert(func_op.getOperation()); + } + } + nested_module->erase(); +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorTPUV1IslandInliningPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_island_coarsening.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_island_coarsening.cc new file mode 100644 index 0000000000000000000000000000000000000000..716d0c21b59cfdc21de23351664adcdf1490a056 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_island_coarsening.cc @@ -0,0 +1,708 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass takes TensorFlow executor dialect IslandOps and +// merges the one that contains operation marked to run on TPU. + +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/core/platform/logging.h" + +#define DEBUG_TYPE "tf-executor-tpu-v1-island-coarsening" + +namespace mlir { +namespace tf_executor { + +namespace { + +constexpr llvm::StringRef kTpuStatusAttr = "_tpu_compilation_status"; +constexpr llvm::StringRef kNoReplicationCluster = "__no_replication_cluster"; + +#define GEN_PASS_DEF_TPUV1BRIDGEEXECUTORISLANDCOARSENINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// This pass is a variant of the island coarsening that is limited to +// TPU-annotated operations and intended to preserve backward compatibility with +// TFv1. +struct TpuV1BridgeExecutorIslandCoarsening + : public impl::TpuV1BridgeExecutorIslandCoarseningPassBase< + TpuV1BridgeExecutorIslandCoarsening> { + void runOnOperation() override; +}; + +// Returns name of TPU cluster, if op belongs to a TPU cluster. Otherwise, +// returns `std::nullopt`. +std::optional GetTpuClusterName(Operation* op) { + if (auto tpu_status = op->getAttrOfType(kTpuStatusAttr)) { + // Borrow cluster name from TPU status (for `TPUCompilationResult` op). + return tpu_status.getValue(); + } + auto device_type = op->getAttrOfType(TF::kCompileDeviceTypeAttr); + if (!device_type || device_type.getValue() != TF::kTpuDevice) { + // Op does not belong to a TPU cluster. + return std::nullopt; + } + // Op belongs to a TPU cluster. + if (auto replication_info = + op->getAttrOfType(TF::kReplicationInfoAttr)) { + // Borrow cluster name from replication info. + return replication_info.getValue(); + } + // Use special cluster name for non-replicated case. + return kNoReplicationCluster; +} + +bool HasDataDependencyWithUnscheduledOp( + Operation& op, Block* block, SmallPtrSet& unscheduled_ops) { + WalkResult ready_to_schedule = op.walk([&](Operation* nested_op) { + for (Value operand : nested_op->getOperands()) { + Operation* defining_op = operand.getDefiningOp(); + if (!defining_op) continue; + Operation* producer_in_block = block->findAncestorOpInBlock(*defining_op); + if (producer_in_block && producer_in_block != &op && + unscheduled_ops.count(producer_in_block)) { + // Found an operand that isn't scheduled yet, interrupt the walk. + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + return ready_to_schedule.wasInterrupted(); +} + +bool HasControlDependencyWithUnscheduledOp( + Operation& op, Block* block, SmallPtrSet& unscheduled_ops) { + IslandOp island_op = dyn_cast(op); + if (!island_op) { + return false; + } + for (Value input : island_op.getControlInputs()) { + Operation* defining_op = input.getDefiningOp(); + if (!defining_op) continue; + Operation* producer_in_block = block->findAncestorOpInBlock(*defining_op); + if (producer_in_block && producer_in_block != &op && + unscheduled_ops.count(producer_in_block)) { + // Found an operand that isn't scheduled yet, return true. + return true; + } + } + return false; +} + +// Sorts the operations in the provided range to enforce dominance. +// This is useful after fusing / reorganizing Operations in a block and later +// needing to readjust the ordering to ensure dominance. +LogicalResult SortTopologically(Block::iterator begin, Block::iterator end) { + Block* block = begin->getBlock(); + // Either sort from `begin` to end of block or both `begin` and + // `end` should belong to the same block. + assert(end == block->end() || + end->getBlock() == block && "ops must be in the same block"); + + // Track the ops that still need to be scheduled in a set. + SmallPtrSet unscheduled_ops; + for (Operation& op : llvm::make_range(begin, end)) + unscheduled_ops.insert(&op); + + Block::iterator last_scheduled_op = begin; + while (!unscheduled_ops.empty()) { + bool scheduled_at_least_once = false; + // Loop over the ops that are not sorted yet, try to find the ones "ready", + // i.e. the ones for which there aren't any operand produced by an op in the + // set, and "schedule" it (move it before the last_scheduled_op). + for (Operation& op : llvm::make_range(last_scheduled_op, end)) { + if (HasDataDependencyWithUnscheduledOp(op, block, unscheduled_ops) || + HasControlDependencyWithUnscheduledOp(op, block, unscheduled_ops)) { + continue; + } + unscheduled_ops.erase(&op); + if (Block::iterator(op) != last_scheduled_op) + op.moveBefore(block, last_scheduled_op); + else + ++last_scheduled_op; + scheduled_at_least_once = true; + } + if (!scheduled_at_least_once) return failure(); + } + return success(); +} + +// Looks for an IslandOp that wraps a single operation tagged with the +// _replication_info attribute, and merges it with all the following operations +// in the block. Sets the `changed` boolean to true if any island is merged. +// Returns a failure if a cycle prevents the merge from happening correctly +// without breaking dominance. The IR is left in invalid state in case of +// failure. +void CollectCandidateIslands( + llvm::function_ref + is_op_calling_func_for_cluster, + Operation* op, StringRef cluster_name, + SmallPtrSet& islands_set, + SmallPtrSet& wrapped_ops) { + for (Operation& candidate_op : llvm::make_early_inc_range( + llvm::make_range(op->getIterator(), op->getBlock()->end()))) { + IslandOp candidate_island = dyn_cast(candidate_op); + if (!candidate_island || !candidate_island.WrapsSingleOp()) continue; + // Check if we have an operation with the expected attribute. + Operation& candidate_wrapped_op = candidate_island.GetBody().front(); + + // The op might be a special TPU input/output op and may have been already + // added to the list of islands to be merged. + if (wrapped_ops.contains(&candidate_wrapped_op)) { + continue; + } + + std::optional result = + GetTpuClusterName(&candidate_wrapped_op); + llvm::StringRef candidate_cluster_name; + if (result.has_value()) { + candidate_cluster_name = result.value(); + } else if (is_op_calling_func_for_cluster(cluster_name, + &candidate_wrapped_op)) { + candidate_cluster_name = cluster_name; + } + if (candidate_cluster_name != cluster_name) continue; + + // Add the current op to the set of ops which are planned to be merged into + // one cluster. + islands_set.insert(candidate_island); + wrapped_ops.insert(&candidate_wrapped_op); + } +} + +IslandOp CreateMergedIsland(IslandOp island, SmallVector& islands, + SmallPtrSet& wrapped_ops) { + // Compute the result of the merged island, these are the values produced by + // the islands that are merged if they have a use in an island not merged, + // i.e. a value that escapes. + llvm::SmallVector result_types; + for (IslandOp new_op : islands) { + for (Value result : new_op.getOutputs()) { + if (llvm::any_of(result.getUsers(), [&](OpOperand user) { + return !wrapped_ops.count(user.getOwner()); + })) + result_types.push_back(result.getType()); + } + } + + OpBuilder builder(island); + IslandOp new_island = + IslandOp::create(builder, island.getLoc(), result_types, + /*control=*/ControlType::get(island.getContext()), + /*controlInputs=*/island.getOperands()); + new_island.getBody().push_back(new Block); + + // Move the operations in the new island, gather the results of the new yield. + Block& island_body = new_island.GetBody(); + SmallVector yield_operands; + for (IslandOp island : islands) { + Operation& wrapped_op = island.GetBody().front(); + wrapped_op.moveBefore(&island_body, island_body.end()); + + // For every result of the wrapped_op, it needs to get passed to the yield + // operation, only if it escapes the island. + for (auto result : + llvm::zip(island.getOutputs(), wrapped_op.getResults())) { + if (llvm::any_of(std::get<0>(result).getUsers(), [&](OpOperand user) { + return !wrapped_ops.count(user.getOwner()); + })) + yield_operands.push_back(std::get<1>(result)); + } + } + builder.setInsertionPointToEnd(&island_body); + YieldOp::create(builder, new_island.getLoc(), yield_operands); + + // remap results of the new islands to the user outside of the island. + int current_result = 0; + Value control = new_island.getControl(); + for (IslandOp island : islands) { + YieldOp yield_op = island.GetYield(); + for (const auto& idx_result : llvm::enumerate(island.getOutputs())) { + Value result = idx_result.value(); + + bool has_external_use = false; + for (OpOperand& use : llvm::make_early_inc_range(result.getUses())) { + if (wrapped_ops.count(use.getOwner())) + use.set(yield_op.getOperand(idx_result.index())); + else + has_external_use = true; + } + if (has_external_use) { + result.replaceAllUsesWith(new_island.getResult(current_result)); + ++current_result; + } + } + island.getControl().replaceAllUsesWith(control); + island.erase(); + } + return new_island; +} + +// Looks for an IslandOp that wraps a single operation tagged with the +// _replication_info attribute, and merges it with all the following operations +// in the block. Sets the `changed` boolean to true if any island is merged. +// Returns a failure if a cycle prevents the merge from happening correctly +// without breaking dominance. The IR is left in invalid state in case of +// failure. +LogicalResult MergeIsland( + llvm::function_ref + is_op_calling_func_for_cluster, + llvm::SmallDenseMap>& + cluster_to_tpu_ops_map, + Operation* op, bool* changed) { + // Find the first island wrapping a single operation with the + // `_replication_info` attribute, it'll be used as the root of the algorithm + // to find the other operations that are part of the same cluster. + IslandOp island = dyn_cast(*op); + if (!island || !island.WrapsSingleOp()) return success(); + Operation& wrapped_op = island.GetBody().front(); + + std::optional result = GetTpuClusterName(&wrapped_op); + if (!result.has_value()) return success(); + llvm::StringRef cluster_name = result.value(); + + // We found a _replication_info, let's build an island for the full cluster! + LLVM_DEBUG(llvm::dbgs() << "Processing candidate island: " + << *island.getOperation() << "\n"); + + // Collect the islands to merge together in this new cluster starting with the + // given island. + SmallVector islands; + SmallPtrSet islands_set; + SmallPtrSet wrapped_ops; + + CollectCandidateIslands(is_op_calling_func_for_cluster, op, cluster_name, + islands_set, wrapped_ops); + + if (cluster_to_tpu_ops_map.count(cluster_name)) { + for (auto tpu_op : cluster_to_tpu_ops_map[cluster_name]) { + islands_set.insert(tpu_op); + wrapped_ops.insert(&dyn_cast(*tpu_op).GetBody().front()); + } + } + + // Get the sequential order of the candidate islands in the block. + // Later we can merge the candidate islands in this order. + // The dominance is guaranteed in this order. + for (Operation& candidate_op : llvm::make_early_inc_range( + llvm::make_range(op->getBlock()->begin(), op->getBlock()->end()))) { + IslandOp candidate_island = dyn_cast(candidate_op); + if (!candidate_island || !candidate_island.WrapsSingleOp()) continue; + if (islands_set.contains(candidate_island)) { + islands.push_back(candidate_island); + } + } + + // If no other island was found to merge with the existing one, just move on. + if (islands.size() <= 1) return success(); + + *changed = true; + Operation* first_op_after = islands.back()->getNextNode(); + + // We create the merged island at the location of the first island that was + // merged (excluding special TPU input/output ops). + IslandOp new_island = CreateMergedIsland(island, islands, wrapped_ops); + + // Ensure dominance by sorting the range of islands that were merged. + return SortTopologically(Block::iterator(new_island.getOperation()), + Block::iterator(first_op_after)); +} + +// Returns all functions that can be reached from TPUPartitionedCall ops. +SmallPtrSet FindTPUPartitionedCallReachableFunctions( + ModuleOp module) { + SymbolTableCollection table; + SymbolUserMap symbol_map(table, module); + llvm::DenseMap> caller_callee_map; + // Creates work queue for determining reachability below. + std::queue function_worklist; + + for (auto func : module.getOps()) { + for (auto user : symbol_map.getUsers(func)) { + // Populates work queue with func ops called from TPUPartionedCall. + if (llvm::isa(user)) { + function_worklist.push(func); + } + // Populates caller to called func map. + if (func::FuncOp caller = user->getParentOfType()) { + caller_callee_map[caller].insert(func); + } + } + } + + // Determines reached ops starting from TPUPartionedCall ops + // and iteratively descending through called ops. + SmallPtrSet reachable_functions; + while (!function_worklist.empty()) { + func::FuncOp caller = function_worklist.front(); + function_worklist.pop(); + if (reachable_functions.insert(caller).second) { + for (auto callee : caller_callee_map[caller]) { + function_worklist.push(callee); + } + } + } + return reachable_functions; +} + +// valid means all the ops in the vector are belong to the same cluster. +bool is_valid_special_tpu_op( + std::vector& ops, llvm::StringRef cluster_name, + llvm::SmallDenseMap>& + cluster_to_tpu_op_map) { + for (IslandOp op : ops) { + Operation* wrapped_op = &op.GetBody().front(); + std::optional wrapped_op_cluster_name = + GetTpuClusterName(wrapped_op); + + bool op_has_inconsistent_cluster_name = + wrapped_op_cluster_name.has_value() && + wrapped_op_cluster_name.value() != cluster_name; + + if (op_has_inconsistent_cluster_name) { + return false; + } + } + return true; +} + +void collect_input_defining_islands(IslandOp op, std::vector& ops) { + Operation* wrapped_op = &op.GetBody().front(); + for (Value operand : wrapped_op->getOperands()) { + IslandOp wrapper = dyn_cast_or_null(operand.getDefiningOp()); + if (!wrapper || !wrapper.WrapsSingleOp()) continue; + ops.push_back(wrapper); + } +} + +void collect_output_users_islands(IslandOp op, std::vector& ops) { + for (Value result : op->getResults()) { + for (OpOperand use : result.getUsers()) { + IslandOp wrapper = + dyn_cast_or_null(use.getOwner()->getParentOp()); + if (!wrapper || !wrapper.WrapsSingleOp()) continue; + ops.push_back(wrapper); + } + } +} + +bool AddSpecialTpuOps( + IslandOp candidate_island, llvm::StringRef cluster_name, + llvm::SmallDenseMap>& + cluster_to_tpu_op_map, + SmallPtrSetImpl& visited_wrapped_ops, bool incoming) { + std::queue op_worklist; + std::vector ops; + + op_worklist.push(candidate_island); + + while (!op_worklist.empty()) { + IslandOp current_op = op_worklist.front(); + op_worklist.pop(); + ops.clear(); + if (incoming) { + collect_input_defining_islands(current_op, ops); + } else { + collect_output_users_islands(current_op, ops); + } + for (IslandOp wrapper : ops) { + Operation* wrapped_op = &wrapper.GetBody().front(); + std::vector child_ops; + if (incoming) { + // Looks at captured operands of `candidate_wrapped_op` to bring special + // TPU ops such as tf.TPUReplicatedInput and tf.TPUPartitionedInput into + // the island as well. These ops are brought in only if they do not + // already have a cluster assigned to them (via `_replication_info` + // attribute value). + // `tf.Identity` op is also treated as special tpu ops since it can play + // a role as connection between `tf.TPUReplicatedInput` or + // `tf.TPUPartitionedInput`. For example, we have the follow pseudocode: + // %0 = tf_executor.island wraps "tf.OpA" (){_replication_info = 'c'} + // %1 = tf_executor.island wraps "tf.Identity(%0) + // %2 = tf_executor.island wraps "tf.TPUReplicatedInput"(%1) + + if (!isa(wrapped_op)) + continue; + collect_output_users_islands(wrapper, child_ops); + } else { + // Looks at the results of `candidate_island` to bring special TPU + // ops such as tf.TPUReplicatedOutput and tf.TPUPartitionedOutput into + // the island as well. These ops are brought in only if they do not + // already have cluster (`_tpu_replicate` attribute) assigned to them. + // `tf.Identity` op is also treated as special tpu ops since it can play + // a role as connection between `tf.TPUReplicatedOutput` or + // `tf.TPUPartitionedInput`. + if (!isa(wrapped_op)) + continue; + collect_input_defining_islands(wrapper, child_ops); + } + if (!is_valid_special_tpu_op(child_ops, cluster_name, + cluster_to_tpu_op_map)) { + return false; + } + + // Only inputs/outputs that do not have a cluster name assigned are + // considered for special handling. Otherwise, island coarsening logic + // should be able to handle it. + if (wrapped_op->hasAttrOfType(TF::kReplicationInfoAttr)) + continue; + if (visited_wrapped_ops.contains(wrapped_op)) continue; + op_worklist.push(wrapper); + cluster_to_tpu_op_map[cluster_name].insert(wrapper); + visited_wrapped_ops.insert(wrapped_op); + } + } + return true; +} + +LogicalResult CollectSpecialTpuOps( + llvm::function_ref + is_op_calling_func_for_cluster, + Operation* op, + llvm::SmallDenseMap>& + cluster_to_tpu_op_map, + SmallPtrSet& visited_wrapped_ops) { + IslandOp island = dyn_cast(*op); + if (!island || !island.WrapsSingleOp()) return success(); + Operation& wrapped_op = island.GetBody().front(); + + if (visited_wrapped_ops.contains(&wrapped_op)) return success(); + + std::optional result = GetTpuClusterName(&wrapped_op); + if (!result.has_value()) return success(); + llvm::StringRef cluster_name = result.value(); + + visited_wrapped_ops.insert(&wrapped_op); + + if (!AddSpecialTpuOps(island, cluster_name, cluster_to_tpu_op_map, + visited_wrapped_ops, /*incoming=*/true)) { + return failure(); + } + if (!AddSpecialTpuOps(island, cluster_name, cluster_to_tpu_op_map, + visited_wrapped_ops, /*incoming=*/false)) { + return failure(); + } + return success(); +} + +// Whenever we find an Identity op that is unqualified, we remove this Identity +// op from the list `tpu_ops`. An unqualified Identity op indicates either its +// inputs or its outputs do not belong to the same cluster. +bool ExcludeIdentityOp(llvm::SmallDenseSet& tpu_ops, + llvm::StringRef& target_cluster_name, bool incoming) { + for (auto iter = tpu_ops.begin(); iter != tpu_ops.end(); iter++) { + auto island_op = llvm::dyn_cast(*iter); + if (llvm::dyn_cast_or_null(island_op.GetBody().front())) { + if (island_op.getOutputs().use_empty()) { + tpu_ops.erase(iter); + return true; + } + std::vector ops; + if (incoming) { + collect_output_users_islands(island_op, ops); + } else { + collect_input_defining_islands(island_op, ops); + } + for (IslandOp wrapper : ops) { + Operation* wrapped_op = &wrapper.GetBody().front(); + auto cluster_name = GetTpuClusterName(wrapped_op); + if (cluster_name.has_value() && + cluster_name.value() != target_cluster_name) { + tpu_ops.erase(iter); + return true; + } + if (!cluster_name.has_value() && + !tpu_ops.count(wrapper.getOperation())) { + tpu_ops.erase(iter); + return true; + } + } + } + } + return false; +} + +void ExcludeUnqualifiedIdentityOp( + llvm::SmallDenseMap>& + cluster_to_tpu_ops_map, + bool incoming) { + for (auto& [target_cluster_name, tpu_ops] : cluster_to_tpu_ops_map) { + bool changed = true; + while (changed) { + changed = ExcludeIdentityOp(tpu_ops, target_cluster_name, incoming); + } + } +} + +void ExcludeUnqualifiedIdentityOp( + llvm::SmallDenseMap>& + cluster_to_tpu_ops_map) { + ExcludeUnqualifiedIdentityOp(cluster_to_tpu_ops_map, /*incoming=*/true); + ExcludeUnqualifiedIdentityOp(cluster_to_tpu_ops_map, /*incoming=*/false); +} + +// Erase Identity op which does not contain `_replication_info` in the merged +// island. +void EraseIdentityWithNoReplicationInfo(Block& graph_body) { + for (Operation& island_op : graph_body) { + IslandOp island = dyn_cast(island_op); + if (!island || island.WrapsSingleOp()) continue; + for (Operation& op : llvm::make_early_inc_range(island.GetBody())) { + std::optional cluster_name = GetTpuClusterName(&op); + if (cluster_name.has_value()) continue; + if (auto identity_op = llvm::dyn_cast_or_null(op)) { + auto identity_input = identity_op.getInput(); + auto output = identity_op.getOutput(); + output.replaceAllUsesWith(identity_input); + identity_op.erase(); + } + } + } +} + +void TpuV1BridgeExecutorIslandCoarsening::runOnOperation() { + SymbolTable symbol_table(getOperation()); + + // Map tpu cluster names to the functions that contain operations for this + // cluster. + DenseMap> tpu_funcs; + for (func::FuncOp func_op : getOperation().getOps()) { + func_op.walk([&](Operation* op) { + std::optional cluster_name_opt = GetTpuClusterName(op); + if (cluster_name_opt.has_value()) { + tpu_funcs[cluster_name_opt.value()].insert(func_op); + } + }); + } + + // Return true if the operation is containing a reference to a function + // containing operations for this cluster. + auto is_op_calling_func_for_cluster = [&](llvm::StringRef cluster, + Operation* op) { + auto funcs_for_cluster = tpu_funcs.find(cluster); + assert(funcs_for_cluster != tpu_funcs.end()); + assert(!funcs_for_cluster->second.empty()); + if (funcs_for_cluster->second.size() == 1) return false; + for (NamedAttribute attr : op->getAttrs()) { + auto symbol_ref = mlir::dyn_cast(attr.getValue()); + if (!symbol_ref) continue; + func::FuncOp callee = + symbol_table.lookup(symbol_ref.getValue()); + if (!callee) continue; + if (funcs_for_cluster->second.count(callee)) return true; + } + return false; + }; + + // Populates skip set with functions reachable from TPUPartionedCall ops. + const auto functions_to_skip = + FindTPUPartitionedCallReachableFunctions(getOperation()); + for (func::FuncOp func_op : getOperation().getOps()) { + if (functions_to_skip.contains(func_op)) { + OpBuilder builder(func_op); + // Mark this function as being skipped in island outlining. + func_op->setAttr(mlir::TF::kSkipIslandOutlining, + builder.getBoolAttr(true)); + continue; + } + + func_op.walk([&](GraphOp graph) { + Block& graph_body = graph.GetBody(); + llvm::SmallDenseMap> + cluster_to_tpu_ops_map; + SmallPtrSet visited_ops; + for (Operation& op : graph_body) { + if (failed(CollectSpecialTpuOps(is_op_calling_func_for_cluster, &op, + cluster_to_tpu_ops_map, visited_ops))) { + graph.emitError() + << "Collect special Tpu ops failed: " + << "Graph contains op with inconsistent cluster info\n"; + signalPassFailure(); + return WalkResult::interrupt(); + } + } + + ExcludeUnqualifiedIdentityOp(cluster_to_tpu_ops_map); + + // Iterate until fixed point on the block, as it may contain multiple + // clusters. + bool changed = true; + while (changed) { + changed = false; + for (Operation& op : graph_body) { + if (failed(MergeIsland(is_op_calling_func_for_cluster, + cluster_to_tpu_ops_map, &op, &changed))) { + graph.emitError() + << "Merging island failed: the TPU cluster likely " + << "contains a cycle with non-TPU operations or has " + "unsupported ops\n"; + signalPassFailure(); + return WalkResult::interrupt(); + } + // If islands were merged, restart scanning the block from the + // beginning as we lost track of where to continue. + if (changed) break; + } + } + + // Need to remove the redundant `Identity` ops in the same cluster. + // Redundant `Identity` op indicates that no `_replicatation_info` + // attribute is attached. + EraseIdentityWithNoReplicationInfo(graph_body); + + return WalkResult::advance(); + }); + } +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorTPUV1IslandCoarseningPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_outline_tpu_island.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_outline_tpu_island.cc new file mode 100644 index 0000000000000000000000000000000000000000..24fe407ceb244afb797e114c456cb956aa3a93a7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/executor_tpuv1_outline_tpu_island.cc @@ -0,0 +1,216 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +namespace mlir { +namespace tf_executor { + +namespace { +constexpr llvm::StringRef kNestedModule = "_tpu_v1_compat_outlined"; +constexpr llvm::StringRef kOutlinedFuncPrefix = "_tpu_v1_compat_outlined_func"; + +#define GEN_PASS_DEF_TPUBRIDGEEXECUTORISLANDOUTLININGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Extract the islands containing a TPU cluster computation into an outlined +// function in a nested module. This will allow to run the usual bridge on this +// nested module which exhibit a more friendly "V2-like" structure. +// This is only intended for V1 compatibility mode where the bridge runs without +// feed/fetches on session create/extend. +struct TPUBridgeExecutorIslandOutlining + : public impl::TPUBridgeExecutorIslandOutliningPassBase< + TPUBridgeExecutorIslandOutlining> { + void runOnOperation() override; +}; + +// Move FuncOp referenced by `symbol_ref` from one symbol table to another. +void MoveFuncOp(FlatSymbolRefAttr &symbol_ref, SymbolTable &from, + SymbolTable &to) { + if (to.lookup(symbol_ref.getValue())) return; + func::FuncOp callee = from.lookup(symbol_ref.getValue()); + callee.getOperation()->getBlock()->getOperations().remove( + callee.getOperation()); + to.insert(callee); +} + +void TPUBridgeExecutorIslandOutlining::runOnOperation() { + MLIRContext *ctx = &getContext(); + + SymbolTable symbol_table(getOperation()); + if (Operation *nested_module = symbol_table.lookup(kNestedModule)) { + nested_module->emitOpError("unexpected already present outlined module."); + return signalPassFailure(); + } + ModuleOp outlined_module = ModuleOp::create(getOperation().getLoc()); + outlined_module->setAttrs(getOperation()->getAttrDictionary()); + outlined_module->setAttr(SymbolTable::getSymbolAttrName(), + StringAttr::get(ctx, kNestedModule)); + symbol_table.insert(outlined_module); + SymbolTable outlined_symbol_table(outlined_module); + + // Find every island that contains a TPU node and extract it into a new module + // to run the V1 bridge there. + llvm::SmallVector islands_to_outline; + getOperation().walk([&](IslandOp island_op) { + auto parent_func = island_op->getParentOfType(); + auto skip_island_outlining = + parent_func->getAttrOfType(mlir::TF::kSkipIslandOutlining); + if (skip_island_outlining && skip_island_outlining.getValue()) { + // Island was marked to be skipped. + return WalkResult::advance(); + } + for (Operation &op : island_op.GetBody().without_terminator()) { + if (isa(&op)) { + // Handle replicated TPU case. + islands_to_outline.push_back(island_op); + break; + } + auto device_type = + op.getAttrOfType(TF::kCompileDeviceTypeAttr); + if (device_type && device_type.getValue() == TF::kTpuDevice && + !op.hasAttrOfType(TF::kReplicationInfoAttr)) { + // Handle single-core TPU case (no `TPUReplicateMetadataOp`). + islands_to_outline.push_back(island_op); + break; + } + } + return WalkResult::advance(); + }); + int prefix_id = 0; + for (IslandOp island_op : islands_to_outline) { + // Build the function signature. + + // First the captured values in the island are function arguments + llvm::SetVector operands; + getUsedValuesDefinedAbove(island_op.getBody(), operands); + + SmallVector func_operand_types; + func_operand_types.reserve(operands.size()); + for (Value operand : operands) + func_operand_types.push_back(operand.getType()); + + // Function results are the yield operands + SmallVector func_result_types; + for (Value operand : island_op.GetYield().getOperands()) + func_result_types.push_back(operand.getType()); + FunctionType func_type = + FunctionType::get(ctx, func_operand_types, func_result_types); + + // Create the outlined function + SmallString<32> name = kOutlinedFuncPrefix; + name += llvm::Twine(prefix_id++).str(); + auto builder = OpBuilder(ctx); + auto outlined_func = + func::FuncOp::create(builder, island_op.getLoc(), name, func_type); + outlined_symbol_table.insert(outlined_func); + outlined_func.setNested(); + + // We will "steal" the body of the island and replace it with a call to the + // new function later. + { + YieldOp yield_op = island_op.GetYield(); + outlined_func.getBody().takeBody(island_op.getBody()); + + // Replace the yield with a return + OpBuilder replacer(yield_op); + island_op.getBody().push_back(new Block); + mlir::func::ReturnOp::create(replacer, yield_op.getLoc(), + yield_op.getOperands()); + yield_op.erase(); + } + + // Remap the captured operands in the (former) island block with newly + // created entry block arguments in the function body. + { + Block &entry_block = outlined_func.getBody().front(); + auto loc = outlined_func.getLoc(); + for (Value operand : operands) { + BlockArgument newArg = entry_block.addArgument(operand.getType(), loc); + replaceAllUsesInRegionWith(operand, newArg, outlined_func.getBody()); + } + } + + // The function is in place in the nested module, create a call and yield in + // the original island. + builder.setInsertionPointToEnd(&island_op.GetBody()); + auto call_op = mlir::TF::PartitionedCallOp::create( + builder, island_op.getLoc(), func_result_types, operands.getArrayRef(), + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, + SymbolRefAttr::get( + builder.getContext(), kNestedModule, + SymbolRefAttr::get(builder.getContext(), outlined_func.getName())), + /*config=*/builder.getStringAttr(""), + /*config_proto=*/builder.getStringAttr(""), + /*executor_type=*/builder.getStringAttr("")); + SmallVector yield_operands(call_op.getResults()); + YieldOp::create(builder, island_op.getLoc(), yield_operands); + } + + // Outline all the transitively called functions by moving them in the + // outlined module. + for (func::FuncOp func : outlined_module.getOps()) { + func.walk([&](Operation *op) { + for (NamedAttribute attr : op->getAttrs()) { + if (auto symbol_ref = + mlir::dyn_cast(attr.getValue())) { + MoveFuncOp(symbol_ref, symbol_table, outlined_symbol_table); + continue; + } + if (auto array_attr = mlir::dyn_cast(attr.getValue())) { + for (const Attribute &attribute : array_attr) { + auto symbol_ref = mlir::dyn_cast(attribute); + if (!symbol_ref) continue; + MoveFuncOp(symbol_ref, symbol_table, outlined_symbol_table); + } + } + } + }); + } + // Remove `kSkipIslandOutlining` attributes. + for (func::FuncOp func_op : getOperation().getOps()) { + if (func_op->hasAttr(mlir::TF::kSkipIslandOutlining)) { + func_op->removeAttr(mlir::TF::kSkipIslandOutlining); + } + } +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorTPUV1IslandOutliningPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/extract_tpu_copy_with_dynamic_shape_op.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/extract_tpu_copy_with_dynamic_shape_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..95e2e127e5852af7790df717598ef6ef83756758 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/extract_tpu_copy_with_dynamic_shape_op.cc @@ -0,0 +1,237 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +#define DEBUG_TYPE "tf-extract-tpu-copy-with-dynamic-shape-op" + +namespace mlir { +namespace TFTPU { + +namespace { + +#define GEN_PASS_DEF_EXTRACTTPUCOPYWITHDYNAMICSHAPEOPPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class ExtractTPUCopyWithDynamicShapeOpPass + : public impl::ExtractTPUCopyWithDynamicShapeOpPassBase< + ExtractTPUCopyWithDynamicShapeOpPass> { + void runOnOperation() override; +}; + +// Finds op that created a given value. If the value is a BlockArgument, this +// returns the owner of the Block. +Operation* GetOpOfValue(Value value) { + if (auto block_arg = mlir::dyn_cast(value)) + return block_arg.getOwner()->getParentOp(); + + return value.getDefiningOp(); +} + +// Check if the TPUCopyWithDynamicShapeOp is valid. +// 1. The op should be wrapped inside a launch op. +// 2. The wrapped launch op should be placed on CPU. +LogicalResult CheckOpIsValid(Operation* op) { + auto launch_op = llvm::dyn_cast(op->getParentOp()); + if (!launch_op) { + op->emitError() << "TPUCopyWithDynamicShapeOp is not in a launch"; + return failure(); + } + std::string device_str = launch_op.getDeviceAttr().getValue().str(); + std::string cpu0_device; + if (failed(tensorflow::GetNonReplicatedCPU0(op, &cpu0_device))) + return failure(); + if (device_str != tensorflow::GetDeviceAliasForHostOfLogicalCore(0) && + device_str != cpu0_device) { + op->emitError() + << "TPUCopyWithDynamicShapeOp's device is not a recognized host 0: " + << device_str; + return failure(); + } + return success(); +} + +// Check if we can move TPUCopyWithDynamicShapeOp out of a launch. This is the +// case if its results aren't used by other ops except for the return op. +bool CanMove(Operation* op) { + auto launch_op = llvm::dyn_cast(op->getParentOp()); + if (!launch_op) return false; + for (Value result : op->getResults()) { + for (Operation* user : result.getUsers()) { + if (user != launch_op.GetBody().getTerminator()) return false; + } + } + return true; +} + +// Get the new launch op results. This is the results if the copy op is removed +// from the old launch op. +llvm::SmallVector CreateNewLaunchOpResults( + tf_device::LaunchOp* old_launch_op, + Operation* tpu_copy_with_dynamic_shape_op) { + llvm::SmallSetVector new_launch_op_results; + + new_launch_op_results.insert( + old_launch_op->GetBody().getTerminator()->getOperands().begin(), + old_launch_op->GetBody().getTerminator()->getOperands().end()); + + for (Value operand : tpu_copy_with_dynamic_shape_op->getOperands()) { + if (GetOpOfValue(operand)->getParentRegion() == + tpu_copy_with_dynamic_shape_op->getParentRegion()) { + new_launch_op_results.insert(operand); + } + } + + for (Value result : tpu_copy_with_dynamic_shape_op->getResults()) { + new_launch_op_results.remove(result); + } + + return new_launch_op_results.takeVector(); +} + +// Create a new host launch op which contains all the old launch op body +// except the dynamic shape copy op. +tf_device::LaunchOp CreateNewHostLaunchOpWithNewResult( + tf_device::LaunchOp* old_launch_op, + llvm::SmallVector& new_launch_op_results) { + OpBuilder builder(*old_launch_op); + + builder.setInsertionPointAfter(*old_launch_op); + + llvm::SmallVector new_launch_op_results_types; + for (Value result : new_launch_op_results) + new_launch_op_results_types.push_back(result.getType()); + + auto new_launch_op = tf_device::LaunchOp::create( + builder, old_launch_op->getLoc(), old_launch_op->getDeviceAttr(), + /*result_types=*/new_launch_op_results_types); + + new_launch_op.getBody().takeBody(old_launch_op->getBody()); + new_launch_op.GetBody().getTerminator()->setOperands(new_launch_op_results); + + return new_launch_op; +} + +// Create the new device launch op which wraps the copy op. +LogicalResult CreateNewDeviceLaunchOp( + Operation* tpu_copy_with_dynamic_shape_op, bool replicated, + tf_device::LaunchOp& new_device_launch_op) { + OpBuilder builder(tpu_copy_with_dynamic_shape_op); + + builder.setInsertionPointAfter(tpu_copy_with_dynamic_shape_op); + + // Set the copy op's device to the first TPU. + std::string device_str; + if (replicated) { + device_str = tensorflow::GetDeviceAliasForLogicalCore(0); + } else if (failed(tensorflow::GetNonReplicatedTPU0( + tpu_copy_with_dynamic_shape_op, &device_str))) { + return failure(); + } + + new_device_launch_op = tf_device::LaunchOp::create( + builder, tpu_copy_with_dynamic_shape_op->getLoc(), + builder.getStringAttr(device_str), + /*result_types=*/tpu_copy_with_dynamic_shape_op->getResultTypes()); + + new_device_launch_op.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&new_device_launch_op.GetBody()); + auto* return_op = tf_device::ReturnOp::create( + builder, tpu_copy_with_dynamic_shape_op->getLoc(), + tpu_copy_with_dynamic_shape_op->getResults()) + .getOperation(); + tpu_copy_with_dynamic_shape_op->moveBefore(return_op); + return success(); +} + +// Update all the usage of tf_device.return op with launch op result. +void UpdateReturnOpResultWithLaunchOpResult(tf_device::LaunchOp* launch_op) { + auto operand_not_in_launch = [&](OpOperand& operand) { + return !launch_op->getOperation()->isProperAncestor(operand.getOwner()); + }; + + for (auto result : + llvm::zip(launch_op->getResults(), + launch_op->GetBody().getTerminator()->getOperands())) + std::get<1>(result).replaceUsesWithIf(std::get<0>(result), + operand_not_in_launch); +} + +void ExtractTPUCopyWithDynamicShapeOpPass::runOnOperation() { + llvm::SmallVector tpu_copy_with_dynamic_shape_ops; + auto walk_result = getOperation().walk([&](Operation* op) { + if (isa(op)) { + if (failed(CheckOpIsValid(op))) return WalkResult::interrupt(); + if (CanMove(op)) { + tpu_copy_with_dynamic_shape_ops.push_back(op); + } + } + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return signalPassFailure(); + + for (Operation* op : tpu_copy_with_dynamic_shape_ops) { + OpBuilder builder(op); + + auto old_launch_op = llvm::dyn_cast(op->getParentOp()); + + bool replicated = old_launch_op.getDeviceAttr().getValue().str() == + tensorflow::GetDeviceAliasForHostOfLogicalCore(0); + + for (auto result : + llvm::zip(old_launch_op->getResults(), + old_launch_op.GetBody().getTerminator()->getOperands())) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + llvm::SmallVector new_launch_op_results = + CreateNewLaunchOpResults(&old_launch_op, op); + + op->moveAfter(old_launch_op); + + auto new_host_launch_op = CreateNewHostLaunchOpWithNewResult( + &old_launch_op, new_launch_op_results); + UpdateReturnOpResultWithLaunchOpResult(&new_host_launch_op); + + old_launch_op->erase(); + + tf_device::LaunchOp new_device_launch_op; + if (failed(CreateNewDeviceLaunchOp(op, replicated, new_device_launch_op))) + return signalPassFailure(); + UpdateReturnOpResultWithLaunchOpResult(&new_device_launch_op); + } +} + +} // namespace + +std::unique_ptr> +CreateExtractTPUCopyWithDynamicShapeOpPass() { + return std::make_unique(); +} +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fold_broadcast.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fold_broadcast.cc new file mode 100644 index 0000000000000000000000000000000000000000..ee99df58cf8c7b8b21cad18c27aae1db78a19f3f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fold_broadcast.cc @@ -0,0 +1,211 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Traits.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace { + +class ConvertResultsBroadcastableShapeOp : public RewritePattern { + public: + ConvertResultsBroadcastableShapeOp(MLIRContext* context) + : RewritePattern(MatchAnyOpTypeTag(), 1, context) {} + + LogicalResult matchAndRewrite(Operation* op, + PatternRewriter& rewriter) const override; + + private: + template + LogicalResult RewriteEqOp(Operation* op, PatternRewriter& rewriter) const; + + LogicalResult RewriteOp( + Operation* op, PatternRewriter& rewriter, + const std::function, ArrayRef, + SmallVectorImpl&)>& + get_broadcasted_shape) const; + + LogicalResult RewriteBatchMatMulV2Op(Operation* op, + PatternRewriter& rewriter) const; +}; + +#define GEN_PASS_DEF_BROADCASTFOLDPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class BroadcastFoldPass + : public impl::BroadcastFoldPassBase { + public: + void runOnOperation() override; +}; + +LogicalResult ConvertResultsBroadcastableShapeOp::matchAndRewrite( + Operation* op, PatternRewriter& rewriter) const { + if (op->hasTrait()) + return RewriteOp(op, rewriter, OpTrait::util::getBroadcastedShape); + + // tf.Equal and tf.NotEqual ops only satisfy ResultsBroadcastableShape when + // incompatible_shape_error is `true` (what is also checked by the verifier). + if (succeeded(RewriteEqOp(op, rewriter))) return success(); + if (succeeded(RewriteEqOp(op, rewriter))) return success(); + if (succeeded(RewriteBatchMatMulV2Op(op, rewriter))) return success(); + + return failure(); +} + +LogicalResult ConvertResultsBroadcastableShapeOp::RewriteBatchMatMulV2Op( + Operation* op, PatternRewriter& rewriter) const { + auto matmul_op = llvm::dyn_cast(op); + if (!matmul_op) return failure(); + + // Gets the broadcasted output shape for tf.BatchMatMulV2Op. `shape_x` is the + // shape of op's first/left-hand-side operand and `shape_y` is the shape of + // op's second/right-hand-side operand. + const auto get_broadcasted_shape = + [&](ArrayRef shape_x, ArrayRef shape_y, + SmallVectorImpl& result_shape) { + if (shape_x.size() < 2 || shape_y.size() < 2) { + return false; + } + + // Checks outer dimensions (i.e., the dimensions higher than 2D) are + // broadcastable. If true, then get the broadcasted shape for outer + // dimension. + if (!OpTrait::util::getBroadcastedShape( + shape_x.drop_back(2), shape_y.drop_back(2), result_shape)) { + return false; + } + + const int x_row = + matmul_op.getAdjX() ? shape_x.back() : *(shape_x.rbegin() + 1); + const int x_col = + !matmul_op.getAdjX() ? shape_x.back() : *(shape_x.rbegin() + 1); + + const int y_row = + matmul_op.getAdjY() ? shape_y.back() : *(shape_y.rbegin() + 1); + const int y_col = + !matmul_op.getAdjY() ? shape_y.back() : *(shape_y.rbegin() + 1); + + // Checks that matrix multiply can perform a valid contraction. + if (x_col != y_row) { + result_shape.clear(); + return false; + } + + result_shape.push_back(x_row); + result_shape.push_back(y_col); + return true; + }; + + return RewriteOp(op, rewriter, get_broadcasted_shape); +} + +template +LogicalResult ConvertResultsBroadcastableShapeOp::RewriteEqOp( + Operation* op, PatternRewriter& rewriter) const { + auto eq_op = llvm::dyn_cast_or_null(op); + if (eq_op && eq_op.getIncompatibleShapeError()) + return RewriteOp(op, rewriter, OpTrait::util::getBroadcastedShape); + return failure(); +} + +LogicalResult ConvertResultsBroadcastableShapeOp::RewriteOp( + Operation* op, PatternRewriter& rewriter, + const std::function, ArrayRef, + SmallVectorImpl&)>& get_broadcasted_shape) + const { + if (op->getNumOperands() != 2 || op->getResultTypes().size() != 1) + return failure(); + + // Check that the result shape is fully defined. + auto result_type = + mlir::dyn_cast_or_null(op->getResultTypes().front()); + if (!result_type || !result_type.hasStaticShape()) return failure(); + + bool changed = false; + for (uint64_t i = 0, e = op->getNumOperands(); i < e; ++i) { + // Check that the i'th operand is a broadcast. + auto broadcast = llvm::dyn_cast_or_null( + op->getOpOperand(i).get().getDefiningOp()); + if (!broadcast) continue; + + // Check that the operand of the broadcast has fully defined shape. + auto broadcast_arg_type = mlir::dyn_cast_or_null( + broadcast.getInput().getType()); + if (!broadcast_arg_type || !broadcast_arg_type.hasStaticShape()) continue; + + // Check that the other argument has fully defined shape. + auto argument_type = mlir::dyn_cast_or_null( + op->getOpOperand(1 - i).get().getType()); + if (!argument_type || !argument_type.hasStaticShape()) continue; + + // Get the unbroadcasted shapes in the operand order. + std::array, 2> operand_shapes; + operand_shapes[i] = broadcast_arg_type.getShape(); + operand_shapes[1 - i] = argument_type.getShape(); + + // Check that the input of the broadcast and the other operand is broadcast + // compatible. + llvm::SmallVector broadcasted_shape; + if (!get_broadcasted_shape(operand_shapes[0], operand_shapes[1], + broadcasted_shape)) + continue; + + // Check that an implicit broadcast between the operand of the broadcast and + // the other argument would result in the same type as the result type. + if (broadcasted_shape != result_type.getShape()) continue; + + // Update the operand of the op to be the operand of the broadcast. + rewriter.modifyOpInPlace( + op, [&]() { op->getOpOperand(i).set(broadcast.getInput()); }); + changed = true; + } + return success(changed); +} + +void BroadcastFoldPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + auto func = getOperation(); + + patterns.add(func.getContext()); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +namespace TF { +std::unique_ptr> CreateBroadcastFoldPass() { + return std::make_unique(); +} +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_global_tensors.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_global_tensors.cc new file mode 100644 index 0000000000000000000000000000000000000000..1df8781c60ff46b2955cf22b5fcbf3728a0592a9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_global_tensors.cc @@ -0,0 +1,238 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlowFramework.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_dataflow.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h" + +#define DEBUG_TYPE "freeze-global-tensor" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +#define GEN_PASS_DEF_FREEZEGLOBALTENSORSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" +struct FreezeGlobalTensorsPass + : public impl::FreezeGlobalTensorsPassBase { + explicit FreezeGlobalTensorsPass(bool allow_mutable_tensors) { + this->allow_mutable_tensors = allow_mutable_tensors; + } + void runOnOperation() override; +}; + +void FreezeGlobalTensorsPass::runOnOperation() { + auto module = getOperation(); + if (!tf_saved_model::HasTfSavedModelSemantics(module)) return; + + DataFlowSolver solver; + solver.load(); + solver.load(); + TF::LoadResourceDataflowAnalysis(solver); + if (failed(solver.initializeAndRun(module))) return signalPassFailure(); + + DenseSet remaining_global_tensor_ops; + { + auto ops = module.getOps(); + remaining_global_tensor_ops.insert(ops.begin(), ops.end()); + } + + for (auto global_tensor : remaining_global_tensor_ops) { + // This pass assumes that all global tensors as immutable (e.g. by a + // previous optimize global tensors pass). If not, this pass has to fail + // since it cannot perform one of its goals. + if (global_tensor.getIsMutable()) { + if (allow_mutable_tensors) continue; + global_tensor.emitError() + << "is not immutable, try removing mutable variables in your model " + "since mutable variables are currently not supported through " + "this converter"; + return signalPassFailure(); + } + } + + // Collect all those freezable. This is an extra scan but allows for the + // partial behavior from `allow_mutable_tensor`. + DenseMap freezeable; + DenseSet frozen_global_tensors; + for (auto func : module.getOps()) { + if (!mlir::tf_saved_model::IsExported(func)) continue; + + for (BlockArgument val : func.getArguments()) { + if (!mlir::isa(getElementTypeOrSelf(val.getType()))) + continue; + + // Check that there is only a single global tensor associated with arg. + const TF::ResourceDataflowState *latticeElement = + solver.lookupState(val); + if (!latticeElement || latticeElement->getValue().ops.size() != 1) + continue; + + // Don't freeze mutable tensors. + Operation *op = *latticeElement->getValue().ops.begin(); + GlobalTensorOp globalTensor = llvm::dyn_cast(op); + + if (!globalTensor) + continue; // happens if the name is e.g. in a VarHandleOp. + + if (globalTensor.getIsMutable()) { + freezeable[val] = false; + continue; + } + + freezeable[val] = true; + if (freezeable[val] && globalTensor.getValue()) { + frozen_global_tensors.insert(globalTensor); + } + } + } + + // Note: We can't modify the graph while navigating through it, as erasing + // invalidate pointers. + // So instead we capture all the updates in the below map, and then + // process them after. + + // Container to hold all update actions on ops. + // Key: Operation to update. + // Value: optional list of argument indices to delete from this op. + // Note that we use MapVector because we want to iterate on the same order + // of insertion. + llvm::MapVector> + arguments_to_erase; + for (auto &it : freezeable) { + if (!it.second) continue; + + const TF::ResourceDataflowState *latticeElement = + solver.lookupState(it.first); + if (!latticeElement || latticeElement->getValue().ops.size() != 1) continue; + + // Don't freeze mutable tensors. + Operation *op = *latticeElement->getValue().ops.begin(); + GlobalTensorOp globalTensor = llvm::dyn_cast(op); + ElementsAttr elements_attr = globalTensor.getValueAttr(); + auto resource_uses = it.first.getUses(); + if (failed(mlir::tf_saved_model::ReplaceVarWithConstant( + resource_uses, elements_attr, &arguments_to_erase))) { + return signalPassFailure(); + } + } + + if (failed(mlir::tf_saved_model::EraseObsoleteResourceUses( + arguments_to_erase))) { + return signalPassFailure(); + } + + for (auto func : module.getOps()) { + if (!mlir::tf_saved_model::IsExported(func)) continue; + + llvm::BitVector args_to_erase(func.getNumArguments()); + DenseMap remove_operands; + OpBuilder builder(func.getBody()); + + for (BlockArgument val : func.getArguments()) { + if (!freezeable[val]) continue; + + const TF::ResourceDataflowState *latticeElement = + solver.lookupState(val); + Operation *op = *latticeElement->getValue().ops.begin(); + GlobalTensorOp global_tensor = llvm::dyn_cast(op); + if (!global_tensor) + continue; // happens if the name is e.g. in a VarHandleOp. + + if (!global_tensor.getValue()) + continue; // a value wasn't loaded for this tensor. + + SmallVector read_variable_ops_to_erase; + + for (Operation *user : val.getUsers()) { + if (auto read_op = llvm::dyn_cast(user)) { + // Collect all read variable ops so that all its uses can be replaced + // with the tf.constant corresponding to the global tensor op. + read_variable_ops_to_erase.push_back(read_op); + } else { + llvm::BitVector &bvector = remove_operands[user]; + bvector.resize(user->getNumOperands()); + for (OpOperand &use : user->getOpOperands()) + bvector.set(use.getOperandNumber()); + } + } + + if (!read_variable_ops_to_erase.empty()) { + // Replace the arg with a tf.Const op in the function body. + builder.setInsertionPointToStart(&func.getBody().front()); + auto const_op = TF::ConstOp::create(builder, global_tensor.getLoc(), + *global_tensor.getValue()); + for (auto read_op : read_variable_ops_to_erase) { + read_op.getResult().replaceAllUsesWith(const_op.getResult()); + read_op.erase(); + } + } + args_to_erase.set(val.getArgNumber()); + } + // As the other uses are call operations, we simply remove the arguments + // as the function arguments will be removed below once that function is + // processed. + for (auto it : remove_operands) { + it.first->eraseOperands(it.second); + } + + if (failed(func.eraseArguments(args_to_erase))) { + return signalPassFailure(); + } + } + + // Erase all global tensors that were frozen. + for (auto global_tensor : frozen_global_tensors) { + remaining_global_tensor_ops.erase(global_tensor); + global_tensor->erase(); + } + + // Verify that there are no remaining global tensors. + if (!allow_mutable_tensors && !remaining_global_tensor_ops.empty()) { + module.emitError() << "could not freeze all global tensors in the module"; + return signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> CreateFreezeGlobalTensorsPass( + bool allow_mutable_tensors) { + return std::make_unique(allow_mutable_tensors); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_saved_model_assets.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_saved_model_assets.cc new file mode 100644 index 0000000000000000000000000000000000000000..0eda4f867d498f1183cbd1623a4a07ad2cda1cfa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/freeze_saved_model_assets.cc @@ -0,0 +1,124 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/core/platform/path.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +#define GEN_PASS_DEF_FREEZEASSETSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +// This pass will replace a func's saved model asset bound inputs which are +// bound to tf.InitializeTableFromTextFileV2Op ops with tf.Const ops inside the +// func's body. +struct FreezeAssetsPass : public impl::FreezeAssetsPassBase { + FreezeAssetsPass() = default; + + FreezeAssetsPass(const FreezeAssetsPass& pass) {} + explicit FreezeAssetsPass(std::string saved_model_dir) { + this->saved_model_dir = saved_model_dir; + } + void runOnOperation() override; + + private: + std::string saved_model_dir; +}; + +void FreezeAssetsPass::runOnOperation() { + auto module = getOperation(); + if (!tf_saved_model::HasTfSavedModelSemantics(module)) { + return; + } + SymbolTable symbol_table(module); + + for (auto func : module.getOps()) { + llvm::BitVector args_to_erase(func.getNumArguments()); + OpBuilder builder(func.getBody()); + + for (int i = 0, e = func.getNumArguments(); i < e; ++i) { + SmallVector + init_table_from_text_file_ops_to_erase; + auto asset = LookupBoundInputOfType(func, i, symbol_table); + + if (!asset) continue; + + auto arg = func.getArgument(i); + bool arg_is_deletable = true; + for (auto user : arg.getUsers()) { + if (auto read_op = + llvm::dyn_cast(user)) { + init_table_from_text_file_ops_to_erase.push_back(read_op); + } else { + arg_is_deletable = false; + continue; + } + } + if (arg_is_deletable) { + args_to_erase.set(i); + } + + // Replace the arg with a tf.Const op in the function body. + builder.setInsertionPointToStart(&func.getBody().front()); + + std::string asset_filename = asset.getFilename().str(); + std::string filename = + tensorflow::io::JoinPath(saved_model_dir, asset_filename); + ShapedType shaped_type = + RankedTensorType::get({1}, TF::StringType::get(builder.getContext())); + auto const_op = TF::ConstOp::create( + builder, asset.getLoc(), + DenseStringElementsAttr::get(shaped_type, {filename})); + for (auto init_op : init_table_from_text_file_ops_to_erase) { + // Replace the InitializeTableFromTextFileV2Op to use the saved model's + // asset filepath. + builder.setInsertionPoint(init_op); + TF::InitializeTableFromTextFileV2Op::create( + builder, init_op.getLoc(), init_op.getTableHandle(), + const_op.getResult(), init_op.getKeyIndex(), + init_op.getValueIndex(), init_op.getVocabSize(), + init_op.getDelimiter()); + init_op.erase(); + } + } + + if (failed(func.eraseArguments(args_to_erase))) { + return signalPassFailure(); + } + } +} + +} // namespace + +std::unique_ptr> CreateFreezeAssetsPass( + std::string saved_model_dir) { + return std::make_unique(saved_model_dir); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc new file mode 100644 index 0000000000000000000000000000000000000000..d2c8adb11f66a9c27215d103a6676985de949680 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc @@ -0,0 +1,315 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass transforms functional control flow operations in the +// TensorFlow dialect to MLIR Control Flow Graph (CFG) form. + +#include +#include +#include + +#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_FUNCTIONALCONTROLFLOWTOCFGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct FunctionalControlFlowToCFG + : public impl::FunctionalControlFlowToCFGPassBase< + FunctionalControlFlowToCFG> { + void runOnOperation() override; +}; + +// Lowers a general tensor argument that is used as a condition to a functional +// control flow op into an i1 value. +static Value LowerCondition(Location loc, Value value, OpBuilder* builder) { + Value zero_d = ToBoolOp::create(*builder, loc, value); + Value scalar = tensor::ExtractOp::create(*builder, loc, zero_d); + return scalar; +} + +// Calls the function `fn` with arguments provided by the given function and +// return the CallOp. Arguments are cast to the required type before calling +// the function. +// +// Requires the function to provide arguments for each of the `fn` operands +// that is compatible for tensor cast. +static Operation* CallFn(Location loc, const std::function& get_arg, + func::FuncOp fn, OpBuilder* builder) { + FunctionType fn_type = fn.getFunctionType(); + llvm::SmallVector operands; + int num_operands = fn_type.getNumInputs(); + operands.reserve(num_operands); + for (int i = 0; i < num_operands; ++i) { + Value val = get_arg(i); + Type expected = fn_type.getInput(i); + if (val.getType() != expected) { + val = TF::CastOp::create(*builder, loc, expected, val, + /*Truncate=*/builder->getBoolAttr(false)); + } + operands.push_back(val); + } + return func::CallOp::create(*builder, loc, fn, operands).getOperation(); +} + +// Prepares for jump to the given block by introducing necessary tensor_cast +// operations and returning Values of types required by the block. +// +// Requires the function to provide values for each of the block arguments and +// they should be pair-wise compatible for tensor cast. +static llvm::SmallVector PrepareValsForJump( + Location loc, const std::function& get_val, Block* block, + OpBuilder* builder) { + llvm::SmallVector result; + int num_vals = block->getNumArguments(); + result.reserve(num_vals); + for (int i = 0; i < num_vals; ++i) { + Value val = get_val(i); + Type expected = block->getArgument(i).getType(); + if (val.getType() != expected) { + val = TF::CastOp::create(*builder, loc, expected, val, + /*Truncate=*/builder->getBoolAttr(false)); + } + result.push_back(val); + } + return result; +} + +// Jumps to the given block with arguments provided by the function. Arguments +// are cast to the required type before the jump. +// +// Requires the function to provide values for each of the block arguments and +// they should be pair-wise compatible for tensor cast. +static void JumpToBlock(Location loc, const std::function& get_arg, + Block* block, OpBuilder* builder) { + auto operands = PrepareValsForJump(loc, get_arg, block, builder); + cf::BranchOp::create(*builder, loc, block, operands); +} + +// Replaces all uses of the operation results in this block with block +// arguments. +// +// Requires that the block has same number of arguments as number of results of +// the operation and either they have same types or are more generic types and +// it is possible to cast them to results' types. +static void ReplaceOpResultWithBlockArgs(Location loc, Operation* op, + Block* block, OpBuilder* builder) { + assert(op->getNumResults() == block->getNumArguments()); + for (unsigned i = 0, e = op->getNumResults(); i != e; ++i) { + Value arg = block->getArgument(i); + Value result = op->getResult(i); + if (arg.getType() != result.getType()) { + arg = TF::CastOp::create(*builder, loc, result.getType(), arg, + /*Truncate=*/builder->getBoolAttr(false)); + } + result.replaceAllUsesWith(arg); + } +} + +// Given a functional IfOp, transforms the enclosing code to eliminate it +// completely from the IR, breaking it into operations to evaluate the condition +// as a bool, plus some branches. +static LogicalResult LowerIfOp(IfOp op) { + Operation* op_inst = op.getOperation(); + Location loc = op_inst->getLoc(); + + OpBuilder builder(op_inst); + + // Lower the condition to a boolean value (i1). + Value cond_i1 = LowerCondition(loc, op.getCond(), &builder); + if (!cond_i1) return failure(); + + // Split the basic block before the 'if'. The new dest will be our merge + // point. + Block* orig_block = op_inst->getBlock(); + Block* merge_block = orig_block->splitBlock(op); + + // Add the block arguments to the merge point, and replace all uses of the + // original operation results with them. + for (Value value : op_inst->getResults()) + merge_block->addArgument(value.getType(), loc); + ReplaceOpResultWithBlockArgs(loc, op_inst, merge_block, &builder); + + // Get arguments to the branches after dropping the condition which is the + // first operand. + auto get_operand = [&](int i) { return op_inst->getOperand(i + 1); }; + + // Set up the 'then' block. + Block* then_block = builder.createBlock(merge_block); + Operation* call_op = CallFn(loc, get_operand, op.then_function(), &builder); + + auto get_then_result = [&](int i) { return call_op->getResult(i); }; + JumpToBlock(loc, get_then_result, merge_block, &builder); + + // Set up the 'else' block. + Block* else_block = builder.createBlock(merge_block); + call_op = CallFn(loc, get_operand, op.else_function(), &builder); + + auto get_else_result = [&](int i) { return call_op->getResult(i); }; + JumpToBlock(loc, get_else_result, merge_block, &builder); + + // Now that we have the then and else blocks, replace the terminator of the + // orig_block with a conditional branch. + builder.setInsertionPointToEnd(orig_block); + cf::CondBranchOp::create(builder, loc, cond_i1, then_block, + llvm::ArrayRef(), else_block, + llvm::ArrayRef()); + + // Finally, delete the op in question. + op_inst->erase(); + return success(); +} + +// Given a functional WhileOp, transforms the enclosing code to eliminate it +// completely from the IR, breaking it into operations to execute the loop body +// repeatedly while the loop condition is true. +static LogicalResult LowerWhileOp(WhileOp op) { + Operation* op_inst = op.getOperation(); + Location loc = op_inst->getLoc(); + + OpBuilder builder(op_inst); + + auto cond_fn = op.cond_function(); + auto body_fn = op.body_function(); + + // Split the block containing the While op into two blocks. One containing + // operations before the While op and other containing the rest. Create two + // new blocks to call condition and body functions. + // + // The final control flow graph would be as follows: + // + // ... + // orig_block_head(...): + // ... + // br cond_block(...) + // cond_block(...): + // %A = call @cond(...) + // cond br %A, body_block(...), orig_block_tail(...) + // body_block(...): + // %B = call @body(...) + // br cond_block(...) + // orig_block_tail(...): + // ... + // + Block* orig_block_head = op_inst->getBlock(); + Block* orig_block_tail = orig_block_head->splitBlock(op); + Block* cond_block = builder.createBlock(orig_block_tail); + Block* body_block = builder.createBlock(orig_block_tail); + + // Set argument types for the cond_block to be same as the types of the + // condition function and argument types for the other two blocks to be same + // as the input types of the body function. Note that it is always possible + // for body_block and orig_block_tail to have arguments of the same types as + // they have exactly one call-site and they are sharing the operands. + for (Type type : cond_fn.getFunctionType().getInputs()) { + cond_block->addArgument(type, loc); + } + for (Type type : body_fn.getFunctionType().getInputs()) { + body_block->addArgument(type, loc); + orig_block_tail->addArgument(type, loc); + } + + auto get_operand = [&](int i) { return op_inst->getOperand(i); }; + + // Unconditionally branch from the original block to the block containing the + // condition. + builder.setInsertionPointToEnd(orig_block_head); + JumpToBlock(loc, get_operand, cond_block, &builder); + + // Call condition function in the condition block and then branch to the body + // block or remainder of the original block depending on condition function + // result. + builder.setInsertionPointToEnd(cond_block); + + auto get_cond_arg = [&](int i) { return cond_block->getArgument(i); }; + Operation* cond_call_op = CallFn(loc, get_cond_arg, cond_fn, &builder); + + assert(cond_call_op->getNumResults() == 1); + Value condition = LowerCondition(loc, cond_call_op->getResult(0), &builder); + auto br_operands = + PrepareValsForJump(loc, get_cond_arg, body_block, &builder); + cf::CondBranchOp::create(builder, loc, condition, body_block, br_operands, + orig_block_tail, br_operands); + + // Call body function in the body block and then unconditionally branch back + // to the condition block. + builder.setInsertionPointToEnd(body_block); + auto get_body_arg = [&](int i) { return body_block->getArgument(i); }; + Operation* body_call_op = CallFn(loc, get_body_arg, body_fn, &builder); + + auto get_body_result = [&](int i) { return body_call_op->getResult(i); }; + JumpToBlock(loc, get_body_result, cond_block, &builder); + + // Replace use of the while loop results with block inputs in the remainder of + // the original block and then delete the original While operation. + builder.setInsertionPoint(&orig_block_tail->front()); + ReplaceOpResultWithBlockArgs(loc, op_inst, orig_block_tail, &builder); + op_inst->erase(); + + return success(); +} + +void FunctionalControlFlowToCFG::runOnOperation() { + // Scan the function looking for these ops. + for (Block& block : getOperation()) { + for (Operation& op : block) { + // If the operation is one of the control flow ops we know, lower it. + // If we lower an operation, then the current basic block will be split, + // and the operation will be removed, so we should continue looking at + // subsequent blocks. + // + // TODO: Use PatternRewriter to eliminate these function control flow ops. + + if (IfOp if_op = llvm::dyn_cast(op)) { + if (failed(LowerIfOp(if_op))) { + return signalPassFailure(); + } + break; + } + if (WhileOp while_op = llvm::dyn_cast(op)) { + if (failed(LowerWhileOp(while_op))) { + return signalPassFailure(); + } + break; + } + } + } +} + +} // namespace + +std::unique_ptr> +CreateTFFunctionalControlFlowToCFG() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_regions.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_regions.cc new file mode 100644 index 0000000000000000000000000000000000000000..31f3994004168b547b1ad69351d8119eb5154ed5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_regions.cc @@ -0,0 +1,271 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass transforms functional control flow operations in the +// TensorFlow dialect to their region based counterparts, i.e., +// tf.If -> tf.IfRegion and tf.While -> tf.WhileRegion + +#include +#include +#include + +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +#define DEBUG_TYPE "tf-functional-cf-to-region" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_FUNCTIONALCONTROLFLOWTOREGIONSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct FunctionalControlFlowToRegions + : public impl::FunctionalControlFlowToRegionsPassBase< + FunctionalControlFlowToRegions> { + FunctionalControlFlowToRegions() = default; + explicit FunctionalControlFlowToRegions(bool allow_passthrough_args) + : FunctionalControlFlowToRegionsPassBase( + FunctionalControlFlowToRegionsPassOptions{allow_passthrough_args}) { + } + void runOnOperation() override; +}; + +// Creates a call to function `func` in region `caller_region`. Use `args` as +// the call arguments, and terminate the region with a yield. The arguments are +// cast to the required type before the call. `use_region_args` control whether +// the input arguments are used as is (for IfOp) or block arguments of the same +// type as the input arguments are created and then used as call arguments (for +// While). +YieldOp CreateCall(Operation* op, func::FuncOp func, Region& caller_region, + ValueRange args, bool use_region_args, + bool forward_block_args) { + assert(caller_region.empty() && + "Expected empty region for newly created ops"); + OpBuilder builder(caller_region); + Block* entry = builder.createBlock(&caller_region); + + auto loc = op->getLoc(); + if (use_region_args) { + auto inputs = func.getFunctionType().getInputs(); + entry->addArguments(inputs, SmallVector(inputs.size(), loc)); + args = entry->getArguments(); + } + llvm::SmallVector casted_args; + casted_args.reserve(func.getNumArguments()); + for (const auto& ArgAndType : zip(args, func.getFunctionType().getInputs())) { + Value arg = std::get<0>(ArgAndType); + Type expected_type = std::get<1>(ArgAndType); + if (arg.getType() != expected_type) { + arg = CastOp::create(builder, loc, expected_type, arg, + /*Truncate=*/builder.getBoolAttr(false)); + } + casted_args.push_back(arg); + } + auto call = func::CallOp::create(builder, loc, func, casted_args); + + auto results = call.getResults(); + auto block_args = entry->getArguments(); + auto yield_args = std::vector(); + yield_args.insert(yield_args.end(), results.begin(), results.end()); + if (forward_block_args) { + yield_args.insert(yield_args.end(), block_args.begin(), block_args.end()); + } + return YieldOp::create(builder, loc, yield_args); +} + +// Converts the condition for an IfOp/WhileOp to a boolean value. +Value ConvertConditionToBoolean(Operation* op, Value cond) { + if (auto ranked_type = mlir::dyn_cast(cond.getType())) + if (ranked_type.getRank() == 0 && + ranked_type.getElementType().isSignlessInteger(1)) + return cond; + + OpBuilder builder(op); + Value to_bool = TF::ToBoolOp::create(builder, op->getLoc(), cond); + CopyDeviceAndUnderscoredAttributes(op, to_bool.getDefiningOp()); + return to_bool; +} + +// Transform a functional IfOp to a region based IfRegionOp. +LogicalResult ConvertIfOp(IfOp if_op) { + Value cond = ConvertConditionToBoolean(if_op, if_op.getCond()); + OpBuilder builder(if_op); + auto if_region = TF::IfRegionOp::create( + builder, if_op.getLoc(), if_op.getResultTypes(), cond, + if_op.getIsStateless(), + builder.getStringAttr(if_op.then_function().getName()), + builder.getStringAttr(if_op.else_function().getName())); + CopyDeviceAndUnderscoredAttributes(if_op, if_region); + + CreateCall(if_op, if_op.then_function(), + /*caller_region=*/if_region.getThenBranch(), if_op.getInput(), + /*use_region_args=*/false, + /*forward_block_args=*/false); + CreateCall(if_op, if_op.else_function(), + /*caller_region=*/if_region.getElseBranch(), if_op.getInput(), + /*use_region_args=*/false, + /*forward_block_args=*/false); + if_op.replaceAllUsesWith(if_region.getResults()); + if_op.erase(); + return success(); +} + +LogicalResult ConvertCaseOp(CaseOp case_op) { + OpBuilder builder(case_op); + auto case_region = TF::CaseRegionOp::create( + builder, case_op.getLoc(), case_op.getResultTypes(), + case_op.getBranchIndex(), case_op.getIsStateless(), + case_op.getBranches().size()); + CopyDeviceAndUnderscoredAttributes(case_op, case_region); + + for (const auto& item : llvm::enumerate(case_region.getBranches())) { + CreateCall(case_op, case_op.branch_function(item.index()), + /*caller_region=*/item.value(), case_op.getInput(), + /*use_region_args=*/false, + /*forward_block_args=*/false); + } + case_op.replaceAllUsesWith(case_region.getResults()); + case_op.erase(); + return success(); +} + +LogicalResult ConvertWhileOp(WhileOp while_op, bool allow_passthrough_args) { + OpBuilder builder(while_op); + auto while_region = TF::WhileRegionOp::create( + builder, while_op.getLoc(), while_op.getResultTypes(), + while_op.getInput(), while_op.getParallelIterations(), + while_op.getIsStateless(), while_op.getShapeInvariant()); + CopyDeviceAndUnderscoredAttributes(while_op, while_region); + + YieldOp cond_yield = + CreateCall(while_op, while_op.cond_function(), + /*caller_region=*/while_region.getCond(), while_op.getInput(), + /*use_region_args=*/true, + /*forward_block_args=*/allow_passthrough_args); + Value i1_cond = + ConvertConditionToBoolean(cond_yield, cond_yield.getOperand(0)); + cond_yield.setOperand(0, i1_cond); + + CreateCall(while_op, while_op.body_function(), + /*caller_region=*/while_region.getBody(), while_op.getInput(), + /*use_region_args=*/true, + /*forward_block_args=*/false); + while_op.replaceAllUsesWith(while_region.getResults()); + while_op.erase(); + return success(); +} + +LogicalResult ConvertGeneratorDatasetOp(GeneratorDatasetOp generator_op) { + OpBuilder builder(generator_op); + auto generator_region = TF::GeneratorDatasetRegionOp::create( + builder, generator_op.getLoc(), generator_op->getResultTypes(), + generator_op.getInitFuncOtherArgs(), generator_op.getNextFuncOtherArgs(), + generator_op.getFinalizeFuncOtherArgs(), generator_op.getOutputTypes(), + generator_op.getOutputShapes(), generator_op.getMetadata()); + CopyDeviceAndUnderscoredAttributes(generator_op, generator_region); + + func::FuncOp init_function = + SymbolTable::lookupNearestSymbolFrom( + generator_op, generator_op.getInitFunc()); + func::FuncOp next_function = + SymbolTable::lookupNearestSymbolFrom( + generator_op, generator_op.getNextFunc()); + func::FuncOp finalize_function = + SymbolTable::lookupNearestSymbolFrom( + generator_op, generator_op.getFinalizeFunc()); + + if (!init_function || !next_function || !finalize_function) { + return failure(); + } + + CreateCall(generator_op, init_function, generator_region.getInit(), + generator_region.getInitFuncOtherArgs(), + /*use_region_args=*/true, /*forward_block_args=*/false); + CreateCall(generator_op, next_function, generator_region.getNext(), + generator_region.getNextFuncOtherArgs(), + /*use_region_args=*/true, /*forward_block_args=*/false); + CreateCall(generator_op, finalize_function, generator_region.getFinalize(), + generator_region.getFinalizeFuncOtherArgs(), + /*use_region_args=*/true, /*forward_block_args=*/false); + + generator_op->replaceAllUsesWith(generator_region->getResults()); + generator_op->erase(); + + return success(); +} + +void FunctionalControlFlowToRegions::runOnOperation() { + ModuleOp module = getOperation(); + auto result = module.walk([&](Operation* op) { + if (IfOp if_op = llvm::dyn_cast(op)) { + if (failed(ConvertIfOp(if_op))) { + op->emitOpError() << "failed to convert to region form"; + return WalkResult::interrupt(); + } + } else if (CaseOp case_op = llvm::dyn_cast(op)) { + if (failed(ConvertCaseOp(case_op))) { + op->emitOpError() << "failed to convert to region form"; + return WalkResult::interrupt(); + } + } else if (auto while_op = llvm::dyn_cast(op)) { + if (failed(ConvertWhileOp(while_op, allow_passthrough_args_))) { + op->emitOpError() << "failed to convert to region form"; + return WalkResult::interrupt(); + } + } else if (auto generator_op = llvm::dyn_cast(op)) { + if (allow_passthrough_args_) { + if (failed(ConvertGeneratorDatasetOp(generator_op))) { + op->emitOpError() << "failed to convert to region form"; + return WalkResult::interrupt(); + } + } + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return signalPassFailure(); +} +} // namespace + +std::unique_ptr> +CreateTFFunctionalControlFlowToRegions() { + return std::make_unique(); +} +std::unique_ptr> CreateTFFunctionalControlFlowToRegions( + bool allow_passthrough_args) { + return std::make_unique( + allow_passthrough_args); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fused_kernel_matcher.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fused_kernel_matcher.cc new file mode 100644 index 0000000000000000000000000000000000000000..75f7908f1430fc1019c91cda57673f727ec1deb6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/fused_kernel_matcher.cc @@ -0,0 +1,376 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { + +namespace TF { + +namespace { + +// Note: This implements the fusions performed in the old Remapper Grappler +// pass. That pass has specific cases for GPU and based on different +// target configurations on both CPU and GPU (Intel MKL, ROCm, etc.). This MLIR +// pass covers (some of) the general CPU case and at the moment does not account +// for any target-specific configurations. + +// This pass is being ported over from the Grappler Remapper pass based on +// need/usage. File a bug to request porting over additional fusions. + +// TODO(b/158265178): Support GPU-specific fusions. +// TODO(b/158266710): Support CPU MKL configurations. + +#define GEN_PASS_DEF_FUSEDKERNELMATCHERPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Optimizes TF computations by fusing subgraphs/nodes onto more efficient +// implementations to decrease the number of operations needed to perform a +// computation. +struct FusedKernelMatcherPass + : public impl::FusedKernelMatcherPassBase { + void runOnOperation() override; +}; + +bool IsActivationFunction(Operation *op) { + return isa(op); +} + +// Finds and returns an activation op that uses the result of `op`. If there are +// multiple such activations, one is returned (with no guarantee as to which +// one). If there are no activation functions that use the output, returns +// nullptr. +Operation *GetActivation(Value op) { + for (auto &use : op.getUses()) { + if (IsActivationFunction(use.getOwner())) return use.getOwner(); + } + return nullptr; +} + +// Finds and returns a BiasAdd that uses the result of `op` as the `value` +// input. If there are multiple such BiasAdds, one is returned (with no +// guarantee as to which one). If there are no BiasAdds that use the output, +// returns a null BiasAddOp. +BiasAddOp GetBiasAdd(Value op) { + for (auto &use : op.getUses()) { + auto bias_add = dyn_cast_or_null(use.getOwner()); + // If it's a BiasAdd, check that the conv op is the first input. + if (bias_add && bias_add.getValue() == op) return bias_add; + } + // No BiasAddOps found among uses. + return BiasAddOp(); +} + +// Performs a fusion of the following pattern(s), if possible: +// + BiasAdd + -> +// +// Note that fusion with activation is preferred, but a contraction and BiasAdd +// can also be replaced by a _FusedConv2D if there is no other activation +// function. +// i.e., this class also supports the following fusion: +// + BiasAdd -> +// +// TODO(b/158266331): Support fusing activation chains of arbitrary length. +template +class FuseContractionWithBiasAdd : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + // Class users should override this method if there are any op-specific + // compatibility requirements between the contraction op and the BiasAdd op. + virtual bool AreFuseCompatible(SrcOpT contraction_op, BiasAddOp bias_add, + PatternRewriter &rewriter) const { + return true; + } + + // Class users should override this method if there are any op-specific + // compatibility requirements for devices. + virtual bool IsDeviceCompatible(SrcOpT contraction_op, BiasAddOp bias_add, + PatternRewriter &rewriter) const { + return true; + } + + LogicalResult matchAndRewrite(SrcOpT contraction, + PatternRewriter &rewriter) const override { + auto context = rewriter.getContext(); + + // We do support fusion only if the contraction operation is inside one of + // the expected operations with regions. Other operations can have semantics + // that is not compatible with fusion (e.g. region compilation). + if (!isa(contraction->getParentOp())) { + return rewriter.notifyMatchFailure( + contraction, + "fused operation must be nested inside a function, If or While"); + } + + // If the contraction is used in multiple places, fusing it will only create + // more contraction nodes, which is slower. + if (!contraction.getResult().hasOneUse()) + return rewriter.notifyMatchFailure(contraction, + "result is used by multiple ops"); + + BiasAddOp bias_add = GetBiasAdd(contraction.getResult()); + if (!bias_add) { + return rewriter.notifyMatchFailure( + contraction, "does not feed into a tf.BiasAdd/tf.BiasAddV1 op"); + } + + if (!AreFuseCompatible(contraction, bias_add, rewriter)) { + return rewriter.notifyMatchFailure( + contraction, "cannot fuse with the subsequent BiasAdd op"); + } + + if (!IsDeviceCompatible(contraction, bias_add, rewriter)) { + return rewriter.notifyMatchFailure( + contraction, + "cannot fuse with the subsequent op as it's not supported by the " + "target device."); + } + + SmallVector locations{contraction.getLoc(), bias_add.getLoc()}; + SmallVector fused_ops{StringAttr::get( + context, bias_add.getOperation()->getName().stripDialect())}; + + // BiasAdd may or may not feed into an activation function. + auto activation = GetActivation(bias_add); + + // If there is an activation, only fuse it if this is the only op to use the + // result of the BiasAdd. + bool fuse_activation = activation && bias_add.getOutput().hasOneUse(); + Type result_type; + + // Include info about the activation function if applicable. + if (fuse_activation) { + locations.push_back(activation->getLoc()); + fused_ops.push_back( + StringAttr::get(context, activation->getName().stripDialect())); + result_type = activation->getResultTypes().front(); + } else { + result_type = bias_add.getResult().getType(); + } + + auto fused_loc = rewriter.getFusedLoc(locations); + + // The fused contraction has the same operands as the original contraction + // with `bias` from the BiasAddOp appended. + SmallVector operands(contraction.operand_begin(), + contraction.operand_end()); + operands.push_back(bias_add.getBias()); + + // The fused contraction has the same attributes as the original + // contraction, with two additions: the list of ops which have been fused + // together; epsilon (only with FusedBatchNorm). + std::vector attrs = contraction->getAttrs(); + ArrayAttr fused_ops_attr = ArrayAttr::get(context, fused_ops); + attrs.push_back( + NamedAttribute(StringAttr::get(context, "fused_ops"), fused_ops_attr)); + // Epsilon is used only in fusions with the FusedBatchNorm op, so we zero it + // here. + Attribute epsilon = rewriter.getF32FloatAttr(0); + attrs.push_back( + NamedAttribute(StringAttr::get(context, "epsilon"), epsilon)); + + if (std::is_same::value) { + // Here TArgs types do not include types of the first two parameters, + // i.e. the convolution input and the filter. TArgs are parameters for + // the extras like the bias etc. + auto attr = TypeAttr::get(getElementTypeOrSelf(contraction.getType())); + SmallVector targs_values(operands.size() - 2, attr); + + ArrayAttr targs_attr = ArrayAttr::get(context, targs_values); + attrs.push_back( + NamedAttribute(StringAttr::get(context, "TArgs"), targs_attr)); + + auto num_args_attr = IntegerAttr::get(IntegerType::get(context, 64), 1); + attrs.push_back( + NamedAttribute(StringAttr::get(context, "num_args"), num_args_attr)); + + // Fused conv operands are input, filter, args and host args. Here, bias + // input of the BiasAdd op. Host args corresponds to conv_input_scale and + // side_input_scale and not relevant in this case. + auto sizes = mlir::DenseI32ArrayAttr::get(context, {1, 1, 1, 0}); + auto attr_name = + StringAttr::get(context, mlir::OpTrait::AttrSizedOperandSegments< + void>::getOperandSegmentSizeAttr()); + attrs.push_back(NamedAttribute(attr_name, sizes)); + } + + // Insert fused operation right before the BiasAdd operation to guarantee + // that bias value dominates the fused operation. We already verified that + // original operation has a single use, so this is safe to do. + auto *bias_add_op = bias_add.getOperation(); + if (bias_add_op) rewriter.setInsertionPoint(bias_add_op); + + Value fused_op = FusedOpT::create(rewriter, fused_loc, result_type, + ValueRange(operands), attrs); + auto op_to_replace = fuse_activation ? activation : bias_add; + rewriter.replaceOp(op_to_replace, ValueRange({fused_op})); + return success(); + } +}; + +const char kDeviceAttr[] = "device"; +const char kDeviceGpu[] = "GPU"; + +std::optional GetDevice(mlir::Operation *op) { + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) { + return std::nullopt; + } + const std::string device_name = device.str(); + tensorflow::DeviceNameUtils::ParsedName parsed_name; + if (!tensorflow::DeviceNameUtils::ParseFullName(device_name, &parsed_name)) { + return std::nullopt; + } + if (!parsed_name.has_type) { + return std::nullopt; + } + return parsed_name.type; +} + +bool IsGpuDevice(mlir::Operation *op) { + std::optional device = GetDevice(op); + if (!device) return false; + return *device == kDeviceGpu; +} + +// Performs a fusion of the following pattern(s), if possible: +// Conv2D + BiasAdd + -> _FusedConv2D +class FuseConv2DBiasAdd + : public FuseContractionWithBiasAdd { + public: + using FuseContractionWithBiasAdd::FuseContractionWithBiasAdd; + // Verify that the Conv2D and BiasAdd data formats match. This is necessary + // for the ops to fuse correctly, the fused Conv2D op has one data format + // attribute which is shared. + bool AreFuseCompatible(Conv2DOp conv, BiasAddOp bias_add, + PatternRewriter &rewriter) const override { + // Verify that the data formats match and are valid for fusion. + if (conv.getDataFormat() != bias_add.getDataFormat()) { + (void)rewriter.notifyMatchFailure(conv, [&](Diagnostic &diag) { + diag << "data format does not match Conv2D data format (" + << bias_add.getDataFormat() << " vs " << conv.getDataFormat() + << ")"; + }); + return false; + } + + // Verify the data type is supported. + Type element_ty = getElementTypeOrSelf(conv.getType()); + if (!element_ty.isF32() && !element_ty.isF64()) { + (void)rewriter.notifyMatchFailure(conv, [&](Diagnostic &diag) { + diag << "supported data types for _FusedConv2D are float and double, " + << " but got " << element_ty; + }); + return false; + } + return true; + } + + bool IsDeviceCompatible(Conv2DOp conv, BiasAddOp bias_add, + PatternRewriter &rewriter) const override { + // Currently, GPU only supports Conv2D+BiasAdd+Relu fusion. + if (IsGpuDevice(conv)) { + auto activation = GetActivation(bias_add); + if (!activation || activation->getName().stripDialect() != "Relu" || + !bias_add.getOutput().hasOneUse()) { + (void)rewriter.notifyMatchFailure(conv, [&](Diagnostic &diag) { + diag << "GPU only supports Conv2D+BiasAdd+Relu fusion"; + }); + return false; + } + } + return true; + } +}; + +// Performs a fusion of the following pattern(s), if possible: +// MatMulOp + BiasAdd + -> _FusedMatMulOp +class FuseMatMulBiasAdd + : public FuseContractionWithBiasAdd { + using FuseContractionWithBiasAdd::FuseContractionWithBiasAdd; + + bool AreFuseCompatible(MatMulOp matmul, BiasAddOp bias_add, + PatternRewriter &rewriter) const override { + // FusedMatMul kernel supports limited set of data types. + Type element_ty = getElementTypeOrSelf(matmul.getType()); + if (!element_ty.isF32() && !element_ty.isBF16()) { + (void)rewriter.notifyMatchFailure(matmul, [&](Diagnostic &diag) { + diag << "supported data types for _FusedMatMul are float and bfloat16, " + << " but got " << element_ty; + }); + return false; + } + // FusedMatMul kernel does not support grad_a/grad_b attrs + if ((matmul->hasAttr("grad_a") && + mlir::cast(matmul->getAttr("grad_a")).getValue()) || + (matmul->hasAttr("grad_b") && + mlir::cast(matmul->getAttr("grad_b")).getValue())) { + (void)rewriter.notifyMatchFailure(matmul, [&](Diagnostic &diag) { + diag << "FusedMatMul kernel does not support grad_a/grad_b attrs"; + }); + return false; + } + return true; + } + + bool IsDeviceCompatible(MatMulOp matmul, BiasAddOp bias_add, + PatternRewriter &rewriter) const override { + if (IsGpuDevice(matmul)) { + (void)rewriter.notifyMatchFailure(matmul, [&](Diagnostic &diag) { + diag << "_FusedMatMul is not supported by GPU"; + }); + return false; + } + return true; + } +}; + +void FusedKernelMatcherPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + auto func = getOperation(); + patterns.add(&getContext()); + + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +std::unique_ptr> CreateFusedKernelMatcherPass() { + return std::make_unique(); +} + +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/gpu_fusion.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/gpu_fusion.cc new file mode 100644 index 0000000000000000000000000000000000000000..4422cd6c8b107322822694db44e456359b3f74b2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/gpu_fusion.cc @@ -0,0 +1,140 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +#define DEBUG_TYPE "tf-gpu-op-fusion" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_TENSORFLOWGPUFUSION +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// GpuOpFusionPass is a pass performing fusion specific to GPU targets. +// This is an ad-hoc pass for now, but should be integrated with some notion +// of "target" in the MLIR pipeline in the future. +class GpuOpFusionPass : public impl::TensorflowGPUFusionBase { + public: + void runOnOperation() final; +}; + +// %y:6 = "tf.FusedBatchNormV3"(%x, %scale, %offset, %mean, %variance) +// %0 = "tf.Relu"(%y#0) +// -> +// %y:6 = "tf._FusedBatchNormEx"(%x, %scale, %offset, %mean, %variance) +// +// Or: +// %y:6 = "tf.FusedBatchNormV3"(%x, %scale, %offset, %mean, %variance) +// %0 = "tf.AddV2"(%y#0, %side_input) +// %1 = "tf.Relu"(%0) +// -> +// %y:6 = "tf._FusedBatchNormEx"(%x, %scale, %offset, %mean, %variance, +// %side_input) +// TODO(aminim): we should revisit this as a declarative pattern. +// For the second pattern, there is not good way in the framework to handle the +// commutativity of the AddV2: we want the FusedBatchNormV3 on any side. +// Also we need some native calls to handle the "hasOneUse" aspects and the +// optional extra operands for the AddV2 case. +struct ReluToFusedBatchNorm : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(ReluOp relu_op, + PatternRewriter &rewriter) const override { + Operation *relu_input = relu_op.getFeatures().getDefiningOp(); + if (!relu_input) return failure(); + auto batch_norm = dyn_cast_or_null(relu_input); + AddV2Op add_op; + Value side_input; + if (!batch_norm) { + // We don't have a FusedBatchNorm as input to the ReLu, but we can get + // through an AddV2 as well. + add_op = dyn_cast_or_null(relu_input); + if (!add_op) return failure(); + + batch_norm = + dyn_cast_or_null(add_op.getX().getDefiningOp()); + if (batch_norm) { + side_input = add_op.getY(); + } else { + // Didn't get a FusedBatchNorm on the LHS of the AddV2, try the RHS. + batch_norm = + dyn_cast_or_null(add_op.getY().getDefiningOp()); + if (!batch_norm) return failure(); + side_input = add_op.getX(); + } + } + assert(batch_norm); + if (batch_norm.getIsTraining()) return failure(); + if (!batch_norm.getY().hasOneUse()) return failure(); + + // Build the newly fused operation to replace the batch norm + OperationState state(batch_norm.getLoc(), + _FusedBatchNormExOp::getOperationName()); + state.addOperands(batch_norm.getOperands()); + if (side_input) state.operands.push_back(side_input); + state.addTypes(batch_norm.getResultTypes()); + state.addAttributes(batch_norm->getAttrs()); + Operation *op = rewriter.create(state); + rewriter.replaceOp(batch_norm, op->getResults()); + + // Depending on the case, we may fuse the add, the relu, or both. + if (!add_op || add_op.getZ().hasOneUse()) { + // We fuse the Relu only if the add has a single use, otherwise we only + // fuse the add itself. + op->setAttr("activation_mode", rewriter.getStringAttr("Relu")); + rewriter.replaceOp(relu_op, op->getResult(0)); + } + if (add_op) { + rewriter.replaceOp(add_op, op->getResult(0)); + } + + return success(); + } +}; + +void GpuOpFusionPass::runOnOperation() { + func::FuncOp func = getOperation(); + RewritePatternSet patterns(&getContext()); + patterns.add(&getContext()); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +std::unique_ptr> CreateGpuOpFusionPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..a906163f1e95d5790c20eba77427d4cd373fc421 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.cc @@ -0,0 +1,80 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.h" + +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace mlir { +namespace TF { +namespace { +using Status = absl::Status; +using ConfigProto = ::tensorflow::ConfigProto; +using Graph = ::tensorflow::Graph; +} // namespace + +Status MlirGraphOptimizationPass::Run( + const std::string& function_name, const ConfigProto& config_proto, + ModuleOp module, const Graph& graph, + const tensorflow::FunctionLibraryDefinition& function_library) { + if (GetPassState(/*device_set=*/nullptr, config_proto, graph, + function_library) == + ::tensorflow::MlirOptimizationPassState::Disabled) { + VLOG(1) << "Skipping MLIR Graph Optimization Pass" + << ", session flag not enabled"; + return absl::OkStatus(); + } + + VLOG(1) << "Run MLIR Graph Optimization Passes"; + PassManager pm(module.getContext()); + ::tensorflow::applyTensorflowAndCLOptions(pm); + + // Run island coarsening before shape inference to allow more exact shape + // inference using constant folding within islands. + pm.addNestedPass( + tf_executor::CreateTFExecutorIslandCoarseningPass()); + pm.addPass(CreateTFShapeInferencePass()); + + // Assign optimal data layout to layout sensitive operations and delete + // redundant transposes from the IR. + LayoutOptimizationPipelineOptions layout_optimization_options; + CreateLayoutOptimizationPipeline(pm.nest(), + layout_optimization_options); + + // Prepare IR for exporting. + pm.addPass(CreateBreakUpIslandsPass()); + + // In case of failure, the `diag_handler` converts MLIR errors emitted to the + // MLIRContext into a tensorflow::Status. + StatusScopedDiagnosticHandler diag_handler(module.getContext()); + LogicalResult result = pm.run(module); + (void)result; + return diag_handler.ConsumeStatus(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..80b39dcbaf9c6a6d9bbabbadad15426b4b2b11ed --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.h @@ -0,0 +1,55 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GRAPH_OPTIMIZATION_PASS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GRAPH_OPTIMIZATION_PASS_H_ + +#include + +#include "absl/status/status.h" +#include "tensorflow/compiler/mlir/mlir_graph_optimization_pass.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace mlir { +namespace TF { + +// Bundle generic MLIR graph optimization passes (some derived from TF Grappler +// graph optimizers) into a single MLIR optimization pass. +class MlirGraphOptimizationPass : public ::tensorflow::MlirOptimizationPass { + public: + llvm::StringRef name() const override { return "graph_optimization"; } + + ::tensorflow::MlirOptimizationPassState GetPassState( + const ::tensorflow::DeviceSet* device_set, + const ::tensorflow::ConfigProto& config_proto, + const tensorflow::Graph& graph, + const tensorflow::FunctionLibraryDefinition& function_library) + const override { + return config_proto.experimental().enable_mlir_graph_optimization() + ? tensorflow::MlirOptimizationPassState::Enabled + : tensorflow::MlirOptimizationPassState::Disabled; + } + + absl::Status Run( + const std::string& function_name, + const ::tensorflow::ConfigProto& config_proto, ModuleOp module, + const ::tensorflow::Graph& graph, + const tensorflow::FunctionLibraryDefinition& function_library) override; +}; + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GRAPH_OPTIMIZATION_PASS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass_registration.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass_registration.cc new file mode 100644 index 0000000000000000000000000000000000000000..9f3f98186e2b7e936af9d5d1354414a1963372cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass_registration.cc @@ -0,0 +1,30 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/mlir/tensorflow/transforms/graph_optimization_pass.h" + +namespace tensorflow { +namespace { +constexpr int kMlirGraphOptimizationPriority = 0; +} + +static mlir_pass_registration::MlirOptimizationPassRegistration + register_mlir_graph_optimization_pass( + kMlirGraphOptimizationPriority, + std::make_unique()); + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_pruning.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_pruning.cc new file mode 100644 index 0000000000000000000000000000000000000000..fcbecd171745801534c933d6d1a4e20d36405eb2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/graph_pruning.cc @@ -0,0 +1,200 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_side_effects.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace tf_executor { +namespace { + +#define GEN_PASS_DEF_EXECUTORGRAPHPRUNINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// This transformation pass prunes a TF graph eliminating dead-nodes. +class GraphPruningPass + : public impl::ExecutorGraphPruningPassBase { + public: + GraphPruningPass() = default; + explicit GraphPruningPass(llvm::ArrayRef ops_to_preserve); + void runOnOperation() override; + + private: + bool ShouldPreserveOp(Operation* op); + bool ShouldPreserveIsland(IslandOp island); + void PruneGraph(GraphOp graph); + + llvm::SmallDenseSet ops_to_preserve_ids_; +}; + +// Checks if a tf_executor.Graph can be pruned. +// For TensorFlow V1.0 compatibility: when importing a graph without providing +// feeds/fetches/targets we should not attempt to prune. The best approximation +// here is to check if the graph is of the "main" function and does not have the +// "tf.entry_function" attribute defined. +bool CanPruneGraph(func::FuncOp func) { + return func.getName() != "main" || + func->getAttrOfType("tf.entry_function") != nullptr; +} + +// Visits an op's operand if it is an output of an Operation in the same +// tf_executor.graph. +void VisitOpOperand(GraphOp graph, Value operand, + llvm::SmallPtrSetImpl* reachable_ops, + llvm::SmallVectorImpl* ops_to_visit) { + Operation* def = operand.getDefiningOp(); + if (def && def->getParentOp() == graph && reachable_ops->insert(def).second) { + // Op has not been visited, add to queue to visit later. + ops_to_visit->push_back(def); + } +} + +// Visits all operands of an op where each operand is an output of an Operation +// in the same tf_executor.graph. +void VisitOpOperands(GraphOp graph, Operation* op, + llvm::SmallPtrSetImpl* reachable_ops, + llvm::SmallVectorImpl* ops_to_visit) { + for (Value operand : op->getOperands()) + VisitOpOperand(graph, operand, reachable_ops, ops_to_visit); +} + +// Visits an op and it's associated operands. IslandOps are handled differently +// where it's regions op operands are also visited as values may be implicitly +// captured within. NextIterationSourceOp will also visit it's associated +// NextIterationSinkOp. +void VisitOp(GraphOp graph, Operation* op, + llvm::SmallPtrSetImpl* reachable_ops, + llvm::SmallVectorImpl* ops_to_visit) { + if (auto island = llvm::dyn_cast(op)) { + mlir::visitUsedValuesDefinedAbove( + island.getBody(), island.getBody(), [&](OpOperand* operand) { + VisitOpOperand(graph, operand->get(), reachable_ops, ops_to_visit); + }); + } + + VisitOpOperands(graph, op, reachable_ops, ops_to_visit); + + // If op is a `tf_executor.NextIteration.Source`, visit its associated + // `tf_executor.NextIteration.Sink` op. + if (auto source_op = llvm::dyn_cast(op)) { + Operation* sink_op = source_op.GetSink().getOperation(); + if (reachable_ops->insert(sink_op).second) ops_to_visit->push_back(sink_op); + } +} + +GraphPruningPass::GraphPruningPass( + llvm::ArrayRef ops_to_preserve) { + ops_to_preserve_ = ops_to_preserve; +} + +void GraphPruningPass::runOnOperation() { + for (const auto& op_name : ops_to_preserve_) { + ops_to_preserve_ids_.insert(mlir::StringAttr::get(&getContext(), op_name)); + } + if (!CanPruneGraph(getOperation())) return; + getOperation().walk( + [this](tf_executor::GraphOp graph) { PruneGraph(graph); }); +} + +// An op should be preserved if either its identifier is contained in +// `ops_to_preserve_ids_` or if it has a `MustExecute` effect. +bool GraphPruningPass::ShouldPreserveOp(Operation* op) { + if (ops_to_preserve_ids_.contains(op->getName().getIdentifier())) return true; + + llvm::SmallVector effects; + auto interface = dyn_cast(op); + if (interface) interface.getEffects(effects); + + for (const auto& effect : effects) { + if (llvm::isa(effect.getResource())) { + return true; + } + } + return false; +} + +// An island should be preserved if any of its inner ops should be preserved. +bool GraphPruningPass::ShouldPreserveIsland(IslandOp island) { + auto result = island.walk([this](Operation* inner_op) { + if (ShouldPreserveOp(inner_op)) return WalkResult::interrupt(); + return WalkResult::advance(); + }); + return result.wasInterrupted(); +} + +// Prunes unreachable operations of a tf_executor.graph operation. +void GraphPruningPass::PruneGraph(GraphOp graph) { + // A graph has a single block which forms a DAG: operations that aren't + // reachable from the `fetch` operands can be eliminated. + + llvm::SmallPtrSet reachable_ops; + llvm::SmallVector ops_to_visit; + + // Visit fetches first to create a starting point for ops that are reachable. + reachable_ops.insert(graph.GetFetch()); + VisitOpOperands(graph, graph.GetFetch(), &reachable_ops, &ops_to_visit); + + // Find and visit ops that should be preserved regardless of being reachable + // from a fetch. + for (Operation& op : graph.GetBody().without_terminator()) { + auto island = llvm::dyn_cast(op); + if (!island) continue; + if (ShouldPreserveIsland(island)) { + reachable_ops.insert(&op); + VisitOp(graph, &op, &reachable_ops, &ops_to_visit); + } + } + + // Visit transitive ops until no there are no reachable ops left that have not + // been visited. + while (!ops_to_visit.empty()) { + Operation* op = ops_to_visit.pop_back_val(); + VisitOp(graph, op, &reachable_ops, &ops_to_visit); + } + + // Erase unreachable ops in reverse order so references don't need to be + // dropped before removing an op. Going in reverse order will guarantee that + // when an op to be erased is reached, there are no users left. + for (Operation& op : + llvm::make_early_inc_range(llvm::reverse(graph.GetBody()))) + if (!reachable_ops.contains(&op)) op.erase(); +} + +} // namespace + +std::unique_ptr> CreateTFExecutorGraphPruningPass( + llvm::ArrayRef ops_to_preserve) { + return std::make_unique(ops_to_preserve); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.cc new file mode 100644 index 0000000000000000000000000000000000000000..f0632aee0afbb288fa64685ea1c3f6bdeb88d94d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.cc @@ -0,0 +1,237 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +void wrapOpsInFunction(std::vector& ops, int function_id, + Operation* module); + +#define GEN_PASS_DEF_GROUPBYDIALECTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class GroupByDialectPass + : public impl::GroupByDialectPassBase { + public: + void runOnOperation() override; + + private: + void processFunction(mlir::func::FuncOp func, int& counter, + llvm::SmallDenseSet& dialects, + Operation* module); + void processRegion(mlir::Region& region, int& counter, + llvm::SmallDenseSet& dialects, + Operation* module); + + llvm::SmallDenseSet top_level_dialects_ = {"ml_program", "glue", + "func"}; +}; + +// Compute the set of all values which are inputs to `ops`, but not generated +// by an operation in `ops`, and all outputs which are used outside of `ops. +void computeInputsOutputs(std::vector& ops, + std::vector* inputs, + std::vector* outputs) { + // All operations. + llvm::DenseSet all_operations; + + // All results of all ops. + llvm::DenseSet all_internal_results; + for (Operation* outer : ops) { + outer->walk([&](Operation* op) { + all_operations.insert(op); + for (Value result : op->getResults()) { + all_internal_results.insert(result); + } + // We treat block arguments of inner blocks as "results", too, in + // the sense that they're values produced inside this op. + for (Region& region : op->getRegions()) { + for (Block& block : region.getBlocks()) { + for (BlockArgument& arg : block.getArguments()) { + all_internal_results.insert(arg); + } + } + } + }); + } + + // All operand values in our set not produced as result by some op in our set. + llvm::DenseSet inputs_seen; + for (Operation* outer : ops) { + outer->walk([&](Operation* op) { + for (Value operand : op->getOperands()) { + if (!all_internal_results.contains(operand)) { + if (!inputs_seen.contains(operand)) { + inputs->push_back(operand); + inputs_seen.insert(operand); + } + } + } + }); + } + + // All results in our set that have a user outside our set. + llvm::DenseSet outputs_seen; + for (Operation* op : ops) { + for (Value result : op->getResults()) { + for (auto& use : result.getUses()) { + if (!all_operations.contains(use.getOwner())) { + if (!outputs_seen.contains(result)) { + outputs->push_back(result); + outputs_seen.insert(result); + } + break; + } + } + } + } +} + +void wrapOpsInFunction(std::vector& ops, int function_id, + Operation* module) { + if (ops.empty()) { + return; + } + + std::vector inputs; + std::vector outputs; + computeInputsOutputs(ops, &inputs, &outputs); + + std::vector input_types; + std::vector output_types; + + input_types.reserve(inputs.size()); + for (Value v : inputs) { + input_types.push_back(v.getType()); + } + output_types.reserve(outputs.size()); + for (Value v : outputs) { + output_types.push_back(v.getType()); + } + + // Create the function. + MLIRContext* context = ops[0]->getContext(); + StringRef dialect = ops[0]->getName().getDialectNamespace(); + OpBuilder builder(context); + // Every ModuleOp has at least one region and one block. + Block* first_block = &module->getRegion(0).front(); + builder.setInsertionPointToEnd(first_block); + auto func = mlir::func::FuncOp::create( + builder, ops[0]->getLoc(), dialect.str() + std::to_string(function_id), + builder.getFunctionType(input_types, output_types)); + func->setAttr("dialect", builder.getStringAttr(dialect)); + auto block = func.addEntryBlock(); + + llvm::DenseSet all_operations; + for (Operation* outer : ops) { + outer->walk([&](Operation* op) { all_operations.insert(op); }); + } + + for (BlockArgument& arg : block->getArguments()) { + inputs[arg.getArgNumber()].replaceUsesWithIf(arg, [=](OpOperand& o) { + // Within the operations we're moving, we need to replace uses of + // values generated elsewhere. + return all_operations.contains(o.getOwner()); + }); + } + + // Insert function call. + builder.setInsertionPoint(ops[0]); + auto call = mlir::func::CallOp::create(builder, ops[0]->getLoc(), + func.getFunctionType().getResults(), + func.getSymName(), inputs); + for (const auto& v : llvm::enumerate(outputs)) { + v.value().replaceUsesWithIf(call.getResult(v.index()), [=](OpOperand& o) { + // Outside of what we're moving, results of our operations need to + // be replaced by results from the function call. + return !all_operations.contains(o.getOwner()); + }); + } + + // Move ops inside function & add return. + builder.setInsertionPointToEnd(block); + for (Operation* op : ops) { + op->remove(); + builder.insert(op); + } + mlir::func::ReturnOp::create(builder, ops[0]->getLoc(), outputs); +} + +} // namespace + +void GroupByDialectPass::processFunction( + mlir::func::FuncOp func, int& counter, + llvm::SmallDenseSet& dialects, Operation* module) { + // don't re-process functions we generated + if (func->getAttr("dialect")) return; + processRegion(func.getBody(), counter, dialects, module); +} + +void GroupByDialectPass::processRegion(mlir::Region& region, int& counter, + llvm::SmallDenseSet& dialects, + Operation* module) { + for (Block& block : region.getBlocks()) { + StringRef current_dialect(""); + std::vector ops; + for (Operation& op : block.getOperations()) { + StringRef dialect = op.getName().getDialectNamespace(); + for (Region& region : op.getRegions()) { + // When processing nested operations, move all ops (except for func.*) + // that aren't of the parent dialect into a function or their own. + llvm::SmallDenseSet parent_dialect = {dialect, "func"}; + processRegion(region, counter, parent_dialect, module); + } + if (dialect != current_dialect) { + if (!dialects.contains(current_dialect)) { + wrapOpsInFunction(ops, counter++, module); + } + ops.clear(); + current_dialect = dialect; + } + ops.push_back(&op); + } + if (!dialects.contains(current_dialect)) { + wrapOpsInFunction(ops, counter++, module); + } + } +} + +void GroupByDialectPass::runOnOperation() { + int counter = 0; + Operation* module = getOperation(); + module->walk([&](func::FuncOp func) { + processFunction(func, counter, top_level_dialects_, module); + }); +} + +std::unique_ptr CreateGroupByDialectPass() { + return std::make_unique(); +} + +void RegisterGroupByDialectPass() { registerPass(CreateGroupByDialectPass); } + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.h new file mode 100644 index 0000000000000000000000000000000000000000..0e3182e01c846114e75255290bfeef5d3e9d6983 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/group_by_dialect.h @@ -0,0 +1,35 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GROUP_BY_DIALECT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GROUP_BY_DIALECT_H_ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Create a pass that groups ops into functions that only contain one dialect. +std::unique_ptr CreateGroupByDialectPass(); + +// Register this pass in the global registry of MLIR. +void RegisterGroupByDialectPass(); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_GROUP_BY_DIALECT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc new file mode 100644 index 0000000000000000000000000000000000000000..c609276add431f57d9818f5b7a920b39937a10b6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc @@ -0,0 +1,136 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SCCIterator.h" +#include "llvm/ADT/STLExtras.h" +#include "mlir/Analysis/CallGraph.h" // from @llvm-project +#include "mlir/Dialect/Affine/Utils.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +// Check that there is no recursion in the module's call graph. +LogicalResult CheckNoRecursion(ModuleOp module, CallGraph &call_graph) { + for (llvm::scc_iterator scci = + llvm::scc_begin(&call_graph); + !scci.isAtEnd(); ++scci) { + if (scci.hasCycle()) { + auto err = module.emitError() + << "A recursive call graph cannot be transformed to " + "one use for all functions. Functions in the " + "recursive cycle are: "; + llvm::interleaveComma(*scci, err, [&](CallGraphNode *node) { + err << node->getCallableRegion()->getLoc(); + }); + return err; + } + } + return success(); +} + +#define GEN_PASS_DEF_GUARANTEEALLFUNCSONEUSEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Clones FuncOp's until they have a single use only (or no users). +// +// The tf-shape-inference pass doesn't support functions that have more than +// a single use. But some real code from frontends does end up creating code +// like that. For example, the same LSTM cell function or loop body function +// will be reused. +// +// This pass clones functions as needed to establish the invariant that all +// functions have a single use. This can in principle cause exponential code +// size bloat, and should in general be guided by a proper cost model. +// +// There are two factors which should be considered by a principled replacement +// to this pass: +// +// 1. TF currently relies on "sufficiently good shape inference" for +// correctness so for now the cost of doing this seems acceptable since +// pathological cases haven't hit us yet. +// +// 2. Cloning functions can help by allowing code to be specialized (much as +// inlining does). In fact, tf-shape-inference attempts to do specialization +// of callees which is difficult if callees have multiple uses. +class GuaranteeAllFuncsOneUse + : public impl::GuaranteeAllFuncsOneUsePassBase { + public: + void runOnOperation() override { + if (failed(Run())) { + signalPassFailure(); + } + } + + LogicalResult Run() { + auto module = getOperation(); + + // Overall strategy: + // Fixed point iteration, iteratively applying a rule that clones + // any FuncOp with more than one use to eliminate its uses. + SymbolTableCollection symbol_table_collection; + SymbolTable &symbol_table = symbol_table_collection.getSymbolTable(module); + bool made_changes = false; + + if (failed(CheckNoRecursion(module, getAnalysis()))) + return failure(); + + do { + SymbolUserMap symbol_users(symbol_table_collection, module); + + made_changes = false; + for (auto func : + llvm::make_early_inc_range(module.getOps())) { + ArrayRef users = symbol_users.getUsers(func); + if (users.size() <= 1) { + continue; + } + + // At this point, we know we are going to change the module. + made_changes = true; + for (Operation *user : users.drop_front()) { + func::FuncOp new_func = func.clone(); + symbol_table.insert(new_func); + new_func.setPrivate(); + if (failed(SymbolTable::replaceAllSymbolUses( + func, new_func.getSymNameAttr(), user))) { + return func.emitError() << "could not replace symbol use"; + } + } + } + } while (made_changes); + + return success(); + } +}; + +} // namespace + +std::unique_ptr> CreateGuaranteeAllFuncsOneUsePass() { + return std::make_unique(); +} + +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_loop_invariant.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_loop_invariant.cc new file mode 100644 index 0000000000000000000000000000000000000000..f10948ef07c02b9d1e3f18a5097e9057e1dbe8a1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_loop_invariant.cc @@ -0,0 +1,160 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_interfaces.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_HOISTLOOPINVARIANTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Hoists loop invariant ops to the outside of the loop. +// This is similar to LoopInvariantCodeMotion pass, but it also hoists resource +// related ops, e.g., ReadVariableOp, if the variable is read only. +struct HoistLoopInvariantPass + : public impl::HoistLoopInvariantPassBase { + void runOnOperation() override; +}; + +// Get the resource handle of the given `op`. +ResourceHandle GetResourceHandle(Operation *op) { + llvm::StringRef device; + if (auto attr = op->getAttrOfType("device")) { + device = attr.getValue(); + } + + llvm::StringRef container; + if (auto attr = op->getAttrOfType("container")) { + container = attr.getValue(); + } + + llvm::StringRef shared_name; + if (auto attr = op->getAttrOfType("shared_name")) { + shared_name = attr.getValue(); + } + + return {container, shared_name, device, /*op=*/nullptr}; +} + +bool ResourceOpCanBeHoisted( + Operation *op, Region *region, + const llvm::DenseSet &read_only_vars) { + // If the op is ReadVariableOp and the variable is readonly, it can be + // hoisted. + auto read_var_op = llvm::dyn_cast(op); + if (!read_var_op) return false; + auto var_handle_op = llvm::dyn_cast_or_null( + read_var_op.getResource().getDefiningOp()); + if (!var_handle_op) return false; + return read_only_vars.contains(GetResourceHandle(var_handle_op)); +} + +bool ShouldMoveOutOfRegion( + Operation *op, Region *region, + const llvm::DenseSet &read_only_vars) { + return ResourceOpCanBeHoisted(op, region, read_only_vars) || + (isMemoryEffectFree(op) && isSpeculatable(op)); +} + +bool OnlyHasReadEffect(Operation *op) { + auto interface = llvm::dyn_cast(op); + if (!interface) return false; + return interface.onlyHasEffect(); +} + +// Get read only variable handles in `func`. +llvm::DenseSet GetReadOnlyVariables(func::FuncOp func) { + llvm::DenseMap> resources; + + // Get all VarHandleOps in the function. + func.walk([&](Operation *op) { + if (llvm::isa(op)) { + resources[GetResourceHandle(op)].push_back(op); + } + }); + + // Get read only variables by checking if their users only have read effect. + llvm::DenseSet read_only_vars; + for (const auto &[resource_handle, var_handle_ops] : resources) { + if (std::all_of(var_handle_ops.begin(), var_handle_ops.end(), + [](Operation *op) { + for (auto *user : op->getUsers()) { + if (!OnlyHasReadEffect(user)) return false; + } + return true; + })) { + read_only_vars.insert(resource_handle); + } + } + return read_only_vars; +} + +size_t HoistLoopInvariantCode( + LoopLikeOpInterface loopLike, + const llvm::DenseSet &read_only_vars) { + return moveLoopInvariantCode( + loopLike.getLoopRegions(), + [&](Value value, Region *) { + return loopLike.isDefinedOutsideOfLoop(value); + }, + [&](Operation *op, Region *region) { + return ShouldMoveOutOfRegion(op, region, read_only_vars); + }, + [&](Operation *op, Region *) { loopLike.moveOutOfLoop(op); }); +} + +void HoistLoopInvariantPass::runOnOperation() { + func::FuncOp func = getOperation(); + + // Skip the pass if the function inputs contain any resource. + for (const auto &type : func.getArgumentTypes()) { + if (mlir::isa(getElementTypeOrSelf(type))) return; + } + + llvm::DenseSet read_only_vars = GetReadOnlyVariables(func); + + func->walk([&](LoopLikeOpInterface loopLike) { + HoistLoopInvariantCode(loopLike, read_only_vars); + }); +} + +} // namespace + +std::unique_ptr> CreateHoistLoopInvariantPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_replicate_invariant_resource_writes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_replicate_invariant_resource_writes.cc new file mode 100644 index 0000000000000000000000000000000000000000..0315893980974a8dc8f173aa58582c4eea93cb8b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/hoist_replicate_invariant_resource_writes.cc @@ -0,0 +1,164 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +#define DEBUG_TYPE "tf-hoist-replicate-invariant-resource-writes" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_HOISTREPLICATEINVARIANTRESOURCEWRITESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct HoistReplicateInvariantResourceWritesPass + : public impl::HoistReplicateInvariantResourceWritesPassBase< + HoistReplicateInvariantResourceWritesPass> { + void runOnOperation() override; +}; + +// TODO(prakalps): This is a common utility and other passes use something +// similar. Move to common utils. +bool IsResourceType(Type type) { + return llvm::isa(type) || + (llvm::isa(type) && + llvm::isa( + llvm::cast(type).getElementType())); +} + +SmallVector GetAccessedResources(Operation& op) { + SmallVector accessed_resources; + for (auto operand : op.getOperands()) { + if (!IsResourceType(operand.getType())) continue; + accessed_resources.push_back(operand); + } + return std::move(accessed_resources); +} + +// Lifts the tail writes outside of tf_device.replicate. The written value is +// added to the values returned by tf_device.replicate op. Modify the assign +// variable ops to use the value from first replica. +void MoveTailWritesAfterReplicate( + tf_device::ReplicateOp replicate_op, + llvm::ArrayRef tail_assign_variable_ops) { + const auto num_replicas = replicate_op.getN(); + auto return_op = llvm::dyn_cast( + replicate_op.getRegion().front().getTerminator()); + + // Get the new result types. + // TODO(prakalps): Do not add a value to returned values if it is already + // returned. + auto new_result_types = llvm::to_vector<4>(replicate_op->getResultTypes()); + for (auto assign : tail_assign_variable_ops) { + return_op->insertOperands(return_op->getNumOperands(), assign.getValue()); + new_result_types.insert(new_result_types.end(), num_replicas, + assign.getValue().getType()); + } + + OpBuilder builder(replicate_op); + // Clone this old replicate op but with new result types. + auto new_replicate_op = tf_device::ReplicateOp::create( + builder, replicate_op->getLoc(), new_result_types, + replicate_op->getOperands(), replicate_op->getAttrs()); + + // Move region to the new op. + new_replicate_op.getRegion().takeBody(replicate_op.getRegion()); + + // Replace all old uses with new op results. + int old_num_results = replicate_op->getNumResults(); + replicate_op->replaceAllUsesWith( + new_replicate_op->getResults().take_front(old_num_results)); + + // Move assign ops after replicate and use the output of first replica. + for (auto indexed_assign : llvm::enumerate(tail_assign_variable_ops)) { + auto assign_op = indexed_assign.value(); + auto index = indexed_assign.index(); + assign_op->moveAfter(new_replicate_op); + assign_op->setOperand( + 1, new_replicate_op->getResult(old_num_results + num_replicas * index)); + } + replicate_op->erase(); +} + +// Looks for AssignVariable ops from the end of the tf_device.replicate op. It +// returns all the last writes to replicate invariant resource variables +// (resource handles defined outside the tf_device.replicate op). +SmallVector GetTailWritesToReplicateInvariantResourceVars( + tf_device::ReplicateOp replicate_op) { + SmallVector tail_assign_variable_ops; + llvm::SmallDenseSet visited_resources; + for (auto& op : + llvm::reverse(replicate_op.getRegion().front().getOperations())) { + SmallVector op_accessed_resources = GetAccessedResources(op); + if (op_accessed_resources.empty()) continue; + + if (auto assign = llvm::dyn_cast(op)) { + Value resource_var = assign.getResource(); + if (visited_resources.contains(resource_var) || + !resource_var.getParentRegion()->isProperAncestor( + &replicate_op.getRegion())) + continue; + tail_assign_variable_ops.push_back(assign); + } + + for (Value resource : op_accessed_resources) + visited_resources.insert(resource); + } + return std::move(tail_assign_variable_ops); +} + +void HoistReplicateInvariantResourceWritesPass::runOnOperation() { + SmallVector replicate_ops; + getOperation().walk([&](tf_device::ReplicateOp replicate_op) { + replicate_ops.push_back(replicate_op); + }); + for (auto replicate_op : replicate_ops) { + SmallVector tail_writes = + GetTailWritesToReplicateInvariantResourceVars(replicate_op); + + if (tail_writes.empty()) continue; + MoveTailWritesAfterReplicate(replicate_op, tail_writes); + } +} + +} // namespace + +std::unique_ptr> +CreateHoistReplicateInvariantResourceWritesPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..49bd0738a44ae55448f7083afcdc71afbefb3cbc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/BUILD @@ -0,0 +1,275 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir:__subpackages__", + "//tensorflow/compiler/mlir/tensorflow/transforms:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/api:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + "//tensorflow/compiler/tf2xla:__pkg__", + "//tensorflow/dtensor/mlir:__pkg__", + ], + licenses = ["notice"], +) + +cc_library( + name = "lower_cluster_to_runtime_ops", + srcs = ["lower_cluster_to_runtime_ops.cc"], + hdrs = [ + "lower_cluster_to_runtime_ops.h", + ], + deps = [ + ":runtime_passes", + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tensorflow/transforms/sparsecore:sparsecore_passes", + "//tensorflow/core:framework", + "//tensorflow/core:lib_proto_parsing", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:status", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tsl//tsl/platform:error_logging", + "@xla//xla/tsl/platform:errors", + ], +) + +tf_cc_test( + name = "lower_cluster_to_runtime_ops_test", + srcs = ["lower_cluster_to_runtime_ops_test.cc"], + data = [ + "testdata/basic_cluster.mlir", + "testdata/empty_func.mlir", + "testdata/malformed_cluster.mlir", + ], + deps = [ + ":lower_cluster_to_runtime_ops", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "tpu_metadata_utils", + srcs = [ + "tpu_metadata_utils.cc", + ], + hdrs = [ + "tpu_metadata_utils.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@xla//xla:xla_data_proto_cc", + "@xla//xla:xla_proto_cc", + ], +) + +tf_cc_test( + name = "tpu_metadata_utils_test", + srcs = ["tpu_metadata_utils_test.cc"], + data = [ + "testdata/basic_cluster.mlir", + "testdata/spmd.mlir", + ], + deps = [ + ":tpu_metadata_utils", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +gentbl_cc_library( + name = "runtime_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"runtime_passes.h.inc": [ + "-gen-pass-decls", + "-name=RuntimeLowering", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "runtime_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "runtime_passes", + hdrs = [ + "runtime_passes.h", + ], + textual_hdrs = [ + "runtime_passes.h.inc", + ], + deps = [ + ":runtime_passes_inc_gen", + ":tpu_merge_variable_with_execute", + ":tpu_rewrite_pass", + ":tpu_variable_runtime_reformatting", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "tpu_rewrite_pass", + srcs = ["tpu_rewrite_pass.cc"], + hdrs = [ + "runtime_passes.h", + ], + deps = [ + ":runtime_passes_inc_gen", + ":tpu_metadata_utils", + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:parallel_execute_util", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow:xla_rewrite_util", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/core:framework", + "//tensorflow/core:lib_proto_parsing", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:fingerprint", + "//tensorflow/core/platform:status", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tsl//tsl/platform:error_logging", + "@xla//xla:xla_data_proto_cc", + "@xla//xla:xla_proto_cc", + "@xla//xla/mlir_hlo", + "@xla//xla/tsl/platform:errors", + ], +) + +cc_library( + name = "tpu_variable_runtime_reformatting", + srcs = ["tpu_variable_runtime_reformatting.cc"], + hdrs = [ + "runtime_passes.h", + ], + deps = [ + ":runtime_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:random", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@xla//xla/mlir_hlo", + ], +) + +cc_library( + name = "tpu_merge_variable_with_execute", + srcs = ["tpu_merge_variables_with_execute.cc"], + hdrs = [ + "runtime_passes.h", + ], + deps = [ + ":runtime_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/mlir_hlo", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..6743c4c0131ba6a55ab64f36f96ac6a5da6987b7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.cc @@ -0,0 +1,229 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/metrics.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/error_payloads.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +using mlir::LogicalResult; +using mlir::OpPassManager; +using mlir::PassManager; +using mlir::func::FuncOp; +using mlir::TF::StandardPipelineOptions; + +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(PassManager& pm, const std::string& dump_group_name, + llvm::StringRef module_name) { + // Print the whole module after each pass, which requires disabling + // multi-threading as well. + pm.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique<::tensorflow::DataDumperLoggerConfig>( + [module_name, dump_group_name](const std::string& pass_tag_name, + mlir::Operation* op) { + return DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), dump_group_name, pass_tag_name); + }, + /*pass_prefix=*/"", + /*print_module_scope=*/true)); + pm.enableTiming(); +} + +} // namespace + +void AddTPULowerClusterToRuntimeOpsPassPipeline(OpPassManager& pm, + llvm::StringRef module_name) { + pm.addPass(mlir::TFTPU::CreateTPURewritePass(module_name)); + pm.addPass(mlir::createSymbolDCEPass()); + pm.addNestedPass( + mlir::TFDevice::CreateReplicateInvariantOpHoistingPass()); + pm.addNestedPass(mlir::TFDevice::CreateEmbeddingProgramKeyPass()); + pm.addPass(mlir::TFTPU::CreateTPUMergeVariablesWithExecutePass()); + pm.addNestedPass( + mlir::TFTPU::CreateExtractTPUCopyWithDynamicShapeOpPass()); + pm.addNestedPass( + mlir::TFTPU::CreateTPUColocateCompositeResourceOps()); + if (tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_tpu_variable_runtime_reformatting_pass) { + pm.addPass(mlir::TFTPU::CreateTPUVariableRuntimeReformattingPass()); + } +} + +void AddNonTPULowerClusterToRuntimeOpsPassPipeline( + OpPassManager& pm, llvm::StringRef module_name) { + // Rewrite cluster functions into XLA launch ops. + pm.addPass(mlir::TFDevice::CreateXlaRewritePass()); + // Re-run the canonicalizer pass as some cleanup during resource op lifting + // pass opens up some opportunities for canonicalization of cluster ops. + // Specifically, we want to eliminate pass through results from the cluster + // op. + pm.addNestedPass(mlir::createCanonicalizerPass()); + + pm.addNestedPass(mlir::createCSEPass()); + pm.addPass(mlir::createSymbolDCEPass()); +} + +void CreateTPULowerClusterToRuntimeOpsPassPipeline( + OpPassManager& pm, const StandardPipelineOptions& options) { + AddTPULowerClusterToRuntimeOpsPassPipeline(pm, /*module_name=*/""); +} + +void CreateNonTPULowerClusterToRuntimeOpsPassPipeline( + OpPassManager& pm, const StandardPipelineOptions& options) { + AddNonTPULowerClusterToRuntimeOpsPassPipeline(pm, /*module_name=*/""); +} + +// TODO(b/306728216): Move this out of the Bridge component and into a Host +// runtime component. +absl::Status RecordIfErrorStatus(const std::string error_prefix, + std::string bridge_type, + tsl::DeviceType device_type, + absl::Status status) { + if (status.ok()) { + return status; + } + + VLOG(2) << error_prefix << " " << status; + tensorflow::metrics::UpdateTfMlirBridgeFirstPhaseCounter( + bridge_type, + /*bridge_version=*/mlir::TF::kMlirPh1BridgeCounterV2, + device_type.type_string(), + /*fallback_enabled=*/false, + /*result=*/"failure"); + + std::string bridge_subcomponent = "TFXLA_PHASE_ONE_MLIR_TPU_BRIDGE"; + + tsl::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_1, + status); + + if (device_type != DeviceType(DEVICE_TPU_XLA_JIT)) { + bridge_subcomponent = "TFXLA_PHASE_ONE_MLIR_CPU/GPU_BRIDGE"; + } + + tsl::error_logging::Log(mlir::TF::kBridgeComponent, bridge_subcomponent, + status.ToString()) + .IgnoreError(); + + return status; +} + +absl::Status RunLowerClusterToRuntimeOpsPassPipeline( + mlir::ModuleOp module, tsl::DeviceType xla_device_type, + llvm::StringRef module_name) { + PassManager runtime_lowering(module.getContext()); + ::tensorflow::applyTensorflowAndCLOptions(runtime_lowering); + + if (xla_device_type == DeviceType(DEVICE_TPU_XLA_JIT)) { + AddTPULowerClusterToRuntimeOpsPassPipeline(runtime_lowering, module_name); + } else { + AddNonTPULowerClusterToRuntimeOpsPassPipeline(runtime_lowering, + module_name); + } + + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + "runtime_lowering_before"), + module, llvm::StringRef(), &runtime_lowering); + } + + if (VLOG_IS_ON(2) || DEBUG_DATA_DUMPER()->ShouldDump( + module_name.str(), kDebugGroupRuntimeLowering)) { + EnablePassIRPrinting(runtime_lowering, kDebugGroupRuntimeLowering, + module_name); + } + + // Ignore the result since diag_handler consumes it + LogicalResult result = runtime_lowering.run(module); + (void)result; + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + "runtime_lowering_after"), + module, llvm::StringRef(), &runtime_lowering); + } + + std::string bridge_type = xla_device_type == DeviceType(DEVICE_TPU_XLA_JIT) + ? mlir::TF::kMlirPh1BridgeCounterReplicated + : mlir::TF::kMlirPh1BridgeCounterNonReplicated; + auto result_status = diag_handler.ConsumeStatus(); + TF_RETURN_IF_ERROR( + RecordIfErrorStatus(/*error_prefix=*/"lower_cluster_to_runtime", + bridge_type, xla_device_type, result_status)); + + return absl::OkStatus(); +} + +// TODO(b/305211853): Unify the CPU/TPU/GPU Execution Ops and thus these two +// passes should merge together. +void RegisterTPULowerClusterToRuntimeOpsPassPipeline() { + static mlir::PassPipelineRegistration pipeline( + "tfrt-lower-cluster-to-runtime-ops-tpu", + "Run all the passes involved after the clustering transformations from " + "the TF2XLA Bridge. Takes as input a Module with tf_device.cluster ops " + "and outputs TFRT runtime ops such as TPUCompile. This pipeline is for " + "TPU.", + CreateTPULowerClusterToRuntimeOpsPassPipeline); +} + +void RegisterNonTPULowerClusterToRuntimeOpsPassPipeline() { + static mlir::PassPipelineRegistration pipeline( + "tfrt-lower-cluster-to-runtime-ops-non-tpu", + "Run all the passes involved after the clustering transformations from " + "the TF2XLA Bridge. Takes as input a Module with tf_device.cluster ops " + "and outputs TFRT runtime ops such as XlaLaunch. This is for CPU/GPU", + CreateNonTPULowerClusterToRuntimeOpsPassPipeline); +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..64a0d4872078c7d49310c48f3be5b8faa4e1882f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h @@ -0,0 +1,53 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_LOWER_CLUSTER_TO_RUNTIME_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_LOWER_CLUSTER_TO_RUNTIME_OPS_H_ + +#include "absl/base/attributes.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "xla/tsl/framework/device_type.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace tfrt_compiler { + +// Given a MLIR module with tf_device.cluster ops, insert specific Runtime ops +// such as TPUExecute or XlaExecute depending on the device type and specific +// host runtime. Also does some optimization. Will return an error if it fails. +// The output Runtime ops depends on both Device Type and Runtime Host. +// +// Input: +// Tensorflow Dialect MLIR with tf_device.cluster ops and virtual devices. +// xla_device_type - The device type that is being targeted. +// Output: +// Tensorflow Dialect MLIR with Runtime specific ops. All tf_device.cluster +// ops are removed. Physical devices are assigned to ops instead of virtual +// devices. +absl::Status RunLowerClusterToRuntimeOpsPassPipeline( + mlir::ModuleOp module, tsl::DeviceType xla_device_type, + llvm::StringRef module_name = llvm::StringRef()); + +// The same API as RunLowerClusterToRuntimeOpsPassPipeline but as an MLIR pass +// pipeline. +void RegisterTPULowerClusterToRuntimeOpsPassPipeline(); +void RegisterNonTPULowerClusterToRuntimeOpsPassPipeline(); + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_LOWER_CLUSTER_TO_RUNTIME_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..eeef9d17123ef4a9321d1dcc07c6e3f17d2fbd8e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops_test.cc @@ -0,0 +1,199 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h" + +#include +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +using mlir::DialectRegistry; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; +using mlir::func::FuncOp; +using ::tensorflow::monitoring::testing::CellReader; +using tsl::DeviceType; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/"); +} + +static constexpr char kCompilationStreamz[] = + "/tensorflow/core/tf_mlir_bridge_first_phase_v2_count"; + +class LowerClusterToRuntimeOpsTest : public ::testing::Test { + public: + LowerClusterToRuntimeOpsTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + + env_ = Env::Default(); + test_group_name_ = "TestGroup"; + test_dir_ = testing::TmpDir(); + setenv(/*name=*/"TF_DUMP_GRAPH_PREFIX", /*value=*/test_dir_.c_str(), + /*overwrite=*/1); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; + + Env* env_; + std::string test_dir_; + std::string test_group_name_; +}; + +TEST_F(LowerClusterToRuntimeOpsTest, SanityCheck) { + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_TPU_XLA_JIT))); +} + +TEST_F(LowerClusterToRuntimeOpsTest, LowersClusterOpsTPU) { + TF_ASSERT_OK(CreateMlirModule("basic_cluster.mlir")); + + TF_EXPECT_OK(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_TPU_XLA_JIT))); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + bool has_cluster_op = false; + main.walk([&](mlir::tf_device::ClusterOp) { + has_cluster_op = true; + return mlir::WalkResult::interrupt(); + }); + + EXPECT_FALSE(has_cluster_op); +} + +TEST_F(LowerClusterToRuntimeOpsTest, LowersClusterOpsCPU) { + TF_ASSERT_OK(CreateMlirModule("basic_cluster.mlir")); + + TF_EXPECT_OK(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_CPU_XLA_JIT))); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + bool has_cluster_op = false; + main.walk([&](mlir::tf_device::ClusterOp) { + has_cluster_op = true; + return mlir::WalkResult::interrupt(); + }); + + EXPECT_FALSE(has_cluster_op); +} + +TEST_F(LowerClusterToRuntimeOpsTest, LowersClusterOpsGPU) { + TF_ASSERT_OK(CreateMlirModule("basic_cluster.mlir")); + + TF_EXPECT_OK(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_GPU_XLA_JIT))); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + bool has_cluster_op = false; + main.walk([&](mlir::tf_device::ClusterOp) { + has_cluster_op = true; + return mlir::WalkResult::interrupt(); + }); + + EXPECT_FALSE(has_cluster_op); +} + +TEST_F(LowerClusterToRuntimeOpsTest, ErrorsWithBadCluster) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("malformed_cluster.mlir")); + + EXPECT_FALSE(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_TPU_XLA_JIT)) + .ok()); + + EXPECT_EQ( + compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV2, "XLA_TPU_JIT", + "fallback_disabled", "failure"), + 1); +} + +TEST_F(LowerClusterToRuntimeOpsTest, DumpsPipelinePasses) { + std::vector files; + TF_ASSERT_OK(env_->GetChildren(test_dir_, &files)); + EXPECT_THAT(files, ::testing::IsEmpty()); + setenv(/*name=*/"TF_DUMP_GRAPH_NAME_FILTER", /*value=*/"*", /*overwrite=*/1); + setenv(/*name=*/"TF_DUMP_GRAPH_GROUPS", /*value=*/"main,runtime_lowering", + /*overwrite=*/1); + DEBUG_DATA_DUMPER()->LoadEnvvars(); + + TF_ASSERT_OK(CreateMlirModule("basic_cluster.mlir")); + + TF_EXPECT_OK(RunLowerClusterToRuntimeOpsPassPipeline( + *mlir_module_, DeviceType(DEVICE_TPU_XLA_JIT))); + + TF_ASSERT_OK(env_->GetChildren(test_dir_, &files)); + EXPECT_THAT(files, ::testing::SizeIs(15)); +} + +} // namespace +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..ed08cdd22d7bb427318ab7ef19bf0fca599f3c31 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h @@ -0,0 +1,53 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_RUNTIME_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_RUNTIME_PASSES_H_ + +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace TFTPU { + +// Creates a pass that rewrites `tf_device.launch_func` on TPUs into TPU runtime +// ops. +std::unique_ptr> CreateTPURewritePass( + llvm::StringRef module_name = llvm::StringRef()); + +// Creates a pass that adds ops which perform formatting on variables at +// run-time according to compilation result. +std::unique_ptr> +CreateTPUVariableRuntimeReformattingPass(); + +// Creates a pass that merges device variable reads/updates into the surrounded +// TPUExecute node. This allows the execute node to perform in-place variable +// updates. +std::unique_ptr> +CreateTPUMergeVariablesWithExecutePass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_TPUMERGEVARIABLESWITHEXECUTEPASS +#define GEN_PASS_DECL_TPUREWRITEPASS +#define GEN_PASS_DECL_TPUVARIABLERUNTIMEREFORMATTINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h.inc" + +} // namespace TFTPU +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_RUNTIME_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..f16fa389d628a2909a24146881179bf3c06fde6e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.td @@ -0,0 +1,248 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +def TPURewritePass : Pass<"tf-tpu-rewrite", "mlir::ModuleOp"> { + let summary = "Rewrites a `tf_device.cluster_func` on TPUs into TPU runtime operations."; + + let description = [{ + This pass rewrites a `tf_device.cluster_func` operation into a sequence of `tf._TPUCompileMlir` + and `tf.TPUExecute` operations. `tf._TPUCompileMlir` contains a MLIR module that is + functionally equivalent to the function referenced by `tf_device.cluster_func`. + This makes the module to be jit-compiled and executed on TPU. + If it is not possible to rewrite the operation or device assignment fails, + a failure will be returned. + + Note, many parameters to the `tf_device.cluster_func` are omitted in this + and following examples. + For example, a non replicated `tf_device.cluster_func`: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor) { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @func} : (tensor) -> tensor + return + } + ``` + + will be rewritten as: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor) { + %0:2 = "tf_device.launch"() ( { + %compilation_status, %program = "tf._TPUCompileMlir"() {mlir_module = ""} : () -> (tensor, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program : tensor, tensor<3x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>) + "tf_device.launch"() ( { + "tf.TPUCompileSucceededAssert"(%0#0) : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + %1 = "tf_device.launch"() ( { + %2 = "tf.TPUExecute"(%arg0, %0#1) : (tensor, tensor<3x!tf_type.string>) -> tensor + tf_device.return %2 : tensor + }) {device = "/job:worker/replica:0/task:0/device:TPU:0"} : () -> tensor + return + } + ``` + + A replicated `tf_device.cluster_func`: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor, %arg1: tensor) { + %0:2 = tf_device.replicate([%arg0, %arg1] as %ri: tensor) {n = 2 : i32} { + %1 = "tf_device.cluster_func"(%ri) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @func} : (tensor) -> tensor + tf_device.return %1 : tensor + } + return + } + ``` + + will be rewritten as: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor, %arg1: tensor) { + %0:2 = tf_device.replicate([%arg0, %arg1] as %arg2: tensor) {devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:0", "/job:worker/replica:0/task:0/device:TPU:1"], TPU_REPLICATED_HOST_0 = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:CPU:0"]}, n = 2 : i32} { + %1:2 = "tf_device.launch"() ( { + %compilation_status, %program = "tf._TPUCompileMlir"() {mlir_module = ""} : () -> (tensor, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program : tensor, tensor<3x!tf_type.string> + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>) + "tf_device.launch"() ( { + "tf.TPUCompileSucceededAssert"(%1#0) : (tensor) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + %2 = "tf_device.launch"() ( { + %3 = "tf.TPUExecute"(%arg2, %1#1) : (tensor, tensor<3x!tf_type.string>) -> tensor + tf_device.return %3 : tensor + }) {device = "TPU_REPLICATED_CORE_0"} : () -> tensor + tf_device.return %2 : tensor + } + return + } + ``` + + A non replicated `tf_device.cluster_func` with the model parallelism: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor<8xi32>) -> tensor<8xi32> { + %0 = "tf_device.cluster_func"(%arg0) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @func, num_cores_per_replica = 2, input_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"]} : (tensor<8xi32>) -> tensor<8xi32> + return %0 : tensor<8xi32> + } + ``` + + will be rewritten as: + + ```mlir + func @tf_tpu_rewrite(%arg0: tensor<8xi32>) -> tensor<8xi32> { + %0:3 = "tf_device.launch"() ( { + %compilation_status, %program:2 = "tf._TPUCompileMlir"() {mlir_module = ""} : () -> (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program#0, %program#1 : tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string> + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) + "tf_device.launch"() ( { + "tf.TPUCompileSucceededAssert"(%0#0) : (tensor) -> () + tf_device.return + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> () + %1 = "tf_device.parallel_execute"() ( { + %2 = "tf_device.launch"() ( { + %3 = "tf.TPUExecute"(%arg0, %0#1) : (tensor<8xi32>, tensor<3x!tf_type.string>) -> tensor<8xi32> + tf_device.return %3 : tensor<8xi32> + }) {device = "/job:localhost/replica:0/task:0/device:TPU:0"} : () -> tensor<8xi32> + tf_device.return %2 : tensor<8xi32> + }, { + "tf_device.launch"() ( { + "tf.TPUExecute"(%0#2) : (tensor<3x!tf_type.string>) -> () + tf_device.return + }) {device = "/job:localhost/replica:0/task:0/device:TPU:1"} : () -> () + tf_device.return + }) : () -> tensor<8xi32> + return %1 : tensor<8xi32> + } + ``` + }]; + + let options = [ + Option<"tpu_compile_metadata_debug_", "tpu-compile-metadata-debug", "bool", /*default=*/"false", + "Whether to serialize TPUCompileMetadataProto metadata in 'tf._TPUCompileMlir' op as a proto debug string"> + ]; + + let dependentDialects = [ + "mlir::mhlo::MhloDialect", + "mlir::tf_device::TensorFlowDeviceDialect" + ]; + let constructor = "mlir::TFTPU::CreateTPURewritePass()"; +} + +def TPUVariableRuntimeReformattingPass : Pass<"tf-tpu-variable-runtime-reformatting", "ModuleOp"> { + let summary = "Adds device variable formatting op to allow compilation-guided " + "variable formatting."; + let constructor = "TFTPU::CreateTPUVariableRuntimeReformattingPass()"; + let description = [{ + A pass that takes advantage of a loop to add ops that allow the execution to + avoid repeatedly formatting variables back and forth. The desired formatting + is determined by TPU program compilation, so this pass does not include how + to reformat the variables, but only inserts general TPUReshardVariablesOps in + proper places, and TPUReshardVariablesOps interpret the compilation. + + The core idea of this optimization is to keep track of the formatting state + of variables, and when the next desired state does not change, it can avoid + reformatting. We associate a set of variables on a device with a formatting + state, and TPUReshardVariablesOps compares the current state with a desired + state (which can be the compilation result). If they mismatch, + TPUReshardVariablesOp reformats the variables to the desired state; if they + match, TPUReshardVariablesOp is a no-op. + + A major use of this pass is weight-update sharding in data parallelism, so we + require there is a tf_device.replicate in the loop. + + For example, suppose we have a training loop (for simplicity we write the + loop body inine): + + ```mlir + %var0 = ... + %var1 = ... + tf.while (..., %var0, %var1) { + tf_device.replicate ([%var0, %var1] as %rvar) { + %compile:2 = "tf._TPUCompileMlir"() + tf.TPUExecuteAndUpdateVariablesOp(%rvar, compile#1) + } + } + ``` + + This pass will transform it into + + ```mlir + %var0 = ... + %var1 = ... + %state_var0 = ... + %state_var1 = ... + tf.while (..., %var0, %var1, %state_var0, %state_var1) { + tf_device.replicate ([%var0, %var1] as %rvar, + [%state_var0, %state_var1] as %rstate) { + %compile:2 = "tf._TPUCompileMlir"() + tf.TPUReshardVariablesOp(%rvar, %compile#1, %rstate) + tf.TPUExecuteAndUpdateVariablesOp(%rvar, compile#1) + } + } + %default_format = tf.constant() + tf_device.replicate ([%var0, %var1] as %rvar, + [%state_var0, %state_var1] as %rstate) { + tf.TPUReshardVariablesOp(%rvar, %default_format, %rstate) + } + ``` + }]; + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; +} + +def TPUMergeVariablesWithExecutePass : Pass<"tf-tpu-merge-variables-with-execute", "ModuleOp"> { + let summary = "Merges device variable reads and updates into TPU execute ops"; + + let description = [{ + This pass finds on-device resource variable reads and updates surrounding a + `tf.TPUExecute` op and merges them into a `tf.TPUExecuteAndUpdateVariables` + op. This allows the TPU execution to perform more efficient in-place + variable updates. + + For example, + + ```mlir + %0 = "tf.ReadVariableOp"(%arg0) + %1 = "tf.ReadVariableOp"(%arg1) + %2 = "tf.TPUExecute"(%0, %1, %compile) + %3 = "tf.AssignVariableOp"(%arg0, %2) + ``` + + will be transformed into + + ```mlir + %2 = "tf.TPUExecuteAndUpdateVariables"(%arg0, %arg1, %compile) + { device_var_reads_indices = [0, 1], + device_var_updates_indices = [0, -1] } + ```` + + The transformation happens only for on-device variables. The above + transformation requires `%arg0`, `%arg1` to have the same device assignment + as the `TPUExecute` op. + }]; + + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; + + let constructor = "TFTPU::CreateTPUMergeVariablesWithExecutePass()"; +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/basic_cluster.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/basic_cluster.mlir new file mode 100644 index 0000000000000000000000000000000000000000..466aa2e23459423361d5ae950cb92e50d009a3e4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/basic_cluster.mlir @@ -0,0 +1,9 @@ +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @main() { + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/empty_func.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/empty_func.mlir new file mode 100644 index 0000000000000000000000000000000000000000..826ce6955e709a9285f4c917ab8004923075c4ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/empty_func.mlir @@ -0,0 +1,5 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/malformed_cluster.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/malformed_cluster.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f891be7045545283e12ae88cb84e912f7504ec02 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/malformed_cluster.mlir @@ -0,0 +1,10 @@ +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_num_cores_per_replica() { + // expected-error@+1 {{requires attribute 'num_cores_per_replica'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/spmd.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/spmd.mlir new file mode 100644 index 0000000000000000000000000000000000000000..1b71ad63ddd663b2f338a61eec9e3265862a2cdc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/spmd.mlir @@ -0,0 +1,9 @@ +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:2", "/job:localhost/replica:0/task:0/device:TPU:3", "/job:localhost/replica:0/task:0/device:TPU:4", "/job:localhost/replica:0/task:0/device:TPU:5", "/job:localhost/replica:0/task:0/device:TPU:6", "/job:localhost/replica:0/task:0/device:TPU:7"]} { + func.func @main(%arg0: tensor<*xf32> {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) { + "tf_device.cluster_func"(%arg0) <{func = @empty_func}> {_dynamic_arg_index = [], _replication_info = "cluster", _xla_compile_device_type = "TPU", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], host_compute_core = [], input_sharding_configuration = ["{devices=[2,1]0,1}"], num_cores_per_replica = 2 : i64, output_sharding_configuration = [""], padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<*xf32>) -> (tensor<*xf32>) + func.return + } + func.func @empty_func(%arg0: tensor<*xf32>) -> tensor<*xf32> { + func.return %arg0 : tensor<*xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_merge_variables_with_execute.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_merge_variables_with_execute.cc new file mode 100644 index 0000000000000000000000000000000000000000..fd7f71c85b0af6cb35ebe5aef922e131e683fe0e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_merge_variables_with_execute.cc @@ -0,0 +1,639 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/str_join.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +#define DEBUG_TYPE "tf-tpu-merge-variables-with-execute" + +namespace mlir { +namespace TFTPU { + +namespace { +constexpr char kAliasingAttr[] = "tf.aliasing_output"; +constexpr char kDeviceAttr[] = "device"; +constexpr char kFuncDeviceAttr[] = "tf.device"; + +#define GEN_PASS_DEF_TPUMERGEVARIABLESWITHEXECUTEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h.inc" + +class TPUMergeVariablesWithExecutePass + : public impl::TPUMergeVariablesWithExecutePassBase< + TPUMergeVariablesWithExecutePass> { + void runOnOperation() override; +}; + +// Information for a pair of input/output of the TPUExecute op and the +// surrounding read/assign ops. +struct VariableAccessInfo { + int execute_input_index = -1; + int execute_output_index = -1; + Operation* read = nullptr; + Operation* assign = nullptr; +}; + +// Information about all resource accesses to be merged into a TPUExecute op. +struct VariableAccessesForTPUExecute { + // Maps each detected resource to a VariableAccessInfo. Eventually, this will + // contain all values for which we want to merge the accessing ops with a + // TPUExecute op. + llvm::SmallDenseMap per_resource_info; + // The corresponding new output index in TPUExecuteAndUpdateVariables for + // each old output index in TPUExecute. + llvm::SmallVector old_to_new_output_mapping; + // The resources read by ReadVariableOps that are inputs to TPUExecute, + // ordered by the input indices to TPUExecute. + llvm::SmallVector resources_read; + // Operands for the new TPUExecuteAndUpdateVariables. + llvm::SmallVector new_operand_values; +}; + +// Returns true iff the read or assign op associated with `resource` can be +// safely merged. +// +// `resource_ids` contains IDs of all previously accessed resources +// `previous_unknown_resource_access` is true if we had any previous unknown +// resource access. +bool IsResourceSafeForMerge( + Value resource, + const mlir::TF::ResourceAliasAnalysis::Info& resource_analysis_info, + const VariableAccessesForTPUExecute& infos, + const llvm::SmallDenseSet& resource_ids, + bool previous_unknown_resource_access) { + // If we had any unknown resource access before, then we conservatively assume + // that `resource` has been accessed before. + // If `resource` is an unknown resource, then we conservatively assume that + // the same resource has been accessed before. + if (previous_unknown_resource_access || + resource_analysis_info.IsUnknownResource(resource)) + return false; + const auto& ids = resource_analysis_info.GetResourceUniqueIds(resource); + for (int64_t id : ids) { + if (resource_ids.contains(id)) return false; + } + return true; +} + +// Adds IDs of resources which `op` accesses to `resource_ids`. +// Returns true iff op accesses a resource unknown to `resource_analysis_info` +// in which case we have to conservatively assume that any resource might be +// accessed. +bool AddAccessedResourceIds( + Operation* op, + const mlir::TF::ResourceAliasAnalysis::Info& resource_analysis_info, + llvm::SmallDenseSet& resource_ids) { + for (Value operand : TF::filter_resources(op->getOperands())) { + if (resource_analysis_info.IsUnknownResource(operand)) { + VLOG(2) << " unknown access"; + return true; + } + const auto& ids = resource_analysis_info.GetResourceUniqueIds(operand); + VLOG(2) << " accesses following resources: " << absl::StrJoin(ids, ", "); + resource_ids.insert(ids.begin(), ids.end()); + } + return false; +} + +/* Resources may be merged with an execute op when they are on its device or a + * `COMPOSITE`. Note that a `COMPOSITE` represents a set of devices, they + * are typically associated with packed variables. Presently, we assume this + * set spans all the devices. So, a variable on a `COMPOSITE` will have a local + * instance on the execute op's device. + */ +bool IsResourceMergeable(Attribute& resource_attr, Attribute& device_attr) { + return resource_attr && + ((resource_attr == device_attr) || + (llvm::cast(resource_attr).getValue().find("COMPOSITE") != + llvm::StringRef::npos)); +} + +// Finds the variable access info for a TPUExecute op. +// - `check_device` specifies whether it checks the device assignment of the +// variables to match the TPUExecute op. This is optional in some context, +// e.g., guaranteed by replication. +// - `check_same_region` specifies whether the reads/assigns need to be in the +// same region as `execute`. This is needed if `execute` is inside ReplicateOp. +VariableAccessesForTPUExecute BuildVariableAccessInfo( + tf_device::LaunchOp execute_launch, + const mlir::TF::ResourceAliasAnalysis::Info& resource_analysis_info, + bool check_device, bool check_same_region) { + VariableAccessesForTPUExecute var_access_info; + Attribute device_attr = execute_launch.getDeviceAttr(); + if (check_device && !device_attr) return var_access_info; + auto func = execute_launch->getParentOfType(); + + // Track the first read op found, which is used later to check if there are + // assign ops between it and the TPUExecute op. We will exclude reads before + // interfering accesses in a conservative way (see below). We do not consider + // resource accesses in other islands since their ordering is enforced by + // inter-island dependencies. + Operation* first_read = nullptr; + auto execute = cast(execute_launch.GetBody().front()); + auto parallel_execute = llvm::dyn_cast( + execute_launch->getParentOp()); + Operation* execute_parent = + parallel_execute ? parallel_execute.getOperation() : execute_launch; + // Collect all operands of `execute` whose defining ops are variable reads + // that might get merged, and add relevant information to `var_access_info`. + for (auto operand : llvm::enumerate(execute->getOpOperands())) { + var_access_info.new_operand_values.push_back(operand.value().get()); + auto read_op = llvm::dyn_cast_or_null( + operand.value().get().getDefiningOp()); + if (!read_op) continue; + if (check_same_region && + read_op->getParentRegion() != execute_parent->getParentRegion()) + continue; + + auto resource = read_op.getResource(); + if (check_device) { + // TODO(lyandy): Wrap resource ops in tf_device.launch. + if (auto* resource_op = resource.getDefiningOp()) { + auto resource_attr = resource_op->getAttr(kDeviceAttr); + // Check device matching for the node defining the resource. + if (!IsResourceMergeable(resource_attr, device_attr)) continue; + } else { + auto resource_arg = dyn_cast(resource); + assert(resource_arg); + if (resource_arg.getOwner() != &func.front()) continue; + // Check device matching for the argument defining the resource. + auto resource_attr = func.getArgAttrOfType( + resource_arg.getArgNumber(), kFuncDeviceAttr); + if (!IsResourceMergeable(resource_attr, device_attr)) continue; + } + } + + auto emplace_res = var_access_info.per_resource_info.try_emplace( + resource, VariableAccessInfo()); + if (!emplace_res.second) { + LLVM_DEBUG(llvm::dbgs() + << "Skipping execute that has multiple reads of a variable: " + << execute << "\n"); + var_access_info.per_resource_info.shrink_and_clear(); + return var_access_info; + } + + VLOG(2) << "Adding read op to merge candidates: " << debugString(read_op); + auto& info = emplace_res.first->getSecond(); + info.execute_input_index = operand.index(); + info.read = read_op; + var_access_info.new_operand_values[operand.index()] = resource; + var_access_info.resources_read.push_back(resource); + if (!first_read || info.read->isBeforeInBlock(first_read)) { + first_read = info.read; + } + } + + if (!first_read) return var_access_info; + + // Walk backwards from `execute_parent` to `first_read` and remove merge + // candidates based on resource modifications. + llvm::SmallDenseSet resource_ids; + bool previous_unknown_resource_access = false; + for (Operation& op : llvm::reverse(llvm::make_range( + first_read->getIterator(), execute_parent->getIterator()))) { + if (auto read_op = llvm::dyn_cast(&op)) { + VLOG(2) << "Processing read op " << debugString(op); + auto info_it = + var_access_info.per_resource_info.find(read_op.getResource()); + bool is_merge_candidate = + info_it != var_access_info.per_resource_info.end(); + + if (is_merge_candidate && + !IsResourceSafeForMerge(read_op.getResource(), resource_analysis_info, + var_access_info, resource_ids, + previous_unknown_resource_access)) { + VLOG(2) << " removing op from merge candidates"; + int input_index = info_it->getSecond().execute_input_index; + var_access_info.new_operand_values[input_index] = + execute.getOperand(input_index); + var_access_info.per_resource_info.erase(info_it); + } + } + previous_unknown_resource_access |= + AddAccessedResourceIds(&op, resource_analysis_info, resource_ids); + } + + if (var_access_info.per_resource_info.empty()) { + return var_access_info; + } + + // Find outputs that are variable assigns. + Operation* last_assign = nullptr; + llvm::SmallPtrSet all_assigns; + llvm::SmallVector output_merged(execute_launch.getNumResults(), + false); + + auto execute_outputs = + parallel_execute + ? parallel_execute.GetRegionOutputs( + execute_launch->getParentRegion()->getRegionNumber()) + : execute_launch.getResults(); + for (auto execute_output : llvm::enumerate(execute_outputs)) { + // TODO(lyandy): Handle updates to resource writes by remapping to parent + // launch result and checking if launch result is an AssignVariableOp. + auto result = execute_output.value(); + if (!result.hasOneUse()) { + if (VLOG_IS_ON(2)) { + bool any_user_is_assign = false; + for (auto result_user : result.getUsers()) { + any_user_is_assign |= llvm::isa(result_user); + } + if (any_user_is_assign) { + VLOG(2) << "TPUMergeVariablesWithExecutePass: Skipping output " + << execute_output.index() + << " that has more than one user (including an assign):"; + for (auto result_user : result.getUsers()) { + VLOG(2) << " used by: " << debugString(*result_user); + } + } + } + continue; + } + + auto assign_op = llvm::dyn_cast(*result.user_begin()); + if (!assign_op) { + VLOG(2) << "TPUMergeVariablesWithExecutePass: Skipping non-assign op: " + << debugString(*result.user_begin()); + continue; + } + auto resource = assign_op.getResource(); + auto it = var_access_info.per_resource_info.find(resource); + if (it == var_access_info.per_resource_info.end()) continue; + auto& info = it->getSecond(); + if (info.assign) { + LLVM_DEBUG(llvm::dbgs() + << "Skipping execute that has multiple assigns of a variable: " + << execute << "\n"); + var_access_info.per_resource_info.shrink_and_clear(); + return var_access_info; + } + info.execute_output_index = execute_output.index(); + info.assign = assign_op; + if (!last_assign || last_assign->isBeforeInBlock(assign_op)) { + last_assign = assign_op; + } + VLOG(2) << "Adding assign op to merge candidates: " + << debugString(assign_op); + all_assigns.insert(assign_op); + output_merged[execute_output.index()] = true; + } + + if (last_assign != nullptr) { + // Walk forward from `execute_parent` to `last_assign` and remove merge + // candidates based on resource modifications. + resource_ids.clear(); + previous_unknown_resource_access = false; + for (Operation& op : + llvm::make_range(std::next(execute_parent->getIterator()), + std::next(last_assign->getIterator()))) { + if (auto assign_op = llvm::dyn_cast(&op)) { + VLOG(2) << "Processing assign op " << debugString(op); + bool is_merge_candidate = true; + if (all_assigns.count(assign_op) == 0) is_merge_candidate = false; + auto info_it = + var_access_info.per_resource_info.find(assign_op.getResource()); + if (info_it == var_access_info.per_resource_info.end()) + is_merge_candidate = false; + + if (is_merge_candidate && + !IsResourceSafeForMerge(assign_op.getResource(), + resource_analysis_info, var_access_info, + resource_ids, + previous_unknown_resource_access)) { + VLOG(2) << " removing op from merge candidates"; + output_merged[info_it->second.execute_output_index] = false; + info_it->second.execute_output_index = -1; + info_it->second.assign = nullptr; + } + } + previous_unknown_resource_access |= + AddAccessedResourceIds(&op, resource_analysis_info, resource_ids); + } + } + + // Populate var_access_info.old_to_new_output_mapping. + int new_output_index = 0; + var_access_info.old_to_new_output_mapping.resize( + execute_launch.getNumResults()); + for (int i = 0, end = execute_launch.getNumResults(); i < end; ++i) { + if (output_merged[i]) { + var_access_info.old_to_new_output_mapping[i] = -1; + } else { + var_access_info.old_to_new_output_mapping[i] = new_output_index; + ++new_output_index; + } + } + return var_access_info; +} + +// Appends result types of tf_device.parallel_execute from `start` index region +// (inclusive) to `end` index region (exclusive) to `output_types` and returns +// the number of types added. +int AppendTypes(llvm::SmallVectorImpl* output_types, + tf_device::ParallelExecuteOp parallel_execute, int start, + int end) { + const int size_before = output_types->size(); + for (int index = start; index < end; ++index) { + Block& block = parallel_execute.GetRegionBlockWithIndex(index); + auto terminator_operand_types = block.getTerminator()->getOperandTypes(); + output_types->append(terminator_operand_types.begin(), + terminator_operand_types.end()); + } + return output_types->size() - size_before; +} + +// Replaces TPUExecute with TPUExecuteAndUpdateVariables in a +// tf_device.parallel_execute op. +void ReplaceParallelExecute( + tf_device::ParallelExecuteOp parallel_execute, + tf_device::LaunchOp execute_launch, + tf_device::LaunchOp merged_execute_launch, + const VariableAccessesForTPUExecute& var_access_info, OpBuilder* builder) { + Operation* parallel_execute_op = parallel_execute.getOperation(); + + // Collect result types of tf_device.parallel_execute and update region + // result types with the new merged execute result types. + llvm::SmallVector output_types; + const int parallel_execute_num_results = parallel_execute_op->getNumResults(); + output_types.reserve(parallel_execute_num_results); + Region* execute_region = merged_execute_launch->getParentRegion(); + const int region_index = execute_region->getRegionNumber(); + const int num_results_before_region = + AppendTypes(&output_types, parallel_execute, 0, region_index); + // Append updated results from merged execute. + output_types.append(merged_execute_launch.getResultTypes().begin(), + merged_execute_launch.getResultTypes().end()); + const int num_regions = parallel_execute_op->getNumRegions(); + const int num_results_after_region = AppendTypes( + &output_types, parallel_execute, region_index + 1, num_regions); + + builder->setInsertionPoint(parallel_execute); + auto new_parallel_execute = tf_device::ParallelExecuteOp::create( + *builder, parallel_execute.getLoc(), num_regions, output_types); + + // Replace the uses of the original parallel_execute before region containing + // merged execute. + Operation* new_parallel_execute_op = new_parallel_execute.getOperation(); + for (int i = 0; i < num_results_before_region; ++i) + parallel_execute_op->getResult(i).replaceAllUsesWith( + new_parallel_execute_op->getResult(i)); + + // Replace the uses of the original parallel_execute after region containing + // merged execute. The number of results changed in the region containing the + // merged execute, but they should match, so results are replaced starting + // from the ends of both parallel_execute. + const int new_parallel_execute_num_results = + new_parallel_execute_op->getNumResults(); + for (int i = 0; i < num_results_after_region; ++i) + parallel_execute_op->getResult(parallel_execute_num_results - i - 1) + .replaceAllUsesWith(new_parallel_execute_op->getResult( + new_parallel_execute_num_results - i - 1)); + + // Replace the uses of the original parallel_execute for the region containing + // the merged execute. + auto old_region_results = parallel_execute.GetRegionOutputs(region_index); + for (int i = 0, end = var_access_info.old_to_new_output_mapping.size(); + i < end; ++i) { + if (var_access_info.old_to_new_output_mapping[i] < 0) continue; + old_region_results[i].replaceAllUsesWith(new_parallel_execute_op->getResult( + var_access_info.old_to_new_output_mapping[i] + + num_results_before_region)); + } + + // Replace original terminator with new terminator for returning merged + // execute results. + Operation* old_terminator = execute_region->front().getTerminator(); + builder->setInsertionPointToEnd(&execute_region->front()); + tf_device::ReturnOp::create(*builder, old_terminator->getLoc(), + merged_execute_launch.getResults()); + old_terminator->erase(); + + // Remove the original TPUExecute op. + execute_launch.erase(); + + // Move all regions from old parallel_execute to new parallel_execute. + for (auto region : llvm::zip(new_parallel_execute_op->getRegions(), + parallel_execute_op->getRegions())) + std::get<0>(region).takeBody(std::get<1>(region)); + + // Remove the original parallel_execute. + parallel_execute_op->dropAllUses(); + parallel_execute.erase(); +} + +// Replaces TPUExecute with TPUExecuteAndUpdateVariables. +void ReplaceExecute(tf_device::LaunchOp execute_launch, + tf_device::LaunchOp merged_execute_launch, + const VariableAccessesForTPUExecute& var_access_info) { + // Replace the uses. + for (int i = 0, end = var_access_info.old_to_new_output_mapping.size(); + i < end; ++i) { + if (var_access_info.old_to_new_output_mapping[i] < 0) continue; + execute_launch.getResult(i).replaceAllUsesWith( + merged_execute_launch.getResult( + var_access_info.old_to_new_output_mapping[i])); + } + + // Remove the original TPUExecute op. + execute_launch.getOperation()->dropAllUses(); + execute_launch.erase(); +} + +// Merges the variable accesses into one TPUExecute op. +LogicalResult MergeForOneTPUExecute( + tf_device::LaunchOp execute_launch, + const mlir::TF::ResourceAliasAnalysis::Info& resource_analysis_info, + bool check_device, bool check_same_region, OpBuilder* builder) { + VLOG(2) << "MergeForOneTPUExecute: " + << debugString(execute_launch->getName()); + auto var_access_info = BuildVariableAccessInfo( + execute_launch, resource_analysis_info, check_device, check_same_region); + if (var_access_info.per_resource_info.empty()) return success(); + + // Start creating the new TPUExecuteAndUpdateVariables op. + builder->setInsertionPoint(execute_launch); + // Output types. Skip the original outputs for merged assigns. + llvm::SmallVector new_output_types; + int old_output_index = 0; + for (const auto& type : execute_launch.getResultTypes()) { + if (var_access_info.old_to_new_output_mapping[old_output_index] >= 0) { + new_output_types.push_back(type); + } + ++old_output_index; + } + // The attributes for merged variable reads and updates. + llvm::SmallVector device_var_reads_indices; + llvm::SmallVector device_var_updates_indices; + for (auto resource : var_access_info.resources_read) { + auto info_it = var_access_info.per_resource_info.find(resource); + if (info_it == var_access_info.per_resource_info.end()) continue; + device_var_reads_indices.push_back(info_it->second.execute_input_index); + device_var_updates_indices.push_back(info_it->second.execute_output_index); + } + + // Check that all resources are either read or written to. + for (auto it : llvm::enumerate(var_access_info.new_operand_values)) { + Type type = it.value().getType(); + if (isa(type) && + isa(cast(type).getElementType())) { + if (!llvm::is_contained(device_var_reads_indices, it.index()) && + !llvm::is_contained(device_var_updates_indices, it.index())) { + return execute_launch.GetBody().front().emitError("operand #") + << it.index() + << " is a resource that was neither read nor written to; this " + "resource potentially failed to be hoisted"; + } + } + } + + // Create the merged execute and update variables op. + auto merged_execute = TF::TPUExecuteAndUpdateVariablesOp::create( + *builder, execute_launch.getLoc(), new_output_types, + var_access_info.new_operand_values, + llvm::ArrayRef{ + builder->getNamedAttr( + "device_var_reads_indices", + builder->getI64ArrayAttr(device_var_reads_indices)), + builder->getNamedAttr( + "device_var_updates_indices", + builder->getI64ArrayAttr(device_var_updates_indices))}); + + // Wrap in launch for device assignment. + auto merged_execute_launch = tf_device::LaunchOp::create( + *builder, merged_execute.getLoc(), execute_launch.getDeviceAttr(), + merged_execute.getResultTypes()); + merged_execute_launch.getBody().push_back(new Block); + + builder->setInsertionPointToEnd(&merged_execute_launch.GetBody()); + tf_device::ReturnOp::create(*builder, merged_execute.getLoc(), + merged_execute.getResults()); + + merged_execute.getOperation()->moveBefore( + merged_execute_launch.GetBody().getTerminator()); + + if (auto parallel_execute = llvm::dyn_cast( + execute_launch->getParentOp())) + ReplaceParallelExecute(parallel_execute, execute_launch, + merged_execute_launch, var_access_info, builder); + else + ReplaceExecute(execute_launch, merged_execute_launch, var_access_info); + + // Remove the assign ops. + for (const auto& entry : var_access_info.per_resource_info) { + const auto& info = entry.getSecond(); + if (info.assign) info.assign->erase(); + } + + // Remove the read ops if they have no more uses. + for (const auto& entry : var_access_info.per_resource_info) { + const auto& info = entry.getSecond(); + if (info.read->use_empty()) info.read->erase(); + } + return success(); +} + +// Checks if an ops parent is a tf_device.parallel_execute and the region the +// op is in is perfectly wrapped. +bool ParentParallelExecuteWrapsSingleOp(Operation* op) { + auto parallel_execute = + llvm::dyn_cast(op->getParentOp()); + if (!parallel_execute) return true; + + return parallel_execute.RegionWrapsSingleOp( + op->getParentRegion()->getRegionNumber()); +} + +void TPUMergeVariablesWithExecutePass::runOnOperation() { + ModuleOp module = getOperation(); + mlir::TF::ResourceAliasAnalysis resource_analysis(module); + module.walk([&](func::FuncOp func) { + const auto& resource_analysis_info = + resource_analysis.GetAnalysisForFunc(func); + // Find all the executes first, since we will mutate the nodes around each + // execute. + llvm::SmallVector execute_launches; + func.walk([&](tf_device::LaunchOp op) { + if (op.WrapsSingleOp() && + llvm::isa(op.GetBody().front()) && + ParentParallelExecuteWrapsSingleOp(op)) + execute_launches.push_back(op); + }); + + for (auto execute_launch : execute_launches) { + OpBuilder builder(&getContext()); + const bool parent_is_replicate = + llvm::isa(execute_launch->getParentOp()) || + (llvm::isa( + execute_launch->getParentOp()) && + llvm::isa( + execute_launch->getParentOp()->getParentOp())); + + // If this is inside a tf_device::ReplicateOp, the variables are + // guaranteed to be on the same device as the TPUExecute op. Skip device + // checking in that case, but we need to check that we are only merging + // reads/assigns that are also in this replicated region. + if (failed(MergeForOneTPUExecute( + execute_launch, resource_analysis_info, + /*check_device=*/!parent_is_replicate, + /*check_same_region=*/parent_is_replicate, &builder))) { + signalPassFailure(); + return; + } + } + }); +} + +} // namespace + +std::unique_ptr> +CreateTPUMergeVariablesWithExecutePass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..bafea960104ea660f442baed59d1fb2a75d0cfa4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.cc @@ -0,0 +1,256 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h" + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "xla/xla.pb.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace mlir { +namespace TFTPU { +namespace { +constexpr char kStepMarkerLocationAttr[] = "step_marker_location"; +constexpr char kUseXlaSpmdAttr[] = "use_spmd_for_xla_partitioning"; +constexpr char kUseShardyAttr[] = "use_shardy_partitioner"; + +constexpr char kBadStringArrayElementMsg[] = + "bad '{0}' attribute at index {1}, not a string"; +constexpr char kBadArrayElementMsg[] = + "bad '{0}' attribute at index {1} with value '{2}': failed to parse to {3}"; +constexpr char kBadArrayAttrLengthMsg[] = + "bad '{0}' attribute, expected array attribute of size {1}, got size {2}"; + +// Creates a missing attribute error message. +std::string CreateMissingAttributeMsg(llvm::StringRef attribute) { + return llvm::formatv("requires attribute '{0}'", attribute).str(); +} + +// Populates a TPUCompileMetadataProto with StepMarkerLocation from a +// `tf_device::ClusterFuncOp`. +LogicalResult SetMetadataProtoStepMarkerLocation( + tf_device::ClusterFuncOp op, + tensorflow::tpu::TPUCompileMetadataProto* metadata) { + auto step_marker_location = + op->getAttrOfType(kStepMarkerLocationAttr); + if (!step_marker_location) + return op.emitOpError(CreateMissingAttributeMsg(kStepMarkerLocationAttr)); + + // Default to `STEP_MARK_AT_ENTRY` for step marker location if attribute is + // empty. + xla::DebugOptions::StepMarkerLocation location = + xla::DebugOptions::STEP_MARK_AT_ENTRY; + if (!step_marker_location.getValue().empty() && + !xla::DebugOptions::StepMarkerLocation_Parse( + std::string(step_marker_location.getValue()), &location)) + return op.emitOpError(llvm::formatv("bad '{0}' attribute with value '{1}'", + kStepMarkerLocationAttr, + step_marker_location.getValue())); + + metadata->set_step_marker_location(location); + + return success(); +} + +// Parses a xla::OpSharding from a string attribute. +LogicalResult SetOpSharding(Operation* op, Attribute attr, llvm::StringRef name, + int index, xla::OpSharding* sharding_ptr) { + auto sharding_attr = mlir::dyn_cast(attr); + if (!sharding_attr) + return op->emitOpError( + llvm::formatv(kBadStringArrayElementMsg, name, index)); + if (tensorflow::DecodeShardingAttribute(sharding_attr, *sharding_ptr) + .failed()) { + return op->emitOpError(llvm::formatv(kBadArrayElementMsg, name, index, + sharding_attr.getValue(), + "xla::OpSharding")); + } + return success(); +} + +// Populates a TPUCompileMetadataProto with argument types and sharding from a +// `tf_device::ClusterFuncOp`. +LogicalResult SetMetadataProtoArgs( + tf_device::ClusterFuncOp op, + tensorflow::tpu::TPUCompileMetadataProto* metadata) { + auto input_shardings = + op->getAttrOfType(tensorflow::kInputShardingAttr); + if (!input_shardings) + return op.emitOpError( + CreateMissingAttributeMsg(tensorflow::kInputShardingAttr)); + + if (input_shardings.size() != op.getNumOperands()) + return op.emitOpError( + llvm::formatv(kBadArrayAttrLengthMsg, tensorflow::kInputShardingAttr, + op.getNumOperands(), input_shardings.size())); + + // Set args metadata in proto. + mlir::StringAttr replication_attr_name = mlir::StringAttr::get( + op.getContext(), "mhlo.is_same_data_across_replicas"); + + auto dynamic_arg_idx = op->getAttrOfType(TF::kDynamicArgIndexAttr); + llvm::SmallSet dynamic_arg_idx_set; + if (dynamic_arg_idx) { + for (auto idx : dynamic_arg_idx.getValue()) { + dynamic_arg_idx_set.insert(mlir::dyn_cast(idx).getInt()); + } + } + + for (auto operand_type_and_idx : llvm::enumerate(op.getOperandTypes())) { + Type operand_type = operand_type_and_idx.value(); + int index = operand_type_and_idx.index(); + tensorflow::tpu::TPUCompileMetadataProto::Arg* arg = metadata->add_args(); + tensorflow::DataType dtype; + absl::Status status = tensorflow::ConvertToDataType(operand_type, &dtype); + if (!status.ok()) + return op.emitOpError( + llvm::formatv("failed to determine operand type at index {0}: {1}", + index, status.message())); + + arg->set_dtype(dtype); + // TODO(lyandy): Support other arg kinds. + if (dtype == tensorflow::DT_RESOURCE) + arg->set_kind(tensorflow::tpu::TPUCompileMetadataProto::Arg::VARIABLE); + else + arg->set_kind(tensorflow::tpu::TPUCompileMetadataProto::Arg::PARAMETER); + + // Populate argument shapes. + *arg->mutable_shape() = tensorflow::TensorShapeProto(); + if (auto ranked_tensor_type = + mlir::dyn_cast(operand_type)) { + tensorflow::TensorShapeProto shape_proto; + ConvertToTensorShapeProto(ranked_tensor_type.getShape(), &shape_proto); + *arg->mutable_shape() = std::move(shape_proto); + } else { + arg->mutable_shape()->set_unknown_rank(true); + } + + if (failed(SetOpSharding(op, input_shardings.getValue()[index], + tensorflow::kInputShardingAttr, index, + arg->mutable_sharding()))) + return failure(); + + // Populate set_is_same_data_across_replicas + // Note: this information is duplicated and can be removed from the proto + // and here once MLIR bridge phase 2 doesn't fallback to the old bridge. + auto attr = op.getFuncOp().getArgAttrOfType( + index, replication_attr_name); + arg->set_is_same_data_across_replicas(attr != nullptr && attr.getValue()); + + // Currently only support first dimension to be bounded dynamic. + arg->mutable_is_bounded_dynamic_dim()->Add( + dynamic_arg_idx_set.contains(index)); + } + + return success(); +} + +// Populates a TPUCompileMetadataProto with result sharding from a +// `tf_device::ClusterFuncOp`. +LogicalResult SetMetadataProtoRetvals( + tf_device::ClusterFuncOp op, + tensorflow::tpu::TPUCompileMetadataProto* metadata) { + auto output_shardings = + op->getAttrOfType(tensorflow::kOutputShardingAttr); + if (!output_shardings) + return op.emitOpError( + CreateMissingAttributeMsg(tensorflow::kOutputShardingAttr)); + + if (output_shardings.size() != op.getNumResults()) + return op.emitOpError( + llvm::formatv(kBadArrayAttrLengthMsg, tensorflow::kOutputShardingAttr, + op.getNumResults(), output_shardings.size())); + + // Set retvals metadata in proto. + for (auto output_sharding_and_idx : llvm::enumerate(output_shardings)) + if (failed(SetOpSharding(op, output_sharding_and_idx.value(), + tensorflow::kOutputShardingAttr, + output_sharding_and_idx.index(), + metadata->add_retvals()->mutable_sharding()))) + return failure(); + + return success(); +} + +} // namespace + +// Populates a TPUCompileMetadataProto from attributes of a +// `tf_device::ClusterFuncOp`. If any necessary attributes are missing from the +// op, a failure will be returned. +// TODO(lyandy): Support session handle and guaranteed consts. +LogicalResult SetMetadataProtoFromClusterFuncOp( + tf_device::ClusterFuncOp op, int num_replicas, int num_cores_per_replica, + std::optional&& xla_device_assignment, + tensorflow::tpu::TPUCompileMetadataProto* metadata) { + if (auto options_attr = + op->getAttrOfType("tpu_compile_options_proto")) { + if (!metadata->mutable_compile_options()->ParseFromString( + absl::string_view(options_attr.data(), options_attr.size()))) { + return failure(); + } + } + metadata->set_num_replicas(num_replicas); + metadata->set_num_cores_per_replica(num_cores_per_replica); + + if (failed(SetMetadataProtoStepMarkerLocation(op, metadata))) + return failure(); + + if (xla_device_assignment.has_value()) + *metadata->mutable_device_assignment() = + std::move(xla_device_assignment.value()); + + auto use_spmd_attr = op->getAttrOfType(kUseXlaSpmdAttr); + if (!use_spmd_attr) + return op.emitOpError(CreateMissingAttributeMsg(kUseXlaSpmdAttr)); + metadata->set_use_spmd_for_xla_partitioning(use_spmd_attr.getValue()); + + if (auto use_shardy_attr = op->getAttrOfType(kUseShardyAttr)) + metadata->set_use_shardy_partitioner(use_shardy_attr.getValue()); + + if (failed(SetMetadataProtoArgs(op, metadata))) return failure(); + + return SetMetadataProtoRetvals(op, metadata); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..eeac5cc29b27b982d9df3466f273941ef3447917 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h @@ -0,0 +1,43 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_TPU_METADATA_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_TPU_METADATA_UTILS_H_ + +#include + +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "xla/xla.pb.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace mlir { +namespace TFTPU { + +// Populates a TPUCompileMetadataProto from attributes of a +// `tf_device::ClusterFuncOp`. If any necessary attributes are missing from the +// op, a failure will be returned. +// TODO(lyandy): Support session handle and guaranteed consts. +LogicalResult SetMetadataProtoFromClusterFuncOp( + tf_device::ClusterFuncOp op, int num_replicas, int num_cores_per_replica, + std::optional&& xla_device_assignment, + tensorflow::tpu::TPUCompileMetadataProto* metadata); +} // namespace TFTPU +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_HOST_RUNTIME_TPU_METADATA_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..6dbc46f25a485780667fe1eed25407a2e6c7ade2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils_test.cc @@ -0,0 +1,182 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h" + +#include +#include +#include + +#include +#include +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tsl/platform/protobuf.h" + +namespace mlir { +namespace TFTPU { +namespace { + +using mlir::DialectRegistry; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; + +// TODO(b/229726259): Make EqualsProto available in OSS +class ProtoStringMatcher { + public: + explicit ProtoStringMatcher(const tsl::protobuf::Message& expected) + : expected_(expected.SerializeAsString()) {} + + template + bool MatchAndExplain(const Message& p, testing::MatchResultListener*) const { + return p.SerializeAsString() == expected_; + } + + void DescribeTo(::std::ostream* os) const { *os << expected_; } + void DescribeNegationTo(::std::ostream* os) const { + *os << "not equal to expected message: " << expected_; + } + + private: + const std::string expected_; +}; + +inline ::testing::PolymorphicMatcher EqualsProto( + const tsl::protobuf::Message& x) { + return ::testing::MakePolymorphicMatcher(ProtoStringMatcher(x)); +} + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/testdata/"); +} + +class TpuMetadataUtilsTest : public ::testing::Test { + public: + TpuMetadataUtilsTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::StatusOr> GetClusterFuncOps( + absl::string_view mlir_module_filename) { + TF_RETURN_IF_ERROR(CreateMlirModule(mlir_module_filename)); + std::vector cluster_func_ops; + + mlir_module_->walk([&](mlir::tf_device::ClusterFuncOp op) { + cluster_func_ops.push_back(op); + }); + return cluster_func_ops; + } + + private: + absl::Status CreateMlirModule(absl::string_view mlir_module_filename) { + std::string mlir_module_path = + absl::StrCat(TestDataPath(), mlir_module_filename); + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(TpuMetadataUtilsTest, SingleDevice) { + TF_ASSERT_OK_AND_ASSIGN(auto cluster_func_ops, + GetClusterFuncOps("basic_cluster.mlir")); + mlir::tf_device::ClusterFuncOp cluster_func_op = cluster_func_ops.front(); + + tensorflow::tpu::TPUCompileMetadataProto compile_metadata; + + ASSERT_TRUE(mlir::succeeded(SetMetadataProtoFromClusterFuncOp( + cluster_func_op, + /*num_replicas=*/1, /*num_cores_per_replica=*/1, {}, &compile_metadata))); + + tensorflow::tpu::TPUCompileMetadataProto expected_compile_metadata; + ASSERT_TRUE(tsl::protobuf::TextFormat::ParseFromString( + R"pb( + num_replicas: 1 num_cores_per_replica: 1 + )pb", + &expected_compile_metadata)); + + EXPECT_THAT(compile_metadata, EqualsProto(expected_compile_metadata)); +} + +TEST_F(TpuMetadataUtilsTest, spmd) { + TF_ASSERT_OK_AND_ASSIGN(auto cluster_func_ops, + GetClusterFuncOps("spmd.mlir")); + mlir::tf_device::ClusterFuncOp cluster_func_op = cluster_func_ops.front(); + + tensorflow::tpu::TPUCompileMetadataProto compile_metadata; + + ASSERT_TRUE(mlir::succeeded(SetMetadataProtoFromClusterFuncOp( + cluster_func_op, + /*num_replicas=*/1, /*num_cores_per_replica=*/2, {}, &compile_metadata))); + + tensorflow::tpu::TPUCompileMetadataProto expected_compile_metadata; + ASSERT_TRUE(tsl::protobuf::TextFormat::ParseFromString( + R"pb( + args { + dtype: DT_FLOAT + shape { unknown_rank: true } + kind: PARAMETER + sharding { + type: OTHER + tile_assignment_dimensions: 2 + tile_assignment_dimensions: 1 + tile_assignment_devices: 0 + tile_assignment_devices: 1 + } + is_bounded_dynamic_dim: false + } + retvals { sharding {} } + num_replicas: 1 + num_cores_per_replica: 2 + use_spmd_for_xla_partitioning: true + compile_options {} + )pb", + &expected_compile_metadata)); + + EXPECT_THAT(compile_metadata, EqualsProto(expected_compile_metadata)); +} + +} // namespace +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_rewrite_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_rewrite_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..8a4a5297211d40804dc8a6ca1ede3730844205dc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_rewrite_pass.cc @@ -0,0 +1,714 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/xla.pb.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/fingerprint.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TFTPU { + +constexpr char kStepMarkerLocationAttr[] = "step_marker_location"; +constexpr char kDevicesAttr[] = "devices"; +constexpr char kVersionsAttr[] = "tf.versions"; +constexpr char kUseXlaSpmdAttr[] = "use_spmd_for_xla_partitioning"; + +constexpr char kBadStringArrayElementMsg[] = + "bad '{0}' attribute at index {1}, not a string"; +constexpr char kBadArrayElementMsg[] = + "bad '{0}' attribute at index {1} with value '{2}': failed to parse to {3}"; +constexpr char kBadArrayAttrLengthMsg[] = + "bad '{0}' attribute, expected array attribute of size {1}, got size {2}"; + +namespace { + +#define GEN_PASS_DEF_TPUREWRITEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h.inc" + +struct TPURewritePass : public impl::TPURewritePassBase { + explicit TPURewritePass(llvm::StringRef _module_name) + : module_name(_module_name) {} + + void runOnOperation() override; + + llvm::StringRef module_name; +}; + +// Creates a missing attribute error message. +std::string CreateMissingAttributeMsg(llvm::StringRef attribute) { + return llvm::formatv("requires attribute '{0}'", attribute).str(); +} + +LogicalResult EncapsulateFuncAndSerialize(const std::string& module_name, + func::FuncOp entry_func, + std::string* serialized_func_module) { + ModuleOp module = entry_func->getParentOfType(); + SymbolTable entry_module_table(module); + llvm::SmallVector referenced({entry_func}); + + // Create a new module to hold func and all referenced functions. + OwningOpRef module_for_func = + ModuleOp::create(mlir::UnknownLoc::get(entry_func.getContext()), + absl::StrCat("module_", module_name)); + auto parent_module = entry_func->getParentOfType(); + auto versions_attr = parent_module->getAttr(kVersionsAttr); + if (!versions_attr) + return parent_module.emitError(CreateMissingAttributeMsg(kVersionsAttr)); + + module_for_func.get().getOperation()->setAttr(kVersionsAttr, versions_attr); + SymbolTable symbol_table(module_for_func.get()); + + while (!referenced.empty()) { + auto func = referenced.pop_back_val(); + + // Skip functions that have already been cloned into new module. + if (symbol_table.lookup(func.getName())) continue; + + // Find any SymbolRefAttr in func that maps to a FuncOp. We need to clone + // all found FuncOps to new_module to make sure new_module is + // self-contained. + std::optional uses = + SymbolTable::getSymbolUses(func); + assert(uses && "expected to be able to collect symbol uses"); + for (SymbolTable::SymbolUse use : *uses) { + func::FuncOp referenced_func = entry_module_table.lookup( + mlir::cast(use.getSymbolRef()).getValue()); + + // Skip Symbols that do not map to a function. + if (!referenced_func) continue; + + referenced.emplace_back(referenced_func); + } + + auto clone = func.clone(); + if (clone.getName() == entry_func.getName()) { + // We can simply change name of TPU program's main function because there + // should be no other reference to it. + clone.setName(StringAttr::get(clone.getContext(), "main")); + clone.setPublic(); + } else { + clone.setPrivate(); + } + symbol_table.insert(clone); + } + + *serialized_func_module = + tensorflow::SerializeMlirModule(module_for_func.get()); + return success(); +} + +// Create a `tf._TPUCompileMlir` that contains a MLIR module that is +// functionally equivalent to the function referenced by cluster_func. +Operation* BuildCompileOp( + llvm::StringRef module_name, tf_device::ClusterFuncOp cluster_func, + int num_replicas, int num_cores_per_replica, + llvm::StringRef compilation_device, + std::optional&& xla_device_assignment, + OpBuilder* builder, bool tpu_compile_metadata_debug) { + // Set metadata from attributes. + tensorflow::tpu::TPUCompileMetadataProto metadata; + if (!module_name.empty()) metadata.set_module_name(module_name.str()); + if (failed(mlir::TFTPU::SetMetadataProtoFromClusterFuncOp( + cluster_func, num_replicas, num_cores_per_replica, + std::move(xla_device_assignment), &metadata))) + return nullptr; + + // Build a shape op for each input to cluster_func. + // TODO(b/139377366): When shape inference is ready, we can use compile time + // shape inference to get inputs that have static shapes and only use shape + // ops for the rest. + llvm::SmallVector compile_op_operands; + compile_op_operands.reserve(cluster_func.getNumOperands()); + + for (auto operand_and_idx : llvm::enumerate(cluster_func.getOperands())) { + // Skip adding shape op for operands that have static shapes. + tensorflow::PartialTensorShape shape( + metadata.args(operand_and_idx.index()).shape()); + if (shape.IsFullyDefined()) continue; + + VLOG(1) << "Building compile op for module_name: " << module_name.str() + << " dynamic shape for operand index: " << operand_and_idx.index() + << " metadata: " + << metadata.args(operand_and_idx.index()).DebugString(); + + auto shape_op = TF::ShapeOp::create( + *builder, cluster_func.getLoc(), + tensorflow::GetTypeFromTFTensorShape({-1}, builder->getIntegerType(64)), + operand_and_idx.value()); + compile_op_operands.emplace_back(shape_op.getResult()); + } + + FlatSymbolRefAttr func_attr = cluster_func.getFuncAttr(); + func::FuncOp func = + cluster_func->getParentOfType().lookupSymbol( + func_attr.getValue()); + + std::string txt_module; + if (failed(EncapsulateFuncAndSerialize( + module_name.empty() ? "unknown_graph" : module_name.str(), func, + &txt_module))) + return nullptr; + + auto compilation_status_type = + RankedTensorType::get({}, builder->getType()); + auto program_type = + RankedTensorType::get({3}, builder->getType()); + + // Add MLIR module's fingerprint to compile metadata. + uint64_t mlir_fingerprint = tensorflow::Fingerprint64(txt_module); + metadata.set_mlir_fingerprint(mlir_fingerprint); + + std::string txt_metadata; + if (tpu_compile_metadata_debug) { + ::tensorflow::protobuf::TextFormat::Printer printer; + printer.SetExpandAny(true); + printer.PrintToString(metadata, &txt_metadata); + } else { + metadata.SerializeToString(&txt_metadata); + } + + auto compile_op = TF::_TPUCompileMlirOp::create( + *builder, cluster_func.getLoc(), + /*compilation_status=*/compilation_status_type, /*program=*/ + llvm::SmallVector(num_cores_per_replica, program_type), + compile_op_operands, txt_module, txt_metadata); + + return tensorflow::WrapOpInLaunch(builder, compile_op.getLoc(), compile_op, + compilation_device); +} + +// Assigns explicit devices to replicate op. An aliased device is created per +// core, and all replica devices per core are grouped together. +void AssignDevicesToReplicate( + tf_device::ReplicateOp replicate, + llvm::ArrayRef> + tpu_devices, + OpBuilder* builder) { + if (!replicate) return; + + const int num_replicas = tpu_devices.size(); + const int num_cores_per_replica = tpu_devices.front().size(); + + llvm::SmallVector device_attrs; + for (int core = 0; core < num_cores_per_replica; ++core) { + llvm::SmallVector devices_by_core; + devices_by_core.reserve(num_replicas); + llvm::SmallVector hosts_by_core; + hosts_by_core.reserve(num_replicas); + for (int replica = 0; replica < num_replicas; ++replica) { + devices_by_core.push_back(tpu_devices[replica][core].device); + hosts_by_core.push_back(tpu_devices[replica][core].host); + } + + device_attrs.push_back( + builder->getNamedAttr(tensorflow::GetDeviceAliasForLogicalCore(core), + builder->getStrArrayAttr(devices_by_core))); + + // For data parallelism, also add replicated host devices, as these are + // necessary for outside compilation. + device_attrs.push_back(builder->getNamedAttr( + tensorflow::GetDeviceAliasForHostOfLogicalCore(core), + builder->getStrArrayAttr(hosts_by_core))); + } + + replicate->setAttr(kDevicesAttr, builder->getDictionaryAttr(device_attrs)); +} + +// Creates a `tf.TPUExecute` op that executes TPU program. +LogicalResult BuildExecuteOp( + const int core_id, llvm::ArrayRef output_sharding_config, + llvm::ArrayRef inputs, tf_device::ClusterFuncOp cluster_func, + OpBuilder* builder, TF::TPUExecuteOp* execute_op) { + // TODO(b/139377366): Need to snapshot all resource variable inputs in + // follow-up CLs. + llvm::SmallVector output_types; + llvm::SmallVector cluster_to_core_index; + auto result = tensorflow::GetOutputTypesForLogicalDeviceComputation( + core_id, output_sharding_config, cluster_func, &output_types, + &cluster_to_core_index); + if (failed(result)) return failure(); + + // TPUExecute has same output types as cluster_func. + *execute_op = TF::TPUExecuteOp::create(*builder, cluster_func.getLoc(), + output_types, inputs); + auto producer_name_attr = cluster_func->getAttr("_producer_name"); + if (producer_name_attr) + (*execute_op)->setAttr("_producer_name", producer_name_attr); + return success(); +} + +// Given a `ParallelExecute`, replace it with a new `ParallelExecute`. The +// new `ParallelExecute` will replace the child that contains the +// `ClusterFunc` with `num_cores_per_replica` children. It keep other children +// the same. Return values from the child with the `ClusterFunc` will be +// duplicated `num_cores_per_replica` times. +LogicalResult AddToParallelExecuteOp( + llvm::ArrayRef> + tpu_devices, + llvm::ArrayRef output_sharding_config, + llvm::SmallVectorImpl>* cluster_to_core_index, + int num_results_pre_cluster, Operation* compile_op, + tf_device::ClusterFuncOp cluster_func, OpBuilder* builder, + tf_device::ParallelExecuteOp old_parallel_execute, + tf_device::ParallelExecuteOp* new_parallel_execute, int* cluster_idx) { + const int num_cores_per_replica = tpu_devices.front().size(); + // parallel_execute op returns concatenated list of return values of + // all its regions. + // + // TODO(b/149102702): Correctly map inputs to parallel_execute op via + // identifying xla_sharding op in the cluster_func function. + const auto cluster_result_types = cluster_func.getResultTypes(); + llvm::SmallVector concatenated_output_types; + concatenated_output_types.reserve(num_results_pre_cluster + + cluster_result_types.size() * + num_cores_per_replica); + for (mlir::Region& region : old_parallel_execute.getRegions()) { + if (!llvm::isa(region.front().front())) { + for (Type t : region.front().front().getResultTypes()) + concatenated_output_types.emplace_back(t); + } + } + + for (int core = 0; core < num_cores_per_replica; ++core) { + cluster_to_core_index->emplace_back(llvm::SmallVector()); + llvm::SmallVector output_types; + auto result = tensorflow::GetOutputTypesForLogicalDeviceComputation( + core, output_sharding_config, cluster_func, &output_types, + &(*cluster_to_core_index)[core]); + if (failed(result)) return failure(); + + for (Type t : output_types) concatenated_output_types.emplace_back(t); + } + + *cluster_idx = tensorflow::MovePreservedParallelExecuteChildren( + num_cores_per_replica, concatenated_output_types, builder, cluster_func, + old_parallel_execute, new_parallel_execute); + + // Extract inputs for each block of the parallel_execute op. The i-th + // element in the list represents the input lists to TPU computation for + // i-th logical core. + llvm::SmallVector, 4> input_list; + builder->setInsertionPoint(*new_parallel_execute); + auto result = tensorflow::ExtractInputsForLogicalDevices( + num_cores_per_replica, cluster_func, builder, &input_list); + if (failed(result)) return failure(); + + const bool replicated = tpu_devices.size() != 1; + // For each logical core, create a region with TPUExecute op. + assert(input_list.size() == num_cores_per_replica); + for (int core = 0; core < num_cores_per_replica; ++core) { + auto& block = + new_parallel_execute->GetRegionBlockWithIndex((*cluster_idx) + core); + builder->setInsertionPointToEnd(&block); + + // Create Execute op. + // + // TODO(b/148913294): Identify inputs/return values specific to each + // logical core TPU execution by parsing xla_sharding op in + // cluster_func. + auto execute_inputs = input_list[core]; + execute_inputs.emplace_back(compile_op->getResult(core + 1)); + + TF::TPUExecuteOp execute; + result = BuildExecuteOp(core, output_sharding_config, execute_inputs, + cluster_func, builder, &execute); + if (failed(result)) return failure(); + + // If computation is replicated, use aliased device. Otherwise there is only + // one execution device per core and the device is assigned to the execute + // op. + std::string device = replicated + ? tensorflow::GetDeviceAliasForLogicalCore(core) + : tpu_devices.front()[core].device; + auto block_launch_op = tensorflow::WrapOpInLaunch( + builder, block.getParent()->getLoc(), execute, device); + + tf_device::ReturnOp::create(*builder, block.getParent()->getLoc(), + block_launch_op.getResults()); + } + + return success(); +} + +// Creates a `tf.TPUCompileSucceededAssert` operation that parses compilation +// status of `compile_op` to check whether compilation is successful. +void BuildTPUCompileSucceededAssertOp(Operation* compile_op, + Operation* result_id, + llvm::StringRef compilation_device, + OpBuilder* builder) { + auto assert_op = TF::TPUCompileSucceededAssertOp::create( + *builder, compile_op->getLoc(), result_id->getResult(0)); + tensorflow::WrapOpInLaunch(builder, compile_op->getLoc(), assert_op, + compilation_device); +} + +LogicalResult CheckTPUPartitionedInputAndOutputAreValid( + tf_device::ClusterFuncOp cluster, + tf_device::ParallelExecuteOp parallel_execute) { + for (auto cluster_result : parallel_execute.getExecuteOutputs()) { + for (Operation* user : + llvm::make_early_inc_range(cluster_result.getUsers())) { + // Check that user has no outputs that are TPUPartitionedOutputV2 + for (auto result : user->getResults()) { + for (Operation* user : llvm::make_early_inc_range(result.getUsers())) { + if (llvm::isa(user)) { + user->emitError() << "Input of TPUPartitionedOutputV2 must " + << "be in tpu computation."; + return failure(); + } + } + } + } + } + for (auto cluster_operand : cluster.getOperands()) { + Operation* def = cluster_operand.getDefiningOp(); + // This pass assumes that a TPUPartitionedInputV2 is preceeded by + // ReadVariable ops, and not vice versa. An earlier pass, + // TPUResourceReadsWritesPartitioning, should have ensured this + // precondition. + if (!def) continue; + for (auto operand : def->getOperands()) { + Operation* def_of_read = operand.getDefiningOp(); + if (llvm::isa_and_nonnull(def_of_read)) { + def_of_read->emitError() << "Output of TPUPartitionedInputV2 must " + << "be in tpu computation."; + return failure(); + } + } + } + return success(); +} + +LogicalResult CheckParallelExecuteConstainsValidNonClusterProcess( + tf_device::ParallelExecuteOp parallel_execute) { + int num_pre_cluster_regions = 0; + int num_post_cluster_regions = 0; + int num_cluster_regions = 0; + for (mlir::Region& region : parallel_execute.getRegions()) { + if (llvm::isa(region.front().front())) { + if (num_cluster_regions == 0) { + num_pre_cluster_regions++; + } else { + num_post_cluster_regions++; + } + } else { + num_cluster_regions++; + } + } + if (num_post_cluster_regions > 0) { + return failure(); + } + if (num_pre_cluster_regions > 2) { + return failure(); + } + return success(); +} + +int GetNumResultsPreCluster(tf_device::ParallelExecuteOp parallel_execute) { + int num_results_pre_cluster = 0; + for (mlir::Region& region : parallel_execute.getRegions()) { + if (llvm::isa(region.front().front())) { + num_results_pre_cluster = region.front().front().getResultTypes().size(); + } + } + return num_results_pre_cluster; +} + +LogicalResult Rewrite( + llvm::StringRef module_name, tf_device::ClusterFuncOp cluster_func, + llvm::ArrayRef devices, + ArrayRef compilation_result, OpBuilder* builder, + bool tpu_compile_metadata_debug) { + // Fetch the ParallelExecute parent of `cluster_func`, or create it if it does + // not exist. + tf_device::ParallelExecuteOp old_parallel_execute = + cluster_func->getParentOfType(); + if (old_parallel_execute && + cluster_func->getParentOp() != old_parallel_execute) { + cluster_func->emitError() << "The ParallelExecute ancestor of a " + "ClusterFunc must be its direct parent."; + return failure(); + } + if (!old_parallel_execute) + old_parallel_execute = TF::BuildParallelExecuteOp(cluster_func, builder); + + // check TPUPartitionedInputV2 and TPUPartitionedOutputV2 are in valid pattern + if (failed(CheckTPUPartitionedInputAndOutputAreValid(cluster_func, + old_parallel_execute))) + return failure(); + + if (failed(CheckParallelExecuteConstainsValidNonClusterProcess( + old_parallel_execute))) { + old_parallel_execute.emitError() + << "contains invalid number of non TPU Process"; + return failure(); + } + + // After outside compilation the host process can return results, which come + // before the cluster_func's results. Collect the number of the outputs from + // those non cluster_func op + int num_results_pre_cluster = GetNumResultsPreCluster(old_parallel_execute); + + // Collect `num_replicas` and `num_cores_per_replica` attributes. + int num_replicas = 1; + tf_device::ReplicateOp replicate = + cluster_func->getParentOfType(); + if (replicate) num_replicas = replicate.getN(); + + auto num_cores_per_replica_attr = cluster_func->getAttrOfType( + tensorflow::kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) + return cluster_func.emitOpError( + CreateMissingAttributeMsg(tensorflow::kNumCoresPerReplicaAttr)); + + int num_cores_per_replica = num_cores_per_replica_attr.getInt(); + + auto topology_attr = + cluster_func->getAttrOfType(tensorflow::kTopologyAttr); + if (!topology_attr) + return cluster_func.emitOpError( + CreateMissingAttributeMsg(tensorflow::kTopologyAttr)); + + auto device_assignment_attr = cluster_func->getAttrOfType( + tensorflow::kDeviceAssignmentAttr); + if (!device_assignment_attr) + return cluster_func.emitOpError( + llvm::formatv("requires attribute '{0}'", + tensorflow::kDeviceAssignmentAttr) + .str()); + + auto status_or_device_coodinates = + tensorflow::GetDeviceCoordinates(device_assignment_attr); + if (!status_or_device_coodinates.ok()) + return cluster_func.emitError() + << "error in fetching tpu device coordinates: " + << status_or_device_coodinates.status().message(); + + // Determine compilation and execution devices. + auto status_or_tpu_device_assignment = + tensorflow::GetTPUCompilationAndExecutionDevices( + devices, num_replicas, num_cores_per_replica, + topology_attr.getValue(), status_or_device_coodinates.value()); + if (!status_or_tpu_device_assignment.ok()) + return cluster_func.emitError() + << "error in fetching TPU compilation/execution devices: " + << status_or_tpu_device_assignment.status().message(); + + // Create compile op. + auto& tpu_device_assignment = status_or_tpu_device_assignment.value(); + + // Create the TPUCompileMlir and TPUCompileSucceededAssert outside of + // the parallel_execute. + builder->setInsertionPoint(old_parallel_execute); + Operation* compile_op = BuildCompileOp( + module_name, cluster_func, num_replicas, num_cores_per_replica, + tpu_device_assignment.compilation_device, + std::move(tpu_device_assignment.xla_device_assignment), builder, + tpu_compile_metadata_debug); + if (!compile_op) return failure(); + + // This replaces _TPUCompileMlir placeholder ops that are required + // by XlaRecvAtHost and XlaSendFromHost ops add in earlier pass. + // TODO(b/157054714): When a better abstraction instead of _TPUCompileMlirOp + // and _XlaRecvAtHostOp and _XlaSendFromHostOp are used, update to a more + // structured lowering. + old_parallel_execute.walk( + [&](TF::_XlaCompileMlirPlaceholderProgramKeyOp key_op) { + key_op.replaceAllUsesWith(compile_op->getResult(1)); + key_op.erase(); + }); + + // After rewrite, if there is a TPUCompilationResultOp from the same cluster, + // replace it with the result of the compile op. The TPUCompilationResultOp is + // used as a placeholder to hook during graph creation the other ops that are + // intended to consume the compile result. + Operation* result_id = compile_op; + // TODO(jpienaar): Remove this later. + auto compile_device_op = compile_op->getAttr("device"); + for (auto res : compilation_result) { + // Build identity op with the same location/name as the original compilation + // result op. + result_id = TF::IdentityOp::create(*builder, res.getLoc(), + compile_op->getResult(0).getType(), + result_id->getResult(0)); + // Assign to same device as result is currently set, unless unset and then + // assign to the device on which compilation will happen. + // TODO(jpienaar): Remove this later. + if (auto device = res->getAttrOfType("device")) { + if (!device.getValue().empty()) + result_id->setAttr("device", device); + else + result_id->setAttr("device", compile_device_op); + } else if (compile_device_op) { + result_id->setAttr("device", compile_device_op); + } + res.getOutput().replaceAllUsesWith(compile_op->getResult(0)); + } + + BuildTPUCompileSucceededAssertOp( + compile_op, result_id, tpu_device_assignment.compilation_device, builder); + + AssignDevicesToReplicate(replicate, tpu_device_assignment.tpu_devices, + builder); + + llvm::SmallVector output_shardings; + auto result = tensorflow::ParseAndValidateOutputSharding( + num_cores_per_replica, cluster_func, &output_shardings); + if (failed(result)) return failure(); + + // For model parallelism, mlir::tf_device.parallel_execute is used to express + // concurrent device execution across multiple logical devices. + tf_device::ParallelExecuteOp new_parallel_execute; + int cluster_idx; + llvm::SmallVector, 4> cluster_to_core_index; + cluster_to_core_index.reserve(num_cores_per_replica); + result = AddToParallelExecuteOp( + tpu_device_assignment.tpu_devices, output_shardings, + &cluster_to_core_index, num_results_pre_cluster, compile_op, cluster_func, + builder, old_parallel_execute, &new_parallel_execute, &cluster_idx); + if (failed(result)) return failure(); + + // As mlir::tf_device.parallel_execute wraps # logical cores number of + // TPUExecute ops, the number of return values of parallel_execute op exceeds + // that of cluster_func op. As such, each return value of parallel_execute op + // must be mapped with corresponding return value usages of cluster_func. + result = tensorflow::RemapOutputsFromLogicalDevices( + cluster_func.getLoc(), output_shardings, cluster_to_core_index, + num_results_pre_cluster, old_parallel_execute, cluster_idx, + new_parallel_execute, builder); + if (failed(result)) return failure(); + + return TF::RemoveSingletonParallelExecuteOp(new_parallel_execute, builder); +} + +void TPURewritePass::runOnOperation() { + TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(getOperation(), &devices))) + return signalPassFailure(); + + // Collect compilation results. + llvm::DenseMap> + compilation_results; + auto result_init = getOperation().walk([&](TF::TPUCompilationResultOp op) { + auto cluster_id = op->getAttrOfType("_tpu_compilation_status"); + if (!cluster_id) { + op->emitOpError("missing '_tpu_compilation_status'"); + return WalkResult::interrupt(); + } + compilation_results[cluster_id].push_back(op); + return WalkResult::advance(); + }); + if (result_init.wasInterrupted()) return signalPassFailure(); + llvm::SmallVector to_be_erased; + OpBuilder builder(&getContext()); + auto result = getOperation().walk([&](tf_device::ClusterFuncOp op) { + if (failed(TF::HasValidCompilationAndReplicationAttributes(*op))) + return WalkResult::interrupt(); + // Skip non-tpu device cluster_func. + auto cluster_id = op->getAttrOfType(TF::kReplicationInfoAttr); + if (!cluster_id) return WalkResult::advance(); + + if (failed(Rewrite(module_name, op, devices.device_names(), + compilation_results[cluster_id], &builder, + tpu_compile_metadata_debug_))) + return WalkResult::interrupt(); + + to_be_erased.push_back(op); + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return signalPassFailure(); + + if (failed(tensorflow::EraseClusterFuncs(to_be_erased))) + return signalPassFailure(); + + // Eliminate TPUCompilationResultOp now that the rewrite is complete. + for (auto& it : compilation_results) { + for (auto op : it.second) { + if (!op.use_empty()) { + mlir::InFlightDiagnostic err = op.emitError("uses remain post rewrite"); + for (auto user : op->getUsers()) + err.attachNote(user->getLoc()) << "remaining user"; + return signalPassFailure(); + } + op.erase(); + } + } + + // TODO(b/139377366): Remove functions that are no longer needed. +} + +} // namespace + +std::unique_ptr> CreateTPURewritePass( + llvm::StringRef module_name) { + return std::make_unique(module_name); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_variable_runtime_reformatting.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_variable_runtime_reformatting.cc new file mode 100644 index 0000000000000000000000000000000000000000..207b2e37aaf71c375326b96fb834fb0335a36034 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_variable_runtime_reformatting.cc @@ -0,0 +1,513 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/random.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDeviceAttr[] = "device"; +constexpr char kFuncDeviceAttr[] = "tf.device"; +constexpr char kDefaultShardingValue[] = ""; +constexpr char kMirroredVariableIndicesAttr[] = "_mirrored_variable_indices"; + +std::string GetRandomStateVariableName() { + return absl::StrCat("VariablesFormatState_", tensorflow::random::New64()); +} + +#define GEN_PASS_DEF_TPUVARIABLERUNTIMEREFORMATTINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h.inc" + +struct TPUVariableRuntimeReformattingPass + : public impl::TPUVariableRuntimeReformattingPassBase< + TPUVariableRuntimeReformattingPass> { + void runOnOperation() final; +}; + +// Returns the earlier value of which `v` is an identity. If `skipped` is +// provided, it will be used to store the identity nodes skipped. +Value SkipIdentity(Value v, bool allow_other_use, + llvm::SmallPtrSet* skipped = nullptr) { + while (auto result = mlir::dyn_cast(v)) { + if (!(allow_other_use || v.hasOneUse())) break; + auto op = result.getDefiningOp(); + if (!llvm::isa(op)) { + break; + } + v = op->getOperand(result.getResultNumber()); + if (skipped) skipped->insert(op); + } + return v; +} + +// Finds the formattable arguments of `execute` and annotates the metadata of +// `compile` to record these arguments. In addition, it returns a mapping from +// the formattable arguments of `execute` to the corresponding operand of +// `replicate`. The +// entries in the mapping are sorted in the order of operands of `execute`. +llvm::SmallVector>, 4> +AnnotateCompileOpAndGetExecuteArgToWhileArgsMapping( + TF::WhileRegionOp while_op, tf_device::ReplicateOp replicate, + TF::TPUExecuteAndUpdateVariablesOp execute, + tf_device::LaunchOp compile_launch) { + Region& body = while_op.getBody(); + Region& cond = while_op.getCond(); + + llvm::SmallVector>, 4> mapping; + auto mirrored_variable_indices_attr = + replicate->getAttrOfType(kMirroredVariableIndicesAttr); + if (!mirrored_variable_indices_attr) return mapping; + + // Finds the mapping from a replicate argument to an execute operand. + llvm::SmallDenseMap replicate_arg_to_execute_arg; + for (auto index_and_arg : llvm::enumerate(execute.getArgs())) { + auto arg = SkipIdentity(index_and_arg.value(), /*allow_other_use=*/false); + if (!arg.hasOneUse() || + !mlir::isa(getElementTypeOrSelf(arg.getType()))) { + continue; + } + auto block_arg = mlir::dyn_cast(arg); + if (!block_arg || block_arg.getOwner() != &replicate.GetBody()) continue; + assert(replicate_arg_to_execute_arg.count(block_arg.getArgNumber()) == 0 && + "Found duplicate use of a resource in the execute op."); + replicate_arg_to_execute_arg[block_arg.getArgNumber()] = + index_and_arg.index(); + } + if (replicate_arg_to_execute_arg.empty()) return mapping; + + // Parse the original compile metadata. + Operation& compile = compile_launch.GetBody().front(); + auto metadata_str = compile.getAttrOfType("metadata"); + assert(metadata_str && "Missing compilation metadata"); + tensorflow::tpu::TPUCompileMetadataProto metadata; + metadata.ParseFromString(std::string(metadata_str.getValue())); + int64_t num_replicas = replicate.getN(); + // Find the formattable operands of `execute`, which must be mirrored + // variables (arguments of `replicate`), and must be pass-throughs from while + // operands. + for (const auto& mirrored_index : mirrored_variable_indices_attr) { + int64_t replicate_arg = mlir::cast(mirrored_index).getInt(); + // Check if the mirrored variable is an input to `execute`. + auto it = replicate_arg_to_execute_arg.find(replicate_arg); + if (it == replicate_arg_to_execute_arg.end()) continue; + // Get the data type of the resource. + auto subtypes = mlir::cast( + getElementTypeOrSelf(execute.getOperand(it->second))) + .getSubtypes(); + if (subtypes.size() != 1) continue; + auto data_type = getElementTypeOrSelf(subtypes[0]); + // The XLA backend does not yet support formatting 64-bit data types. + if (data_type.getIntOrFloatBitWidth() == 64) continue; + + const auto& block_arg = replicate.GetBody().getArgument(replicate_arg); + + int64_t num_inputs = 0; + if (replicate.IsReplicatedBlockArgument(block_arg)) { + num_inputs = num_replicas; + } else { + num_inputs = 1; + } + + // We have found a mirrored variable which is an input to the replicated + // `execute`. Now find if this mirrored variable is a pass-through of while + // arguments. + llvm::SmallVector replicate_args; + for (int64_t i = 0; i < num_inputs; ++i) { + llvm::SmallPtrSet skipped_identities; + + auto replicate_operand = SkipIdentity( + replicate.GetReplicaOperandForBlockArgument(block_arg, i), + /*allow_other_use=*/false, &skipped_identities); + // For region based control flow, the resource operand for the replicate + // should be a region capture. If this has any use other than the + // replicate op (within the body of the while) or the skipped identities, + // then do not apply the transformation to this variable. + bool is_region_capture = + replicate_operand.getParentRegion()->isProperAncestor(&body); + bool has_other_use_in_body = + llvm::any_of(replicate_operand.getUsers(), [&](Operation* user) { + // Ignore uses that are not in the while body or condition. + if (!body.isAncestor(user->getParentRegion()) && + !cond.isAncestor(user->getParentRegion())) + return false; + // Within the body or cond, only uses in replicate and the skipped + // identities is allowed. + return user != replicate && skipped_identities.count(user) == 0; + }); + + if (!is_region_capture || has_other_use_in_body) { + replicate_args.clear(); + break; + } + replicate_args.push_back(replicate_operand); + } + if (replicate_args.empty()) continue; + // Now set the enable_xla_sharding field in the metadata to inform the + // compile op. + auto metadata_arg = metadata.mutable_args(it->second); + metadata_arg->set_enable_xla_sharding( + ::tensorflow::tpu::TPUCompileMetadataProto_Arg::ALLOWED); + mapping.emplace_back(it->second, std::move(replicate_args)); + } + // Sort the mapping according to execute operand order. + llvm::sort(mapping, llvm::less_first()); + // Populate the `retval_index_for_sharding` field of the argument metadate. + for (auto entry : llvm::enumerate(execute.getDeviceVarReadsIndices())) { + int64_t arg_index = mlir::cast(entry.value()).getInt(); + auto arg_metadata = metadata.mutable_args(arg_index); + if (arg_metadata->enable_xla_sharding() == + ::tensorflow::tpu::TPUCompileMetadataProto_Arg::ALLOWED) { + int64_t ret_index = + mlir::cast( + execute.getDeviceVarUpdatesIndices().getValue()[entry.index()]) + .getInt(); + arg_metadata->set_retval_index_for_sharding(ret_index); + } + } + // Update the metadata of the compile op. + compile.setAttr("metadata", StringAttr::get(compile.getContext(), + metadata.SerializeAsString())); + return mapping; +} + +// Adds a new replicated input to the replicate op. +tf_device::ReplicateOp AddInputsToReplicateOp( + tf_device::ReplicateOp replicate, + MutableArrayRef new_inputs, + const llvm::SmallDenseMap>& + devices) { + int64_t num_replicas = replicate.getN(); + assert(new_inputs.size() == num_replicas); + + // As model parallelism is not yet supported, we assume that all ops are + // placed in logical core 0. + // TODO(b/148913020): Remove this constraint once model parallelism is + // supported. + assert(devices.find(tensorflow::GetDeviceAliasForLogicalCore(0)) + ->getSecond() + .size() == num_replicas); + + llvm::SmallVector, 8> new_replicated_inputs; + llvm::SmallVector new_packed_inputs; + llvm::SmallVector, 8> replicated_inputs; + replicated_inputs.reserve(replicate.GetNumReplicatedBlockArguments()); + new_packed_inputs.reserve(replicate.GetNumPackedBlockArguments()); + for (const auto& arg : replicate.GetReplicatedBlockArguments()) { + replicated_inputs.emplace_back(); + for (int64_t i = 0; i < num_replicas; ++i) { + replicated_inputs.back().push_back( + replicate.GetReplicaOperandForBlockArgument(arg, i)); + } + new_replicated_inputs.emplace_back(replicated_inputs.back(), arg.getType()); + } + for (const auto& arg : replicate.GetPackedBlockArguments()) { + new_packed_inputs.emplace_back( + replicate.GetReplicaOperandForBlockArgument(arg, /*replica=*/0)); + } + SmallVector new_input_values; + new_input_values.reserve(new_inputs.size()); + for (auto var : new_inputs) new_input_values.push_back(var.getResource()); + new_replicated_inputs.emplace_back(new_input_values, + new_input_values.front().getType()); + OpBuilder builder(replicate); + auto new_replicate = tf_device::ReplicateOp::create( + builder, replicate.getLoc(), num_replicas, devices, new_replicated_inputs, + new_packed_inputs, + replicate.GetBody().getTerminator()->getOperandTypes()); + for (auto arg : replicate.GetBody().getArguments()) { + if (replicate.IsReplicatedBlockArgument(arg)) { + arg.replaceAllUsesWith( + new_replicate.GetBody().getArgument(arg.getArgNumber())); + } else { + // There is a new added replicated state variable between replicated args + // and packed args. + arg.replaceAllUsesWith( + new_replicate.GetBody().getArgument(arg.getArgNumber() + 1)); + } + } + for (auto& op : llvm::make_early_inc_range(replicate.GetBody())) { + op.moveBefore(&new_replicate.GetBody(), new_replicate.GetBody().end()); + } + replicate.replaceAllUsesWith(new_replicate); + replicate.erase(); + return new_replicate; +} + +// Creates the per-device variables that represent the formatting state of each +// device. +llvm::SmallVector CreateStateVars( + const llvm::SmallDenseMap>& + devices, + Location loc, RankedTensorType key_type, OpBuilder* builder) { + llvm::SmallVector state_vars; + + // TODO(b/148913020): Remove this constraint once model parallelism is + // supported. + const auto& device_list = + devices.find(tensorflow::GetDeviceAliasForLogicalCore(0))->getSecond(); + + // Create the state variable for each device. + for (llvm::StringRef device : device_list) { + state_vars.push_back(TF::VarHandleOp::create( + *builder, loc, + llvm::ArrayRef{RankedTensorType::get( + {}, TF::ResourceType::get(llvm::ArrayRef{key_type}, + builder->getContext()))}, + llvm::ArrayRef{}, + llvm::ArrayRef{ + builder->getNamedAttr(kDeviceAttr, builder->getStringAttr(device)), + builder->getNamedAttr("container", builder->getStringAttr("")), + builder->getNamedAttr( + "shared_name", + builder->getStringAttr(GetRandomStateVariableName()))})); + } + return state_vars; +} + +// Wraps single op in `tf_device.launch` for explicit device assignment. +void WrapOpInLaunch(OpBuilder* builder, Location loc, Operation* op, + llvm::StringRef device) { + OpBuilder::InsertPoint insert_point = builder->saveInsertionPoint(); + + auto launch = tf_device::LaunchOp::create( + *builder, loc, builder->getStringAttr(device), op->getResultTypes()); + launch.getBody().push_back(new Block); + + builder->setInsertionPointToEnd(&launch.GetBody()); + tf_device::ReturnOp::create(*builder, loc, op->getResults()); + + // Move op inside launch. + op->moveBefore(launch.GetBody().getTerminator()); + + builder->restoreInsertionPoint(insert_point); +} + +// Performs the transformation for a replicate op inside a while loop. Returns +// true when any change was made by this function. +bool HandleReplicateOp(TF::WhileRegionOp while_op, + tf_device::ReplicateOp replicate) { + int64_t num_replicas = replicate.getN(); + if (num_replicas == 1) return false; + + // Set execute_launch when there is exactly one + // TPUExecuteAndUpdateVariablesOp. More than one means there is model + // parallelism, which is not supported with TPUReshardVariables. None + // means there is no TPU computation. + tf_device::LaunchOp execute_launch; + TF::TPUExecuteAndUpdateVariablesOp execute; + replicate.walk([&](TF::TPUExecuteAndUpdateVariablesOp execute_op) { + execute_launch = + llvm::dyn_cast(execute_op->getParentOp()); + if (execute_launch == nullptr) { + // This pass requires execute_op to be wrapped in a launch. + return WalkResult::interrupt(); + } + if (execute == nullptr) { + execute = execute_op; + return WalkResult::advance(); + } + execute = nullptr; + return WalkResult::interrupt(); + }); + if (!execute) return false; + auto compile = + SkipIdentity(execute.getKey(), /*allow_other_use=*/true).getDefiningOp(); + if (!compile) return false; + auto compile_launch = llvm::dyn_cast(compile); + if (!compile_launch || !compile_launch.WrapsSingleOp() || + !llvm::isa(compile_launch.GetBody().front())) + return false; + + // Analyze the formattable inputs. + auto execute_arg_to_outer_args = + AnnotateCompileOpAndGetExecuteArgToWhileArgsMapping( + while_op, replicate, execute, compile_launch); + if (execute_arg_to_outer_args.empty()) return false; + + // Extract the replicated devices. + auto devices_attr = replicate.getDevices(); + if (!devices_attr) return false; + + auto device_map = devices_attr.value(); + llvm::SmallDenseMap> devices; + devices.reserve(device_map.size()); + + for (auto it : device_map) { + auto device_alias = it.getName().strref(); + auto device_list = mlir::cast(it.getValue()); + llvm::SmallVector device_list_for_alias; + device_list_for_alias.reserve(device_list.size()); + + for (auto device : device_list) + device_list_for_alias.emplace_back( + mlir::cast(device).getValue()); + + devices.insert({device_alias, device_list_for_alias}); + } + + OpBuilder builder(replicate); + builder.setInsertionPoint(while_op); + // Create per-device variables for formatting state, and add them to the while + // loop. + auto key_type = + RankedTensorType::get({2}, TF::StringType::get(builder.getContext())); + auto state_vars = + CreateStateVars(devices, while_op.getLoc(), key_type, &builder); + replicate = AddInputsToReplicateOp(replicate, state_vars, devices); + // Build the reformat according to the compilation. Build it inside + // `replicate`. + llvm::SmallVector reformat_operands; + for (const auto& entry : execute_arg_to_outer_args) { + reformat_operands.push_back(execute.getArgs()[entry.first]); + } + reformat_operands.push_back(compile_launch.getResult(1)); + reformat_operands.push_back(replicate.GetBody().getArgument( + replicate.GetNumReplicatedBlockArguments() - 1)); + builder.setInsertionPoint(execute_launch); + auto reformat_op = TF::TPUReshardVariablesOp::create( + builder, execute_launch.getLoc(), llvm::ArrayRef{}, + reformat_operands); + WrapOpInLaunch(&builder, execute_launch.getLoc(), reformat_op, + execute_launch.getDevice()); + + // Build the replicated unformat op after the loop. First prepare building the + // replicate op. + llvm::SmallVector, 8> unformat_replicate_operands; + llvm::SmallVector unformat_packed_operands; + for (const auto& entry : execute_arg_to_outer_args) { + if (entry.second.size() > 1) { + unformat_replicate_operands.emplace_back(entry.second, + entry.second.front().getType()); + } else { + unformat_packed_operands.emplace_back(entry.second.front()); + } + } + llvm::SmallVector state_var_vals(state_vars.size()); + for (const auto& entry : llvm::enumerate(state_vars)) { + state_var_vals[entry.index()] = entry.value().getResource(); + } + // Add the replicated state var to the end of the replicate operands. + unformat_replicate_operands.emplace_back(state_var_vals, + state_var_vals.front().getType()); + // Build a constant default key to specify that the unformatting should + // transform the variables to the original format. + builder.setInsertionPointAfter(while_op); + tensorflow::Tensor default_key_tensor(tensorflow::DT_STRING, {3}); + default_key_tensor.vec()(0) = kDefaultShardingValue; + default_key_tensor.vec()(1) = kDefaultShardingValue; + default_key_tensor.vec()(2) = kDefaultShardingValue; + auto default_state_key = TF::ConstOp::create( + builder, while_op.getLoc(), + tensorflow::ConvertTensor(default_key_tensor, &builder).value()); + // With all replicated inputs, now build the replicate op. + auto unformat_replicate = tf_device::ReplicateOp::create( + builder, while_op.getLoc(), num_replicas, devices, + unformat_replicate_operands, unformat_packed_operands, TypeRange{}); + // Then build the unformat op in the replicate op. + builder.setInsertionPointToEnd(&unformat_replicate.GetBody()); + llvm::SmallVector unformat_operands; + // Add the replicated state var (the last replicated operand of the + // ReplicateOp) as the last operand of TPUReshardVariablesOp. + BlockArgument state = unformat_replicate.GetReplicatedBlockArguments().back(); + auto replicated_block_args = + unformat_replicate.GetReplicatedBlockArguments().drop_back(1); + auto packed_block_args = unformat_replicate.GetPackedBlockArguments(); + unformat_operands.append(replicated_block_args.begin(), + replicated_block_args.end()); + unformat_operands.append(packed_block_args.begin(), packed_block_args.end()); + unformat_operands.push_back(state); + + // Insert the default key as the second last operand. + unformat_operands.insert( + unformat_operands.begin() + unformat_operands.size() - 1, + default_state_key.getResult()); + // Unformat op. + auto unformat_op = TF::TPUReshardVariablesOp::create( + builder, while_op.getLoc(), llvm::ArrayRef{}, unformat_operands); + WrapOpInLaunch(&builder, execute_launch.getLoc(), unformat_op, + execute_launch.getDevice()); + tf_device::ReturnOp::create(builder, while_op.getLoc(), ArrayRef{}); + + return true; +} + +void TPUVariableRuntimeReformattingPass::runOnOperation() { + auto module = getOperation(); + bool reshard_was_inserted = false; + module.walk([&](TF::WhileRegionOp while_op) { + tf_device::ReplicateOp replicate; + while_op.getBody().walk([&](tf_device::ReplicateOp replicate_op) { + if (replicate == nullptr) { + replicate = replicate_op; + return WalkResult::advance(); + } + // We do not handle loops with multiple replicate ops. + replicate = nullptr; + return WalkResult::interrupt(); + }); + if (replicate) + reshard_was_inserted |= HandleReplicateOp(while_op, replicate); + }); + if (reshard_was_inserted) + VLOG(1) << "tf-tpu-variable-runtime-reformatting inserted at least one " + "TPUReshardVariables"; + else + VLOG(1) << "tf-tpu-variable-runtime-reformatting inserted no " + "TPUReshardVariables"; +} + +} // namespace + +std::unique_ptr> +CreateTPUVariableRuntimeReformattingPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import.cc new file mode 100644 index 0000000000000000000000000000000000000000..e931b2de266a29d6c9ce77a08f342701810b7cd5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import.cc @@ -0,0 +1,166 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/MemoryBuffer.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/core/lib/io/path.h" + +namespace mlir { +namespace TF { +namespace { + +static constexpr int kTextFileIndex_WholeLine = -2; +static constexpr int kTextFileIndex_LineNumber = -1; + +#define GEN_PASS_DEF_INITTEXTFILETOIMPORTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// InitTextFileToImportPass converts InitializeTableFromTextFileV2Op to the +// corresponding LookupTableImportV2Op if possible. +class InitTextFileToImportPass + : public impl::InitTextFileToImportPassBase { + public: + InitTextFileToImportPass() = default; + InitTextFileToImportPass(const InitTextFileToImportPass&) {} + explicit InitTextFileToImportPass(std::string saved_model_dir) { + saved_model_dir_ = saved_model_dir; + } + + private: + void runOnOperation() override; +}; + +class ConvertInitializeTableFromTextFileV2 + : public OpRewritePattern { + public: + explicit ConvertInitializeTableFromTextFileV2(mlir::MLIRContext* context, + StringRef saved_model_dir) + : OpRewritePattern(context), + saved_model_dir_(saved_model_dir) {} + + LogicalResult matchAndRewrite(InitializeTableFromTextFileV2Op op, + PatternRewriter& rewriter) const override { + // Now, this pattern matching only supports the following case, which is + // commonly used among inference use cases: + // + // tf.lookup.TextFileInitializer( + // "test.txt", tf.string, tf.lookup.TextFileIndex.WHOLE_LINE, + // tf.int64, tf.lookup.TextFileIndex.LINE_NUMBER, delimiter=" ") + // + // In the above case, the delimiter will be not used since the key is just a + // whole line and value is a line number. + if (op.getKeyIndex() != kTextFileIndex_WholeLine || + op.getValueIndex() != kTextFileIndex_LineNumber) { + return failure(); + } + + // Try to find filename from constant op. + DenseStringElementsAttr filename_attr; + if (!matchPattern(op.getFilename().getDefiningOp(), + m_Constant(&filename_attr))) { + return failure(); + } + + if (filename_attr.getRawStringData().size() != 1) { + return failure(); + } + std::string filename = filename_attr.getRawStringData()[0].str(); + + if (!saved_model_dir_.empty()) { + filename = tensorflow::io::JoinPath( + saved_model_dir_.str(), + tensorflow::io::JoinPath("assets", + tensorflow::io::Basename(filename))); + } + + // Read the content of the file. + std::string error_message; + auto file = openInputFile(filename, &error_message); + if (!file) { + return op.emitOpError("failed to open vocabulary file") + << " (" << filename << "): " << error_message; + } + + // Splits into lines. + SmallVector lines; + file->getBuffer().split(lines, "\n", -1, false); + // The resize method is used since split operator puts tail value in the end + // without splitting the leftovers. + if (op.getVocabSize() != -1) lines.resize(op.getVocabSize()); + + // Map each line to line number, starting from zero. + SmallVector line_nums; + line_nums.resize(lines.size()); + std::iota(line_nums.begin(), line_nums.end(), 0); + + // Create constant ops for keys an values. + Value key_constant_tensor = arith::ConstantOp::create( + rewriter, op.getLoc(), + DenseStringElementsAttr::get( + RankedTensorType::get(static_cast(lines.size()), + StringType::get(rewriter.getContext())), + lines)); + + Value value_constant_tensor = arith::ConstantOp::create( + rewriter, op.getLoc(), rewriter.getI64TensorAttr(line_nums)); + + // Replace the given op with LookupTableImportV2Op. + LookupTableImportV2Op::create(rewriter, op.getLoc(), op.getTableHandle(), + key_constant_tensor, value_constant_tensor); + rewriter.eraseOp(op); + return success(); + } + + private: + StringRef saved_model_dir_; +}; + +void InitTextFileToImportPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + MLIRContext* context = &getContext(); + func::FuncOp func = getOperation(); + + patterns.add( + context, StringRef(saved_model_dir_)); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +// Replace InitializeTableFromTextFileV2Ops with LookupTableImportV2Ops. +std::unique_ptr> CreateInitTextFileToImportPass( + std::string saved_model_dir) { + return std::make_unique(saved_model_dir); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6fa75306b411544d9c59a0ef513eae82e7d41aa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/init_text_file_to_import_test_pass.cc @@ -0,0 +1,167 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/Support/Casting.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/ToolOutputFile.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/core/platform/path.h" +#include "tensorflow/core/platform/stringpiece.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_INITTEXTFILETOIMPORTTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +// InitTextFileToImportTestPass generates a temporary file and run the +// InitTextFileToImportPass for testing purpose. +class InitTextFileToImportTestPass + : public impl::InitTextFileToImportTestPassBase< + InitTextFileToImportTestPass> { + public: + explicit InitTextFileToImportTestPass() = default; + + StringRef getArgument() const final { + return "tf-init-text-file-to-import-test"; + } + + StringRef getDescription() const final { + return "generate a temporary file and invoke InitTextFileToImportPass"; + } + + private: + void runOnOperation() override; +}; + +void InitTextFileToImportTestPass::runOnOperation() { + ModuleOp module = getOperation(); + + // Create a temporary vocab file. + int fd; + SmallString<256> filename; + std::error_code error_code = + llvm::sys::fs::createTemporaryFile("text", "vocab", fd, filename); + if (error_code) return signalPassFailure(); + + llvm::ToolOutputFile temp_file(filename, fd); + temp_file.os() << "apple\n"; + temp_file.os() << "banana\n"; + temp_file.os() << "grape"; + temp_file.os().flush(); + + // Replace filename constant ops to use the temporary file. + MLIRContext* context = &getContext(); + + for (func::FuncOp func : module.getOps()) { + llvm::SmallVector constant_ops( + func.getOps()); + for (auto op : constant_ops) { + ShapedType shaped_type = + RankedTensorType::get({1}, StringType::get(context)); + + DenseStringElementsAttr attr; + if (!matchPattern(op.getOperation(), m_Constant(&attr))) { + continue; + } + + ArrayRef values = attr.getRawStringData(); + if (values.size() != 1 || values[0] != "%FILE_PLACEHOLDER") { + continue; + } + + op.setValueAttr(DenseStringElementsAttr::get(shaped_type, {filename})); + } + } + + // Run the lowering pass. + PassManager pm(context); + pm.addNestedPass(CreateInitTextFileToImportPass("")); + if (failed(pm.run(module))) return signalPassFailure(); +} + +#define GEN_PASS_DEF_INITTEXTFILETOIMPORTSAVEDMODELTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +// InitTextFileToImportSavedModelTestPass mimicks a temporary saved model and +// run the InitTextFileToImportPass for testing purpose. +class InitTextFileToImportSavedModelTestPass + : public impl::InitTextFileToImportSavedModelTestPassBase< + InitTextFileToImportSavedModelTestPass> { + public: + explicit InitTextFileToImportSavedModelTestPass() = default; + + private: + void runOnOperation() override; +}; + +void InitTextFileToImportSavedModelTestPass::runOnOperation() { + ModuleOp module = getOperation(); + + // Create a temporary saved model's asset file. + SmallString<256> tempdir; + std::error_code error_code = + llvm::sys::fs::createUniqueDirectory("saved-model", tempdir); + if (error_code) return signalPassFailure(); + error_code = + llvm::sys::fs::create_directories(Twine(tempdir) + "/assets", false); + if (error_code) return signalPassFailure(); + + std::string filename = std::string(tempdir) + "/assets/tokens.txt"; + + std::string error_message; + auto temp_file = openOutputFile(filename, &error_message); + if (!error_message.empty()) return; + temp_file->os() << "apple\n"; + temp_file->os() << "banana\n"; + temp_file->os() << "grape"; + temp_file->os().flush(); + + // Run the lowering pass. + MLIRContext* context = &getContext(); + PassManager pm(context); + pm.addNestedPass( + CreateInitTextFileToImportPass(std::string(tempdir))); + if (failed(pm.run(module))) return signalPassFailure(); +} + +} // namespace +} // namespace TF + +namespace tf_test { +std::unique_ptr> CreateInitTextFileToImportTestPass() { + return std::make_unique(); +} +std::unique_ptr> +CreateInitTextFileToImportSavedModelTestPass() { + return std::make_unique(); +} +} // namespace tf_test + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.cc new file mode 100644 index 0000000000000000000000000000000000000000..320901d06c9281d2049ae2a5805d7710230a8005 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.cc @@ -0,0 +1,174 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/session_utils.h" +#include "tensorflow/core/framework/resource_var.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +void InitializeVariable(TF::VarHandleOp var_handle_op, + tensorflow::Tensor* tensor, + func::FuncOp session_init_func, OpBuilder builder) { + absl::StatusOr tensor_attr_or = + tensorflow::ConvertTensor(*tensor, &builder); + assert(tensor_attr_or.ok() && "Expect valid tensor"); + ElementsAttr tensor_attr = tensor_attr_or.value(); + + builder.setInsertionPointToStart(&session_init_func.getBlocks().front()); + auto var_handle_op_in_init = var_handle_op->clone(); + builder.insert(var_handle_op_in_init); + auto const_op = mlir::arith::ConstantOp::create( + builder, session_init_func.getLoc(), tensor_attr.getType(), tensor_attr); + + TF::AssignVariableOp::create( + builder, session_init_func.getLoc(), llvm::ArrayRef{}, + llvm::ArrayRef{var_handle_op_in_init->getResult(0), + const_op.getResult()}); +} + +func::FuncOp CreateSessionInitFunc(ModuleOp module) { + constexpr char kSessionInitFuncName[] = "SessionInitializerFunction"; + + mlir::OpBuilder builder(module.getBodyRegion()); + auto func_type = + FunctionType::get(module.getContext(), /*inputs=*/{}, /*results=*/{}); + auto func = func::FuncOp::create(builder, module->getLoc(), + kSessionInitFuncName, func_type); + func->setAttr(kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr({kSessionInitFuncName})); + func->setAttr(kTfSavedModelInitializerTypeAttr, + builder.getStringAttr(kTfSavedModelInitializerRestoreType)); + func.setVisibility(mlir::func::FuncOp::Visibility::Public); + auto func_builder = OpBuilder::atBlockBegin(func.addEntryBlock()); + mlir::func::ReturnOp::create(func_builder, func.getLoc()); + // In cases where there is a session initializer op with empty initializer, + // replace the session initializer with the new one that points to the session + // initializer func. + SessionInitializerOp session_init_op = GetSessionInitializerOp(module); + auto new_session_init_op = tf_saved_model::SessionInitializerOp::create( + builder, module->getLoc(), + builder.getArrayAttr( + SymbolRefAttr::get(builder.getContext(), kSessionInitFuncName))); + if (session_init_op) { + session_init_op->replaceAllUsesWith(new_session_init_op); + session_init_op->erase(); + } + return func; +} + +func::FuncOp GetOrCreateSessionInitFunc(ModuleOp module) { + SessionInitializerOp session_init_op = GetSessionInitializerOp(module); + if (!session_init_op) return CreateSessionInitFunc(module); + + auto init_func_op = GetInitializerFunction( + module, /*initializer_type=*/kTfSavedModelInitializerRestoreType); + if (init_func_op) { + return init_func_op; + } else if (!session_init_op.getInitializers().empty()) { + // When the init function with type "restore_op" is not found, fall back to + // taking the init function corresponding to the first symbol in the + // initializers list to be backwards-compatible, before + // tf_saved_model.initializer_type attribute was introduced. + SymbolTable symbol_table(module); + return symbol_table.lookup( + mlir::cast(session_init_op.getInitializers()[0]) + .getValue()); + } else { + return CreateSessionInitFunc(module); + } +} + +} // namespace + +LogicalResult InitializeVariablesInSessionInitializer( + ModuleOp module, tensorflow::Session* session) { + const tensorflow::DeviceMgr* mgr = nullptr; + auto status = session->LocalDeviceManager(&mgr); + if (!status.ok()) { + module->emitError( + absl::StrCat("failed to fetch device manager: ", status.message())); + return failure(); + } + + // Fetch all VarHandleOp. + llvm::StringSet<> variable_names; + llvm::SmallVector var_ops; + for (auto func_op : module.getOps()) { + for (auto var_handle_op : func_op.getOps()) { + auto variable_name = GetVariableName(var_handle_op); + if (variable_names.count(variable_name)) continue; + var_ops.emplace_back(var_handle_op); + variable_names.insert(variable_name); + } + } + + // Get resources from Session. + auto resource_tensors_or = GetResourcesFromSession(var_ops, session); + if (!resource_tensors_or.ok()) { + module->emitError(resource_tensors_or.status().message().data()); + return failure(); + } + + auto session_init_func = GetOrCreateSessionInitFunc(module); + OpBuilder builder(session_init_func.getContext()); + + for (auto var_and_tensor : llvm::zip(var_ops, resource_tensors_or.value())) { + auto& var_op = std::get<0>(var_and_tensor); + auto& resource_tensor = std::get<1>(var_and_tensor); + if (resource_tensor.dtype() != tensorflow::DT_RESOURCE) { + InitializeVariable(var_op, &resource_tensor, session_init_func, builder); + continue; + } + + auto handle = resource_tensor.scalar()(); + auto* var_ptr = GetVariableFromSession(var_op, handle.device(), mgr); + if (!var_ptr) { + // If no value in session, then just skip this variable. + // This can happen if the variable is not saved in checkpoint. + // For example, when the variable is created on every call. + continue; + } + tensorflow::core::RefCountPtr var(var_ptr); + auto* tensor = var_ptr->tensor(); + + InitializeVariable(var_op, tensor, session_init_func, builder); + } + return success(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.h new file mode 100644 index 0000000000000000000000000000000000000000..6eb722acd22d40dad7cdb2958f7155f1ba93673a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.h @@ -0,0 +1,35 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_INITIALIZE_VARIABLES_IN_SESSION_INIT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_INITIALIZE_VARIABLES_IN_SESSION_INIT_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Initializes all variables in Session Init function for all variables in +// 'session'. +LogicalResult InitializeVariablesInSessionInitializer( + ModuleOp module, tensorflow::Session *session); + +} // namespace tf_saved_model + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_INITIALIZE_VARIABLES_IN_SESSION_INIT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..d242ca328bcec058458cbf7781b32bdd0b827072 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init_test_pass.cc @@ -0,0 +1,50 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/fake_session.h" + +namespace mlir { +namespace tf_test { +namespace { + +#define GEN_PASS_DEF_INITIALIZEVARIABLESINSESSIONINITIALIZERPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +class InitializeVariablesInSessionInitializerPass + : public impl::InitializeVariablesInSessionInitializerPassBase< + InitializeVariablesInSessionInitializerPass> { + public: + void runOnOperation() final { + static tensorflow::Session* session = new TF::test_util::FakeSession(); + if (failed(tf_saved_model::InitializeVariablesInSessionInitializer( + getOperation(), session))) + signalPassFailure(); + } + + private: +}; +} // anonymous namespace + +std::unique_ptr> +CreateInitializeVariablesInSessionInitializerTestPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc new file mode 100644 index 0000000000000000000000000000000000000000..ec0eafde6b1b9ed48cc1c3ee67171ff7414fd217 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc @@ -0,0 +1,149 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h" + +namespace mlir { +namespace TFDevice { +namespace { +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_LAUNCHTODEVICEATTRIBUTEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct LaunchToDeviceAttributePass + : public impl::LaunchToDeviceAttributePassBase< + LaunchToDeviceAttributePass> { + public: + explicit LaunchToDeviceAttributePass(bool legacy_graph_export) { + legacy_graph_export_ = legacy_graph_export; + } + + void runOnOperation() override; +}; + +// Assign all ops in region with specified device from launch. +LogicalResult AssignDevicesInRegion(const Dialect* tf_dialect, + tf_device::LaunchOp launch, + Region& region) { + auto parallel_group_attr = + launch->getAttrOfType(TF::kParallelExecAnnotation); + auto result = region.walk([&](Operation* op) -> WalkResult { + if (op->getDialect() != tf_dialect) return WalkResult::advance(); + + if (parallel_group_attr) { + op->setAttr(TF::kParallelExecAnnotation, parallel_group_attr); + } + auto device_attr = op->getAttr(kDeviceAttr); + if (!device_attr) { + op->setAttr(kDeviceAttr, launch.getDeviceAttr()); + return WalkResult::advance(); + } + + if (auto device_str_attr = mlir::dyn_cast(device_attr)) { + if (device_str_attr.getValue().empty()) { + op->setAttr(kDeviceAttr, launch.getDeviceAttr()); + return WalkResult::advance(); + } else if (device_str_attr.getValue() != launch.getDevice()) { + return launch.emitOpError() + << "inner op has conflicting 'device' attribute, " + "got '" + << device_str_attr.getValue() << "' but expected '" + << launch.getDevice() << "'"; + } + } else { + return launch.emitOpError() + << "inner op has bad 'device' attribute, got " << device_attr; + } + + return WalkResult::advance(); + }); + + return failure(result.wasInterrupted()); +} + +LogicalResult HoistOpsAndAnnotateWithDevice(const Dialect* tf_dialect, + tf_device::LaunchOp launch) { + // Forward launch inner op results to launch op results. + launch.replaceAllUsesWith(launch.GetBody().getTerminator()->getOperands()); + + // For all inner ops, assign the launch device as a `device` attribute. + if (failed(AssignDevicesInRegion(tf_dialect, launch, launch.getBody()))) + return failure(); + + // Move all inner ops of the launch to the block containing the launch. + auto body = launch.GetBody().without_terminator(); + Operation* launch_op = launch.getOperation(); + launch_op->getBlock()->getOperations().splice( + launch_op->getIterator(), launch.GetBody().getOperations(), body.begin(), + body.end()); + + launch.erase(); + + return success(); +} + +void LaunchToDeviceAttributePass::runOnOperation() { + const Dialect* tf_dialect = getContext().getLoadedDialect("tf"); + if (!tf_dialect) { + getOperation().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); + } + + auto result = getOperation().walk([&tf_dialect](tf_device::LaunchOp launch) { + if (failed(HoistOpsAndAnnotateWithDevice(tf_dialect, launch))) + return WalkResult::interrupt(); + + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) return signalPassFailure(); + + if (!legacy_graph_export_) { + // Now, split the island into an island per op since we don't want to + // violate the invariant imposed by the GraphExport pipeline that every + // IslandOp perfectly wraps a single op. + auto control_type = + mlir::tf_executor::ControlType::get(tf_dialect->getContext()); + getOperation().walk( + [&control_type](mlir::tf_executor::IslandOp curr_island) { + mlir::TF::SplitIsland(curr_island, control_type); + }); + } +} + +} // anonymous namespace + +std::unique_ptr> CreateLaunchToDeviceAttributePass( + bool legacy_graph_export) { + return std::make_unique(legacy_graph_export); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/layout_optimization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/layout_optimization.cc new file mode 100644 index 0000000000000000000000000000000000000000..bd4c152faab1a23c811cb0f9f1d318ef9bfae622 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/layout_optimization.cc @@ -0,0 +1,518 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_layout_helper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" + +#define DEBUG_TYPE "tf-layout-optimization" + +namespace mlir { +namespace TF { + +namespace { + +// Helper method that returns an op from 'transpose_ops' that match criteria +// for an 'operand' and 'permutation' +TransposeOp ReuseExistingTranspose(const OpOperand* operand, + const SmallVector& permutation, + Operation* op, ConstOp permutation_op, + SmallVector* transpose_ops) { + for (auto it = transpose_ops->begin(); it != transpose_ops->end(); ++it) { + auto tranpose_op = *it; + for (auto tranpose_operand : tranpose_op.getOperands()) { + auto ranked_tranpose_type = + mlir::dyn_cast_or_null(tranpose_operand.getType()); + if (!ranked_tranpose_type) continue; + if (ranked_tranpose_type.getRank() == permutation.size() && + operand->get().getType() == + ShuffleRankedTensorType(ranked_tranpose_type, permutation)) { + TransposeOp transpose = tranpose_op; + transpose.getOperation()->moveBefore(op); + transpose.setOperand(0, operand->get()); + transpose.setOperand(1, permutation_op); + transpose_ops->erase(it); + return transpose; + } + } + } + return nullptr; +} + +#define GEN_PASS_DEF_LAYOUTASSIGNMENTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// LayoutAssignmentPass assigns optimal data layout (data format) for all +// layout sensitive operations. +class LayoutAssignmentPass + : public impl::LayoutAssignmentPassBase { + public: + LayoutAssignmentPass() = default; + explicit LayoutAssignmentPass(const std::string& force_data_format) { + force_data_format_ = force_data_format; + } + + LayoutAssignmentPass(const LayoutAssignmentPass& pass) {} + + void runOnOperation() final; +}; + +#define GEN_PASS_DEF_MOVETRANSPOSESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// MoveTransposesPass moves all Transpose ops to the beginning or to the end of +// the basic block where they are defined. This will allow canonicalzer to +// delete redundant transposes. +class MoveTransposesPass + : public impl::MoveTransposesPassBase { + public: + MoveTransposesPass() = default; + explicit MoveTransposesPass(MoveTransposeDirection direction, + bool fold_transpose_in_ops) { + this->direction_ = direction; + this->fold_transpose_in_ops_ = fold_transpose_in_ops; + } + MoveTransposesPass(const MoveTransposesPass& pass) {} + + void runOnOperation() final; +}; + +using Permutation = SmallVector; + +void LayoutAssignmentPass::runOnOperation() { + func::FuncOp func = getOperation(); + + // Get runtime devices information from the closest parent module. + RuntimeDevices devices; + if (failed(::tensorflow::GetDevicesFromOp(func->getParentOfType(), + &devices))) + return signalPassFailure(); + + // If there is no runtime device information and data format is not explicitly + // forced, there is nothing to do. + if (devices.NumDevices() == 0 && force_data_format_.empty()) return; + + func.walk([&](LayoutSensitiveInterface layout_sensitive_interface) { + // Get desired op data format. + StringRef target_data_format = force_data_format_; + if (target_data_format.empty()) { + target_data_format = layout_sensitive_interface.GetOptimalLayout(devices); + } + + // Skip ops that already use target data format. + auto data_format = layout_sensitive_interface.data_format(); + if (data_format == target_data_format) return; + + // Transpose arguments into the target data format. + Permutation args_permutation = + GetDataFormatPermutation(data_format, target_data_format); + + // Transpose results back to the original data format. + Permutation res_permutation = + GetDataFormatPermutation(target_data_format, data_format); + + if (args_permutation.empty() || res_permutation.empty()) return; + + mlir::Operation* op = layout_sensitive_interface.getOperation(); + Location loc = op->getLoc(); + OpBuilder builder = OpBuilder::atBlockEnd(op->getBlock()); + + auto perm_attr = [&](Permutation permutation) -> DenseIntElementsAttr { + auto perm_ty = RankedTensorType::get({4}, builder.getIntegerType(64)); + return DenseIntElementsAttr::get(perm_ty, permutation); + }; + + // Change operation data format. + if (failed(layout_sensitive_interface.UpdateDataFormat(target_data_format))) + return; + + // Permute arguments into the target data format. + builder.setInsertionPoint(op); + auto arg_perm = ConstOp::create(builder, loc, perm_attr(args_permutation)); + + for (int64_t arg : layout_sensitive_interface.GetLayoutDependentArgs()) { + op->setOperand(arg, TransposeOp::create(builder, loc, op->getOperand(arg), + arg_perm)); + } + + // Permute results back to the original data format. + builder.setInsertionPointAfter(op); + auto res_perm = ConstOp::create(builder, loc, perm_attr(res_permutation)); + + for (int64_t res : layout_sensitive_interface.GetLayoutDependentResults()) { + OpResult result = op->getResult(res); + + auto transposed_res = TransposeOp::create(builder, loc, result, res_perm); + result.replaceAllUsesWith(transposed_res); + transposed_res.setOperand(0, result); + } + }); +} + +// Move Transpose operations that permute `op` results before the `op`. +void MoveTransposeBefore(Operation* op, SmallVector* work_list) { + // TODO(ezhulenev): Move transpose across layout sensitive operations. + if (!op->hasTrait()) return; + + // Transpose operations that use operation results. + SmallVector transpose_ops; + + // Constant operation that defines permutation indices for result transposes. + ConstOp permutation_op; + + // All operation results must be used by transpose operations with the same + // permutation indices. + for (OpResult result : op->getResults()) { + for (Operation* user : result.getUsers()) { + // Result user must be a transpose operation. + TransposeOp transpose = dyn_cast(user); + if (!transpose) return; + + // With permutation defined by constant operation. + ConstOp perm = + dyn_cast_or_null(transpose.getOperand(1).getDefiningOp()); + if (!perm) return; + + // With the same permutation indices. + auto dense_elem_attr = mlir::dyn_cast(perm.getValue()); + if (!dense_elem_attr) return; + + if (!permutation_op) permutation_op = perm; + + // Check that permutation matches for all result transposes. + if (perm.getValue() != permutation_op.getValue()) return; + + // Add a transpose operation for later reuse. + transpose_ops.push_back(transpose); + } + } + + // Nothing to do here. + if (!permutation_op || transpose_ops.empty()) return; + SmallVector permutation; + auto perm_attr = mlir::cast(permutation_op.getValue()); + for (const auto& value : perm_attr.getValues()) + permutation.push_back(value.getSExtValue()); + + // We want to make sure the shape of the operand equals the transposed shape. + // mismatch can happen if 'op' supports broadcasting and the operands have + // different ranks. + if (op->hasTrait()) { + auto transpose_op = *transpose_ops.begin(); + auto result_type = + mlir::dyn_cast_or_null(transpose_op.getResult().getType()); + auto is_valid_move = + llvm::all_of(op->getOperands(), [result_type](Value operand) -> bool { + auto operand_type = + mlir::dyn_cast_or_null(operand.getType()); + return result_type && operand_type && result_type.hasRank() && + operand_type.hasRank() && + result_type.getRank() == operand_type.getRank(); + }); + if (!is_valid_move) return; + } + + // At this point we checked that we can safely move Transpose node before + // `op`, and bypass all result transposes. + Location loc = op->getLoc(); + + // Move constant op defining result permutation to the beginning of the block. + permutation_op.getOperation()->moveBefore(&op->getBlock()->front()); + + // Bypass Transpose nodes for all results. + for (OpResult result : op->getResults()) { + result.setType( + cast(*result.getUsers().begin()).getY().getType()); + for (Operation* transpose : result.getUsers()) { + transpose->getResult(0).replaceAllUsesWith(result); + } + } + + // Maybe add a Transpose node for all operands (or reuse existing transposes). + OpBuilder builder(op); + builder.setInsertionPoint(op); + + for (OpOperand& operand : op->getOpOperands()) { + // Try to push transpose further up. + if (Operation* operand_op = operand.get().getDefiningOp()) + work_list->push_back(operand_op); + + // Try to reuse result transposes. + TransposeOp transpose = ReuseExistingTranspose( + &operand, permutation, op, permutation_op, &transpose_ops); + // If no transpose available for using, create new one. + if (!transpose) + transpose = + TransposeOp::create(builder, loc, operand.get(), permutation_op); + + operand.set(transpose); + } + + // Remove unused transpose operations. + while (!transpose_ops.empty()) { + TransposeOp transpose = transpose_ops.pop_back_val(); + transpose.erase(); + } +} + +// Revert the permutation applied in `type`. +static mlir::ShapedType ReversePermuteShapedType( + mlir::ShapedType type, ArrayRef permutation) { + if (!type.hasRank()) return type; + + auto shape = type.getShape(); + SmallVector new_shape(shape.size()); + + for (int i = 0; i < permutation.size(); ++i) { + int64_t index = permutation[i]; + assert(index < shape.size()); + new_shape[index] = shape[i]; + } + + return type.clone(new_shape); +} + +// Move Transpose operations that permute `op` operands after the `op`. +void MoveTransposeAfter(Operation* op, SmallVector* work_list, + bool fold_transpose_in_ops) { + // Indices of operands and results that depend on data layout. + SmallVector layout_dependent_operands; + SmallVector layout_dependent_results; + + auto fold_operands = dyn_cast(op); + bool layout_agnostic = op->hasTrait(); + + if (fold_operands && fold_transpose_in_ops) { + layout_dependent_operands = fold_operands.GetLayoutDependentArgs(); + layout_dependent_results = fold_operands.GetLayoutDependentResults(); + + } else if (layout_agnostic) { + // For layout agnostic operation (e.g. element wise operations) all operands + // and results must have the same data layout. + for (unsigned i = 0; i < op->getNumOperands(); ++i) + layout_dependent_operands.push_back(i); + for (unsigned i = 0; i < op->getNumResults(); ++i) + layout_dependent_results.push_back(i); + } + + // Transpose operations that are operands of the `op`. + SmallVector transpose_ops; + + // Constant operation that defines permutation indices for operand transposes. + ConstOp permutation_op; + + // Layout dependent operands must be transpose operations with the same + // permutation indices. + for (unsigned idx : layout_dependent_operands) { + OpOperand& operand = op->getOpOperand(idx); + + // Operand must be defined by a transpose op. + TransposeOp transpose = + dyn_cast_or_null(operand.get().getDefiningOp()); + if (!transpose) return; + + // With permutation defined by constant operation. + ConstOp perm = + dyn_cast_or_null(transpose.getOperand(1).getDefiningOp()); + if (!perm) return; + + // With the same permutation indices. + auto dense_elem_attr = mlir::dyn_cast(perm.getValue()); + if (!dense_elem_attr) return; + + if (!permutation_op) permutation_op = perm; + + // Check that permutation matches for all result transposes. + if (perm.getValue() != permutation_op.getValue()) return; + + // Add a transpose operation for later reuse only if it's used once. + if (transpose.getResult().hasOneUse()) transpose_ops.push_back(transpose); + } + + // Nothing to do here. + if (!permutation_op) return; + + // All results after transpose must preserve the original result type. + SmallVector original_type(op->getNumResults()); + for (unsigned idx : layout_dependent_results) + original_type[idx] = op->getResult(idx).getType(); + + SmallVector permutation; + + auto attr = mlir::cast(permutation_op.getValue()); + for (const auto& value : attr.getValues()) + permutation.push_back(value.getSExtValue()); + + // Check if we can fold transpose into the operation. + if (fold_operands && fold_transpose_in_ops) { + SmallVector permutation; + + auto attr = mlir::cast(permutation_op.getValue()); + for (const auto& value : attr.getValues()) + permutation.push_back(value.getSExtValue()); + + if (failed(fold_operands.FoldOperandsPermutation(permutation))) return; + } + + // At this point we checked that we can safely move Transpose node after + // `op`, bypass all operands transposes, and transpose op results. + Location loc = op->getLoc(); + + // Move constant op defining result permutation to the beginning of the block. + permutation_op.getOperation()->moveBefore(&op->getBlock()->front()); + + // Bypass Transpose nodes for layout dependent operands. + for (unsigned idx : layout_dependent_operands) { + OpOperand& operand = op->getOpOperand(idx); + TransposeOp transpose = + dyn_cast(operand.get().getDefiningOp()); + operand.set(transpose.getOperand(0)); + } + + // Maybe add Transpose nodes for layout dependent results + // (or reuse existing transposes). + OpBuilder builder(op); + builder.setInsertionPointAfter(op); + + for (unsigned idx : layout_dependent_results) { + OpResult result = op->getResult(idx); + + // If the op is layout agnostic, the new result type can be generated by + // reverting `permutation`. Otherwise, operations with custom folding will + // update the result type in `FoldOperandsPermutation`. + if (layout_agnostic) + result.setType(ReversePermuteShapedType( + mlir::cast(result.getType()), permutation)); + + // Try to push transpose further down. + for (Operation* user : result.getUsers()) { + if (!llvm::isa(user)) work_list->push_back(user); + } + + // Try to reuse operand transposes. + TransposeOp transpose; + if (!transpose_ops.empty()) { + transpose = transpose_ops.pop_back_val(); + transpose.getOperation()->moveBefore(op->getNextNode()); + transpose.setOperand(0, result); + transpose.setOperand(1, permutation_op); + transpose.getResult().setType(mlir::cast(original_type[idx])); + } else { + transpose = TransposeOp::create(builder, loc, result, permutation_op); + } + + // Forward all users to the transpose operation. + result.replaceAllUsesWith(transpose); + transpose.setOperand(0, result); + } + + // Remove unused transpose operations. + while (!transpose_ops.empty()) { + TransposeOp transpose = transpose_ops.pop_back_val(); + transpose.erase(); + } +} + +void MoveTransposesPass::runOnOperation() { + func::FuncOp func = getOperation(); + + SmallVector work_list; + + func.walk([&](TransposeOp transpose) { + if (direction_ == MoveTransposeDirection::kBegin) { + // Try to push transpose before the operand operation. + for (auto operand : transpose.getOperands()) { + if (auto op = operand.getDefiningOp()) work_list.push_back(op); + } + } else { + // Try to push transpose after the user operation. + for (Operation* user : transpose.getY().getUsers()) { + if (!llvm::isa(user)) work_list.push_back(user); + } + } + }); + + while (!work_list.empty()) { + Operation* op = work_list.pop_back_val(); + if (direction_ == MoveTransposeDirection::kBegin) { + MoveTransposeBefore(op, &work_list); + } else if (direction_ == MoveTransposeDirection::kEnd) { + MoveTransposeAfter(op, &work_list, fold_transpose_in_ops_); + } + } + + func.walk([&](TransposeOp transpose) { + OpBuilder builder(transpose); + SmallVector fold_result; + if (succeeded(builder.tryFold(transpose.getOperation(), fold_result))) { + assert(fold_result.size() == 1); + transpose.replaceAllUsesWith(fold_result[0]); + } + }); +} + +} // namespace + +void CreateLayoutOptimizationPipeline( + OpPassManager& pm, // NOLINT - MLIR contract is pass by mutable reference. + const LayoutOptimizationPipelineOptions& options) { + // Assign optimal layout for layout sensitive ops. + pm.addPass(std::make_unique(options.force_data_format)); + + // Move transposes to the beginning of the block and try to fold them. + pm.addPass(std::make_unique( + MoveTransposeDirection::kBegin, !options.skip_fold_transpose_in_ops)); + + // Move transposes to the end of the block and try to fold them. + pm.addPass(std::make_unique( + MoveTransposeDirection::kEnd, !options.skip_fold_transpose_in_ops)); +} + +std::unique_ptr> CreateLayoutAssignmentPass() { + // This static is kind of hack, it hooks the pipeline registration for the + // command line and piggy-back to the TableGen generated registration code. + static mlir::PassPipelineRegistration + pipeline("tf-layout-optimization", + "Assigns optimal data layout to all layout sensitive operations " + "and cancel redundant transpose operations.", + CreateLayoutOptimizationPipeline); + return std::make_unique(); +} + +std::unique_ptr> CreateMoveTransposesPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.cc new file mode 100644 index 0000000000000000000000000000000000000000..1408718b8a36bee916af2fcceb9439cb7d091fd7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.cc @@ -0,0 +1,229 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h" + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/resource_var.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/threadpool_options.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +using llvm::SmallSet; +using ::tensorflow::Device; +using ::tensorflow::DeviceMgr; +using ::tensorflow::mutex_lock; +using ::tensorflow::ResourceHandle; +using ::tensorflow::Session; +using ::tensorflow::Status; +using ::tensorflow::StatusOr; +using ::tensorflow::Tensor; +using ::tensorflow::Var; + +namespace { + +constexpr char kResourceNameArgAttr[] = "tf.resource_name"; +constexpr char kSavedModelArgAttr[] = "tf_saved_model.bound_input"; + +LogicalResult LiftVariablesFromSession( + ModuleOp module, Session* session, + const SmallSet& resource_names, + bool import_variables_as_dense_resources) { + OpBuilder builder(module.getBodyRegion()); + + if (!session) return module.emitOpError() << "no session provided"; + + // Read all resource variables from the session. + std::vector variable_names; + variable_names.reserve(resource_names.size()); + for (StringRef name : resource_names) variable_names.push_back(name.str()); + + std::vector resource_tensors; + Status status = session->Run( + /*inputs=*/{}, variable_names, + /*target_tensor_names=*/{}, &resource_tensors); + if (!status.ok()) { + return module.emitOpError() + << "failed to run the provided session: " << status.message(); + } + + const DeviceMgr* device_manager; + if (!(session->LocalDeviceManager(&device_manager).ok())) { + return module.emitOpError() << "failed to get local device manager"; + } + + // Read all underlying tensors of the variables from the session. + std::vector tensors; + tensors.reserve(resource_tensors.size()); + for (const Tensor& resource_tensor : resource_tensors) { + if (resource_tensor.dtype() != tensorflow::DT_RESOURCE) { + tensors.push_back(resource_tensor); + continue; + } + + const ResourceHandle& resource_handle = + resource_tensor.scalar()(); + + Device* device; + if (!(device_manager->LookupDevice(resource_handle.device(), &device) + .ok())) { + return module.emitOpError() << "failed to look up device"; + } + + tensorflow::Var* var_ptr; + if (!(device->resource_manager() + ->Lookup(resource_handle.container(), resource_handle.name(), + &var_ptr) + .ok())) { + return module.emitOpError() << "failed to look up resource value"; + } + tensorflow::core::RefCountPtr var(var_ptr); + + // The variable tensor is already loaded into corresponding device's + // resource manager when we load the saved model using LoadSavedModel(). + // Here we just read its value. + mutex_lock ml(*var->mu()); + tensors.push_back(*var->tensor()); + } + + for (const auto iter : llvm::zip(resource_names, tensors)) { + const StringRef name = std::get<0>(iter); + const Tensor& tensor = std::get<1>(iter); + + // Create tensor attribute for this variable. + absl::StatusOr tensor_attr_or = ConvertTensor( + tensor, &builder, + /*convert_to_dense_resource=*/import_variables_as_dense_resources); + if (!tensor_attr_or.ok()) { + return module.emitOpError() + << "failed to convert tensor (name: " << name.str() << ")- " + << tensor_attr_or.status().ToString(); + } + ElementsAttr tensor_attr = tensor_attr_or.value(); + + tf_saved_model::GlobalTensorOp::create( + builder, NameLoc::get(builder.getStringAttr(name.str())), + builder.getStringAttr(name), tensor_attr, + TypeAttr::get(tensor_attr.getType()), builder.getUnitAttr()); + } + + return success(); +} + +} // namespace + +LogicalResult LiftVariables(ModuleOp module, Session* session, + bool import_variables_as_dense_resources) { + MLIRContext* context = module.getContext(); + mlir::Builder builder(context); + StringAttr resource_name_id = builder.getStringAttr(kResourceNameArgAttr); + + SmallSet resource_names; + + for (func::FuncOp func : module.getOps()) { + for (int i = 0, e = func.getNumArguments(); i < e; ++i) { + auto resource_arg = + func.getArgAttrOfType(i, kResourceNameArgAttr); + if (!resource_arg) continue; + + StringRef resource_name = resource_arg.getValue(); + auto flat_symbol_ref_attr = + FlatSymbolRefAttr::get(context, resource_name); + + // Add the corresponding `tf_saved_model.bound_input` attribute. + func.setArgAttr(i, kSavedModelArgAttr, flat_symbol_ref_attr); + + resource_names.insert(flat_symbol_ref_attr.getValue()); + + // Remove the existing `tf.resource_name` attribute. + func.removeArgAttr(i, resource_name_id); + } + } + + if (resource_names.empty()) return success(); + + if (failed(LiftVariablesFromSession(module, session, resource_names, + /*import_variables_as_dense_resources=*/ + import_variables_as_dense_resources))) + return failure(); + + // Now that we have all global tensors created, we set the corresponding + // bound_inputs' types correctly. + SymbolTable symbol_table(module); + for (auto func : module.getOps()) { + for (auto arg : func.getArguments()) { + unsigned arg_number = arg.getArgNumber(); + auto global_tensor = LookupBoundInputOfType( + func, arg_number, symbol_table); + if (!global_tensor) continue; + + auto arg_type = mlir::cast(arg.getType()); + assert(arg_type.getRank() == 0); + llvm::ArrayRef underlying_type = + mlir::cast(arg_type.getElementType()).getSubtypes(); + + // If the arg type already matches the global_tensor type, we don't need + // to do anything. + if (!underlying_type.empty() && + underlying_type[0] == global_tensor.getType()) { + assert(underlying_type.size() == 1); + continue; + } + + // Otherwise, set this argument's type to the global_tensor's type. + auto new_arg_type = mlir::RankedTensorType::get( + /*shape=*/{}, + mlir::TF::ResourceType::get( + /*subtypes=*/{mlir::cast(global_tensor.getType())}, + module.getContext())); + + arg.setType(new_arg_type); + } + + // Update the function type. + func.setType(mlir::FunctionType::get(module.getContext(), + func.getBody().getArgumentTypes(), + func.getFunctionType().getResults())); + } + return success(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h new file mode 100644 index 0000000000000000000000000000000000000000..090aeb5cc553140ed361b24298f296404ff1244b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h @@ -0,0 +1,35 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LIFT_VARIABLES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LIFT_VARIABLES_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Creates GlobalTensorOp for each variable from function arguments and converts +// them to the corresponding saved model arguments. +LogicalResult LiftVariables(ModuleOp module, ::tensorflow::Session* session, + bool import_variables_as_dense_resources = false); + +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LIFT_VARIABLES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..42c93f14954236d47d9243471b7d3605cde02b5b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lift_variables_test_pass.cc @@ -0,0 +1,83 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/fake_session.h" + +namespace mlir { +namespace tf_test { +namespace { +using ::tensorflow::Session; + +#define GEN_PASS_DEF_LIFTVARIABLESTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +// This pass is only available in the tf-opt binary for testing. +class LiftVariablesTestPass + : public impl::LiftVariablesTestPassBase { + public: + LiftVariablesTestPass() { session_ = new TF::test_util::FakeSession(); } + + ~LiftVariablesTestPass() override { delete session_; } + + void runOnOperation() override { + ModuleOp module = getOperation(); + if (failed(tf_saved_model::LiftVariables( + module, session_, /*import_variables_as_dense_resources=*/ + import_variables_as_dense_resources_))) + signalPassFailure(); + } + + private: + Session* session_; +}; + +#define GEN_PASS_DEF_LIFTVARIABLESINVALIDSESSIONTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +// This pass is only available in the tf-opt binary for testing. +class LiftVariablesInvalidSessionTestPass + : public impl::LiftVariablesInvalidSessionTestPassBase< + LiftVariablesInvalidSessionTestPass> { + public: + void runOnOperation() override { + ModuleOp module = getOperation(); + // Pass an invalid session argument, which is a nullptr. + if (failed(tf_saved_model::LiftVariables(module, /*session=*/nullptr))) + signalPassFailure(); + } +}; + +} // namespace +} // namespace tf_test + +namespace tf_test { + +std::unique_ptr> CreateLiftVariablesTestPass() { + return std::make_unique(); +} + +std::unique_ptr> +CreateLiftVariablesInvalidSessionTestPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/localize_var_handles.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/localize_var_handles.cc new file mode 100644 index 0000000000000000000000000000000000000000..f53b99180d10537500080cd34443092aaa14a57b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/localize_var_handles.cc @@ -0,0 +1,135 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlowFramework.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_dataflow.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_LOCALIZEVARHANDLESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Analyzes the inputs to ClusterFuncOps in the module, and annotates their +// invoked functions whether each input has the same data across replicas. +struct LocalizeVarHandlesPass + : public impl::LocalizeVarHandlesPassBase { + void runOnOperation() override; +}; + +void MaybeCreateVarHandleForOp(Operation* op, DataFlowSolver& solver) { + Value resource; + if (auto read = llvm::dyn_cast(op)) { + resource = read.getResource(); + } else if (auto write = llvm::dyn_cast(op)) { + resource = write.getResource(); + } else if (auto next = llvm::dyn_cast(op)) { + resource = next.getIterator(); + } + + if (llvm::dyn_cast_or_null(resource.getDefiningOp())) { + return; // We're already directly after a VarHandleOp. + } + + const TF::ResourceDataflowState* state = + solver.lookupState(resource); + if (!state) { + // Can't actually happen. Even for gaps in the dataflow, we'll receive + // the initial state. + return; + } + auto ops = state->getValue().ops; + if (ops.size() != 1) { + return; + } + Operation* source = *ops.begin(); + llvm::StringRef container; + llvm::StringRef shared_name; + if (auto global = llvm::dyn_cast(source)) { + container = ""; + shared_name = global.getSymName(); + } else if (auto handle = llvm::dyn_cast(source)) { + container = handle.getContainer(); + shared_name = handle.getSharedName(); + } else if (auto it = llvm::dyn_cast(source)) { + container = it.getContainer(); + shared_name = it.getSharedName(); + } else { + // Can't happen, as long as this file and resource_dataflow.cc are in sync. + return; + } + + OpBuilder builder(op); + Operation* resource_op = nullptr; + if (auto it = llvm::dyn_cast(source)) { + // We can't use VarHandleOp for iterators since they don't have a shape. + // So use IteratorOp instead. (The latter is using LookupOrCreate for + // the resource, and we use it to just look up. + // See core/kernels/data/iterator_ops.cc.) + resource_op = TF::IteratorOp::create( + builder, op->getLoc(), resource.getType(), shared_name, container, + it.getOutputTypes(), it.getOutputShapes()); + } else { + resource_op = TF::VarHandleOp::create( + builder, op->getLoc(), resource.getType(), container, shared_name); + } + op->setOperand(0, resource_op->getResult(0)); +} + +void LocalizeVarHandlesPass::runOnOperation() { + auto module = getOperation(); + + DataFlowSolver solver; + solver.load(); + solver.load(); + TF::LoadResourceDataflowAnalysis(solver); + if (failed(solver.initializeAndRun(module))) return signalPassFailure(); + + OpBuilder globalBuilder(module.getBodyRegion()); + + module.walk( + [&](TF::ReadVariableOp op) { MaybeCreateVarHandleForOp(op, solver); }); + module.walk( + [&](TF::AssignVariableOp op) { MaybeCreateVarHandleForOp(op, solver); }); + module.walk( + [&](TF::IteratorGetNextOp op) { MaybeCreateVarHandleForOp(op, solver); }); +} + +} // namespace + +std::unique_ptr> CreateLocalizeVarHandlesPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.cc new file mode 100644 index 0000000000000000000000000000000000000000..72659082db3d7953529517ef057a214617888860 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.cc @@ -0,0 +1,232 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "mlir/Dialect/Affine/Utils.h" // from @llvm-project +#include "mlir/Dialect/MLProgram/IR/MLProgram.h" // from @llvm-project +#include "mlir/Dialect/MLProgram/IR/MLProgramAttributes.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/RegionGraphTraits.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace tf_saved_model { + +// Trace upwards until we hit a (block) argument a value originates from. +static LogicalResult traceUpwardsToArgument(Value v, llvm::DenseSet seen, + BlockArgument *out) { + if (seen.contains(v)) { + return failure(); // infinite loop + } + seen.insert(v); + + if (auto blockArg = mlir::dyn_cast(v)) { + Operation *op = blockArg.getOwner()->getParentOp(); + + // If we're in the first block, then the argument to that block is the + // one we're looking for. + if (auto func = cast(op)) { + if (blockArg.getOwner()->isEntryBlock()) { + *out = blockArg; + return success(); + } + } + + // If we're in an inner block, the we have to find all ops that branch + // to that block, and trace through them. + llvm::DenseSet options; + for (Block *block : blockArg.getOwner()->getPredecessors()) { + if (!block->empty()) { + Operation *back = &block->back(); + if (BranchOpInterface branchOp = dyn_cast(back)) { + for (int i = 0; i < branchOp->getNumSuccessors(); i++) { + if (branchOp->getSuccessor(i) != blockArg.getOwner()) continue; + SuccessorOperands operands = branchOp.getSuccessorOperands(i); + BlockArgument arg; + if (traceUpwardsToArgument(operands[blockArg.getArgNumber()], seen, + &arg) + .succeeded()) { + options.insert(arg); + } + } + } else { + op->emitOpError("Predecessor op doesn't implement BranchOpInterface"); + return failure(); + } + } + } + if (!options.empty()) { + if (options.size() != 1) { + op->emitOpError("Incompatible code paths."); + return failure(); + } else { + *out = *options.begin(); + return success(); + } + } + return op->emitOpError("Block has no predecessor"); + } + + if (v.getDefiningOp()->getNumOperands() == 1) { + // If the value is originating from an unary op, assume it's something + // simple like "cast" and keep tracing. + return traceUpwardsToArgument(v.getDefiningOp()->getOperand(0), seen, out); + } else { + // Typically a tf.VarHandle op. + return v.getDefiningOp()->emitOpError("Non constant predecessor"); + } +} + +// For the "resource" attribute in a ReadVariable or AssignVariable op, +// determine the symbol reference to the (new) ml_program::GlobalOp. +SymbolRefAttr lookupGlobalTensor(func::FuncOp func, Value resource, + SymbolTable &syms, + DenseMap opToName) { + llvm::DenseSet seen; + BlockArgument arg; + if (traceUpwardsToArgument(resource, seen, &arg).failed()) return nullptr; + Operation *global = + tf_saved_model::LookupBoundInputOfType( + func, arg.getArgNumber(), syms); + auto name = opToName[global]; + if (name.empty()) return nullptr; + return SymbolRefAttr::get(func->getContext(), opToName[global]); +} + +static LogicalResult convertTFGlobals(ModuleOp module) { + OpBuilder globalBuilder(module.getBodyRegion()); + DenseMap opToName; + for (auto globalTensor : module.getOps()) { + auto exportedNames = tf_saved_model::GetExportedNames(globalTensor); + std::string name; + if (exportedNames.empty()) { + name = "global_ml_" + globalTensor.getSymName().str(); + } else if (exportedNames.size() == 1) { + name = exportedNames[0].str(); + } else { + return globalTensor.emitError() + << "Multiple exported names for global tensor not supported yet"; + } + Attribute initial_value; + if (globalTensor.getValue()) { + initial_value = *globalTensor.getValue(); + } else { + initial_value = mlir::Attribute(); + } + opToName[globalTensor] = name; + auto variableOp = ml_program::GlobalOp::create( + globalBuilder, globalTensor.getLoc(), name, globalTensor.getType(), + globalTensor.getIsMutable(), initial_value, + /*visibility=*/globalBuilder.getStringAttr("private")); + variableOp.setPrivate(); + } + + SymbolTable syms(module); + for (auto func : module.getOps()) { + if (!tf_saved_model::IsExported(func)) { + continue; + } + bool success = true; + func.walk([&](mlir::TF::ReadVariableOp op) { + auto sym = lookupGlobalTensor(func, op.getResource(), syms, opToName); + success &= !!sym; + if (!success) return; + OpBuilder builder(op); + auto load = mlir::ml_program::GlobalLoadOp::create( + builder, op.getLoc(), op.getValue().getType(), sym); + op.getValue().replaceAllUsesWith(load.getResult()); + op.erase(); + }); + func.walk([&](mlir::TF::AssignVariableOp op) { + auto sym = lookupGlobalTensor(func, op.getResource(), syms, opToName); + success &= !!sym; + if (!success) return; + OpBuilder builder(op); + mlir::ml_program::GlobalStoreOp::create(builder, op.getLoc(), sym, + op.getValue()); + op.erase(); + }); + if (!success) return failure(); + + // Erase tf_saved_model attributes we consumed. We can't delete them + // right away, since they might weave through blocks, but we can replace + // them with a dummy which DCE can pick up later. + llvm::BitVector argsToErase(func.getNumArguments()); + for (int i = 0; i < func.getNumArguments(); i++) { + if (auto global = tf_saved_model::LookupBoundInputOfType< + tf_saved_model::GlobalTensorOp>(func, i, syms)) { + OpBuilder builder(func.getBody()); + auto dummy = TF::VarHandleOp::create(builder, global.getLoc(), + func.getArgument(i).getType(), + "dummy", "dummy"); + func.getArgument(i).replaceAllUsesWith(dummy.getResult()); + argsToErase.set(i); + } + } + if (failed(func.eraseArguments(argsToErase))) { + return failure(); + } + } + + // Erase all the global tensors. + for (auto globalTensor : llvm::make_early_inc_range( + module.getOps())) { + globalTensor.erase(); + } + return success(); +} + +#define GEN_PASS_DEF_LOWERGLOBALSTOMLPROGRAMPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" +class LowerGlobalsToMlProgram + : public impl::LowerGlobalsToMlProgramPassBase { + public: + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + + void runOnOperation() override { + if (failed(convertTFGlobals(getOperation()))) { + signalPassFailure(); + } + } +}; + +std::unique_ptr CreateLowerGlobalsToMlProgramPass() { + return std::make_unique(); +} + +void RegisterLowerGlobalsToMlProgramPass() { + registerPass(CreateLowerGlobalsToMlProgramPass); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.h new file mode 100644 index 0000000000000000000000000000000000000000..40db0ae802d8c45c8ca1265b8942e214bc8b0bc2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_globals_to_ml_program.h @@ -0,0 +1,35 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_GLOBALS_TO_ML_PROGRAM_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_GLOBALS_TO_ML_PROGRAM_H_ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace tf_saved_model { + +// Create a pass that removes function arguments that map to global tensors. +std::unique_ptr CreateLowerGlobalsToMlProgramPass(); + +// Register this pass in the global registry of MLIR. +void RegisterLowerGlobalsToMlProgramPass(); + +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_GLOBALS_TO_ML_PROGRAM_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_quantized.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_quantized.cc new file mode 100644 index 0000000000000000000000000000000000000000..41c8fe68d6d35d416d2ddc1914a9af303b4526a9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_quantized.cc @@ -0,0 +1,53 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Rewrites ops that require quantized inputs or outputs to ops that allow +// non-quantized inputs and outputs. + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h" + +#define DEBUG_TYPE "tf-lower-quantized" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_LOWERQUANTIZEDPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class LowerQuantizedPass + : public impl::LowerQuantizedPassBase { + public: + void runOnOperation() override { + RewritePatternSet patterns(&getContext()); + mlir::TF::PopulateLoweringQuantizedPatterns(&getContext(), &patterns); + (void)applyPatternsGreedily(getOperation(), std::move(patterns)); + } +}; + +} // namespace + +std::unique_ptr> CreateLowerQuantizedPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..fd31305bcad9e312aae6126c7118a30418ab3574 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc @@ -0,0 +1,1882 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeRange.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_remaining_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/core/util/tensor_format.h" + +namespace mlir { +namespace TF { +namespace { + +// Returns 1D 64-bit dense elements attribute with the given values. +static DenseIntElementsAttr GetI64ElementsAttr(ArrayRef values, + Builder *builder) { + RankedTensorType ty = tensorflow::GetTypeFromTFTensorShape( + {static_cast(values.size())}, builder->getIntegerType(64)); + return DenseIntElementsAttr::get(ty, values); +} + +// Returns a 1-d i64 elements attribute populated with numbers from start to +// end, excluding. +static DenseIntElementsAttr GetI64ElementsAttrForSeq(int start, int end, + Builder *builder) { + int size = end - start; + + SmallVector vals; + vals.resize(size); + std::iota(vals.begin(), vals.end(), start); + + TensorType ty = + tensorflow::GetTypeFromTFTensorShape({size}, builder->getIntegerType(64)); + return DenseIntElementsAttr::get(ty, vals); +} + +// Return an Attr representation of the value. +static DenseElementsAttr GetF32Scalar(OpBuilder *builder, float value) { + return DenseElementsAttr::get( + tensorflow::GetTypeFromTFTensorShape({}, builder->getF32Type()), + FloatAttr::get(builder->getF32Type(), value)); +} + +// Returns a TF_CastOp to F32. This function is used for CastOps that are +// intermediate nodes in a TableGen pattern result. In such a case, the +// destination type is not inferred and must be given explicitly. +// +// Preconditions: The given value must have a ShapedType. +static Value CreateTFCastOpF32(OpBuilder *builder, Location loc, Value x, + BoolAttr truncate) { + auto x_type = mlir::dyn_cast_or_null(x.getType()); + if (!x_type) llvm_unreachable("unsupported type"); + Type type = x_type.clone(builder->getF32Type()); + return CastOp::create(*builder, loc, type, x, truncate); +} + +// Returns a TF_CastOp to I32. This function is used for CastOps that are +// intermediate nodes in a TableGen pattern result. In such a case, the +// destination type is not inferred and must be given explicitly. +// +// Preconditions: The given value must have a ShapedType. +static Value CreateTFCastOpI32(OpBuilder *builder, Location loc, Value x, + BoolAttr truncate) { + auto x_type = mlir::dyn_cast_or_null(x.getType()); + if (!x_type) llvm_unreachable("unsupported type"); + Type type = x_type.clone(builder->getI32Type()); + return CastOp::create(*builder, loc, type, x, truncate); +} + +static APFloat ConvertToAPFloat(double val, Type type) { + if (type.getIntOrFloatBitWidth() == 32) { + return APFloat(static_cast(val)); + } + + return APFloat(val); +} + +// Performs the operation of `Shape(input)[idx]`. +static Value GetDimensionSize(OpBuilder *builder, Location loc, Value input, + int32_t idx, BoolAttr use_32bit) { + if (auto ranked_ty = + mlir::dyn_cast_or_null(input.getType())) { + // Canonicalize negative index. + if (idx < 0) { + idx += ranked_ty.getRank(); + } + // Return a ConstOp if it's static dimension. + if (!ranked_ty.isDynamicDim(idx)) { + return TF::ConstOp::create( + *builder, loc, + GetScalarOfType( + builder->getIntegerType(use_32bit.getValue() ? 32 : 64), + ranked_ty.getDimSize(idx))); + } + } + + auto shape = TF::ShapeOp::create(*builder, loc, input, use_32bit); + return TF::StridedSliceOp::create( + *builder, loc, + mlir::RankedTensorType::get({}, getElementTypeOrSelf(shape)), shape, + /*begin=*/ + TF::ConstOp::create(*builder, loc, builder->getI32TensorAttr({idx})), + /*end=*/ + TF::ConstOp::create(*builder, loc, builder->getI32TensorAttr({idx + 1})), + /*strides=*/ + TF::ConstOp::create(*builder, loc, builder->getI32TensorAttr({1})), + /*begin_mask=*/0, /*end_mask=*/0, /*ellipsis_mask=*/0, + /*new_axis_mask=*/0, /*shrink_axis_mask=*/1); +} + +// Return true if the passed quantized type is unsigned. +bool QuantizedTypeIsUnsigned(Type type) { + return TypeSwitch(type) + .Case([](Type) { return false; }) + .Case([](Type) { return false; }) + .Case([](Type) { return false; }) + .Case([](Type) { return true; }) + .Case([](Type) { return true; }) + .Default([](Type) { + llvm_unreachable("QuantizedTypeIsUnsigned: not a quantized type"); + return false; + }); +} + +// Return the half_range value that is used by DequantizeOp. half_range is used +// to offset the quantized representation before it gets scaled. In the case +// of negative quantize types, this offset is half the type's range. +static DenseElementsAttr DequantizeHalfRange(OpBuilder *builder, Value input) { + auto input_type = mlir::dyn_cast_or_null(input.getType()); + if (!input_type) llvm_unreachable("DequantizeHalfRange: not a ShapedType"); + bool is_unsigned = QuantizedTypeIsUnsigned(input_type.getElementType()); + float half_range = is_unsigned ? 0 : 128; + return GetScalarOfType(builder->getF32Type(), half_range); +} + +// Returns reduction indices to use while lowering tf.BiasAddGrad op to tf.Sum +// op. +DenseIntElementsAttr GetBiasAddGradReductionIndices(int64_t rank, + StringAttr data_format, + Builder *builder) { + tensorflow::TensorFormat format; + if (!FormatFromString(data_format.getValue().str(), &format)) return {}; + + // Reduce along all dimensions except the feature dimension. + int64_t feature_dim = GetTensorFeatureDimIndex(rank, format); + llvm::SmallVector dims_to_reduce(rank - 1); + std::iota(dims_to_reduce.begin(), dims_to_reduce.begin() + feature_dim, 0); + std::iota(dims_to_reduce.begin() + feature_dim, dims_to_reduce.end(), + feature_dim + 1); + return GetI64ElementsAttr(dims_to_reduce, builder); +} + +#include "tensorflow/compiler/mlir/tensorflow/transforms/generated_lower_tf.inc" + +// Infers ExpandDims op output type for the given input type `ty` and dimension +// to expand at the given `axis`. +Type InferExpandDimsType(Type ty, int64_t axis, Builder *builder) { + auto ranked_ty = mlir::dyn_cast(ty); + + // Unranked type. + if (!ranked_ty) return ty; + + auto shape = llvm::to_vector<4>(ranked_ty.getShape()); + if (axis < 0) axis += ranked_ty.getRank() + 1; + + shape.insert(shape.begin() + axis, 1); + return tensorflow::GetTypeFromTFTensorShape(shape, + ranked_ty.getElementType()); +} + +// Converts individual Values to a tensor of rank 1. Each input Value has rank 1 +// and size 1. +Value ValuesToRank1(PatternRewriter &rewriter, Location loc, Type dtype, + ArrayRef vals) { + int64_t length = vals.size(); + auto type = tensorflow::GetTypeFromTFTensorShape({length}, dtype); + auto axis = ConstOp::create(rewriter, loc, + GetScalarOfType(rewriter.getIntegerType(64), 0)); + return ConcatV2Op::create(rewriter, loc, type, ValueRange(vals), axis); +} + +// Lowers AddN op to a sequence of AddV2 ops to accumulate operands. +// +// Note that to improve the parallelism, AddN op uses tree-based reduction. +// For example, tf.AddN([0, 1, 2, 3, 4]) behaves as follows: +// +// 0 1 2 3 4 +// | | | | | +// ------- ------- | +// | | | +// 5 6 | +// | | | +// ------------- | +// | | +// 7 | +// | | +// ---------------- +// | +// 8 +// +// Example: +// +// %result = "tf.AddN"(%0, %1, %2) +// +// is lowered to: +// +// %sum0 = "tf.AddV2"(%0, %1) +// %result = "tf.AddV2"(%sum0, %2) +// +// While +// +// %result = "tf.AddN"(%0, %1, %2, %3, %4) +// +// is lowered to: +// +// %sum0 = "tf.AddV2"(%0, %1) +// %sum1 = "tf.AddV2"(%2, %3) +// %sum2 = "tf.AddV2"(%sum0, %sum1) +// %result = "tf.AddV2"(%sum2, %4) +// +class LowerAddNOp : public RewritePattern { + public: + explicit LowerAddNOp(MLIRContext *context) + : RewritePattern(AddNOp::getOperationName(), 1, context, + {AddV2Op::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *op, + PatternRewriter &rewriter) const override { + auto addn_op = cast(op); + + // TODO(hinsu): Support variant with TensorList type. tf.AddV2 doesn't + // support variant type so variant types require special handling. + if (mlir::isa(getElementTypeOrSelf(addn_op.getType()))) + return failure(); + llvm::SmallVector operands(addn_op.getInputs().begin(), + addn_op.getInputs().end()); + + int64_t n = operands.size(); + // Keep doing tree-based reduction when there are more than one operand. + while (n > 1) { + for (int64_t i = 0; i < n; i += 2) { + // Add two adjacent operands if applicable. + operands[i / 2] = (i + 1 < n) + ? AddV2Op::create(rewriter, addn_op.getLoc(), + operands[i], operands[i + 1]) + : operands[i]; + } + n = (n + 1) / 2; + } + + rewriter.replaceOp(addn_op, operands[0]); + return success(); + } +}; + +// Lowers DynamicStitch op with constant indices and with static input and +// output shapes using Reshape, UnPack and Pack op. +// +// %indices0 = "tf.Const"() {value = dense<4> : tensor} +// %indices1 = "tf.Const"() {value = dense<[[3, 2], [1, 0]]> : +// tensor<2x2xi32>} %0 = "tf.DynamicStitch"(%indices0, %indices1, %arg0, +// %arg1) +// : (tensor, tensor<2x2xi32>, tensor<2xf32>, tensor<2x2x2xf32>) +// -> tensor<5x2xf32> +// +// is lowered to +// +// %shape = "tf.Const"() {value = dense<[-1, 2]> : tensor<2xi64>} +// %inp0 = "tf.Reshape"(%arg0, %shape) +// : (tensor<2xf32>, tensor<2xi64>) -> tensor<1x2xf32> +// %inp1 = "tf.Reshape"(%arg1, %shape) +// : (tensor<2x2x2xf32>, tensor<2xi64>) -> tensor<4x2xf32> +// %items0 = "tf.Unpack"(%[[INP0]]) {axis = 0 : i64} +// : (tensor<1x2xf32>) -> tensor<2xf32> +// %items1:4 = "tf.Unpack"(%[[INP1]]) {axis = 0 : i64} +// : (tensor<4x2xf32>) -> (tensor<2xf32>, tensor<2xf32>, tensor<2xf32>, +// tensor<2xf32>) +// %axis = "tf.Const"() {value = dense<0> : tensor} +// %0 = "tf.Pack"(items1#3, items1#2, items1#1, items1#0, %items0, %axis) +// : (tensor<2xf32>, tensor<2xf32>, tensor<2xf32>, tensor<2xf32>, +// tensor<2xf32>, tensor) -> tensor<5x2xf32> +// +template +class LowerDynamicStitchOp : public RewritePattern { + public: + explicit LowerDynamicStitchOp(MLIRContext *context) + : RewritePattern( + OpT::getOperationName(), 1, context, + {ConstOp::getOperationName(), ReshapeOp::getOperationName(), + UnpackOp::getOperationName(), PackOp::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + // Static output type is used to compute intermediate values. Note that the + // output type doesn't have to be static but if input types and indices are + // constant, then the output type can be statically determined. + RankedTensorType out_ty = mlir::dyn_cast(op.getType()); + if (!out_ty || !out_ty.hasStaticShape()) return failure(); + + // Extract out all the constant indices' attributes and verify that data + // types are static. + SmallVector indices; + indices.reserve(op.getN()); + for (auto it : llvm::zip(op.getIndices(), op.getData())) { + Value index = std::get<0>(it); + Value data = std::get<1>(it); + + DenseIntElementsAttr index_attr; + if (!matchPattern(index, m_Constant(&index_attr))) return failure(); + indices.push_back(index_attr); + + RankedTensorType data_ty = + mlir::dyn_cast(data.getType()); + if (!data_ty || !data_ty.hasStaticShape()) return failure(); + } + + // Compute type of each of the items and shape to use while reshaping inputs + // so that they can be unpacked to extract out individual items. + ArrayRef item_shape = out_ty.getShape().drop_front(1); + auto item_ty = tensorflow::GetTypeFromTFTensorShape( + item_shape, out_ty.getElementType()); + + SmallVector packed_shape; + packed_shape.push_back(-1); + packed_shape.append(item_shape.begin(), item_shape.end()); + Location loc = op.getLoc(); + auto packed_shape_val = ConstOp::create( + rewriter, loc, GetI64ElementsAttr(packed_shape, &rewriter)); + + // Prepare each of the output item by unpacking data and then putting it to + // the specified index. + SmallVector values(out_ty.getDimSize(0)); + for (auto it : llvm::zip(indices, op.getData())) { + DenseIntElementsAttr index_attr = std::get<0>(it); + Value data = std::get<1>(it); + + auto reshaped_data = + ReshapeOp::create(rewriter, loc, data, packed_shape_val); + auto num_items = + mlir::cast(reshaped_data.getType()).getShape()[0]; + auto items = UnpackOp::create(rewriter, loc, + SmallVector(num_items, item_ty), + reshaped_data, + /*axis=*/0); + for (auto index_item : llvm::zip(index_attr, items.getResults())) { + int64_t output_index = std::get<0>(index_item).getSExtValue(); + Value item = std::get<1>(index_item); + values[output_index] = item; + } + } + + rewriter.replaceOpWithNewOp(op, op.getType(), values); + return success(); + } +}; + +// This pass performs a manual conversion with FakeQuant, converting between +// floating point and quantized space. It is designed to reproduce TF's +// implementation, mirroring the previous XLA implementation. +// +// 1. Computing proper quantized bounds. This involves nudging the input bounds. +// 2. Converting the input bounds to quantized space, rounding values. +// 3. Convert back into floating point space. +class ConvertFakeQuantWithMinMaxVarsOp : public RewritePattern { + public: + explicit ConvertFakeQuantWithMinMaxVarsOp(MLIRContext *context) + : RewritePattern( + FakeQuantWithMinMaxVarsOp::getOperationName(), 1, context, + {AddV2Op::getOperationName(), SubOp::getOperationName(), + ConstOp::getOperationName(), MulOp::getOperationName(), + FloorOp::getOperationName(), ClipByValueOp::getOperationName(), + DivOp::getOperationName(), RoundOp::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + auto input = op.getInputs(); + auto input_ty = mlir::cast(input.getType()); + auto element_ty = input_ty.getElementType(); + auto scalar_ty = tensorflow::GetTypeFromTFTensorShape({}, element_ty); + + auto num_bits = op.getNumBits(); + auto narrow_range = op.getNarrowRange(); + const double bits_min = narrow_range ? 1 : 0; + const double bits_max = (1 << num_bits) - 1; + + auto float_min = op.getMin(); + auto float_max = op.getMax(); + + auto float_diff = + SubOp::create(rewriter, op.getLoc(), float_max, float_min); + + // Compute the range when quantized. + auto quant_min = + ConstOp::create(rewriter, op.getLoc(), + DenseElementsAttr::get( + scalar_ty, ConvertToAPFloat(bits_min, element_ty))); + + auto quant_max = + ConstOp::create(rewriter, op.getLoc(), + DenseElementsAttr::get( + scalar_ty, ConvertToAPFloat(bits_max, element_ty))); + + auto quant_diff = ConstOp::create( + rewriter, op.getLoc(), + DenseElementsAttr::get( + scalar_ty, ConvertToAPFloat(bits_max - bits_min, element_ty))); + + auto quant_to_float = + DivOp::create(rewriter, op.getLoc(), float_diff, quant_diff); + + auto float_to_quant = + DivOp::create(rewriter, op.getLoc(), quant_diff, float_diff); + + // During quantization, the quantized min/max values may not line up + // perfectly with the specified min/max. Nudge them into the right range. + auto min_scaled = + DivOp::create(rewriter, op.getLoc(), float_min, quant_to_float); + auto min_scaled_sub = + SubOp::create(rewriter, op.getLoc(), quant_min, min_scaled); + + auto half_val = ConstOp::create( + rewriter, op.getLoc(), + DenseElementsAttr::get(scalar_ty, ConvertToAPFloat(0.5, element_ty))); + Value mid_added = + AddV2Op::create(rewriter, op.getLoc(), min_scaled_sub, half_val); + auto mid_rounded = FloorOp::create(rewriter, op.getLoc(), mid_added); + + auto nudged_zero_point_val = ClipByValueOp::create( + rewriter, op.getLoc(), scalar_ty, mid_rounded, quant_min, quant_max); + + auto quant_min_sub = + SubOp::create(rewriter, op.getLoc(), quant_min, nudged_zero_point_val); + auto quant_max_sub = + SubOp::create(rewriter, op.getLoc(), quant_max, nudged_zero_point_val); + + auto nudged_float_min = + MulOp::create(rewriter, op.getLoc(), quant_min_sub, quant_to_float); + + auto nudged_float_max = + MulOp::create(rewriter, op.getLoc(), quant_max_sub, quant_to_float); + + // Now quantize the input value with the approximated min/max values. + + // Move the input value into quantized space + Value quantized_input = + ClipByValueOp::create(rewriter, op.getLoc(), input_ty, input, + nudged_float_min, nudged_float_max); + + quantized_input = SubOp::create(rewriter, op.getLoc(), input_ty, + quantized_input, nudged_float_min); + + quantized_input = MulOp::create(rewriter, op.getLoc(), input_ty, + quantized_input, float_to_quant); + + // Round the quantized input always to the positive direction. + quantized_input = AddV2Op::create(rewriter, op.getLoc(), input_ty, + quantized_input, half_val); + + quantized_input = FloorOp::create(rewriter, op.getLoc(), quantized_input); + + // Convert back into floating point spae. + Value output = MulOp::create(rewriter, op.getLoc(), input_ty, + quantized_input, quant_to_float); + + output = AddV2Op::create(rewriter, op.getLoc(), input_ty, output, + nudged_float_min); + + rewriter.replaceOp(op, {output}); + return success(); + } +}; + +// Lowers InvertPermutation op to TensorScatterUpdate op. +// +// Example: +// +// %x = "tf.Const"() {value = dense<[3, 4, 0, 1, 2]> : tensor<5xi32>} +// "tf.InvertPermutation"(%x) : (tensor<5xi32>) -> tensor<5xi32> +// +// is lowered to +// +// %x = "tf.Const"() {value = dense<[3, 4, 0, 1, 2]> : tensor<5xi32>} +// %start = "tf.Const"() {value = dense<0> : tensor} +// %limit = "tf.Const"() {value = dense<5> : tensor} +// %delta = "tf.Const"() {value = dense<1> : tensor} +// %updates = "tf.Range"(%start, %limit, %delta) : +// (tensor, tensor, tensor) -> tensor<5xi32> +// %shape = "tf.Const"() {value = dense<[5, 1]> : tensor<2xi32>} +// %indices = "tf.Reshape"(%x, %shape) : (tensor<5xi32, tensor<2xi32) -> +// tensor<5x1xi32> +// "tf.TensorScatterUpdate"(%x, %indices, %updates) : +// (tensor<5xi32>, tensor<5x1xi32>, tensor<5xi32>) -> tensor<5xi32> +// +class LowerInvertPermutationOp : public RewritePattern { + public: + explicit LowerInvertPermutationOp(MLIRContext *context) + : RewritePattern( + InvertPermutationOp::getOperationName(), 1, context, + {ConstOp::getOperationName(), RangeOp::getOperationName(), + ReshapeOp::getOperationName(), + TensorScatterUpdateOp::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + Location loc = op.getLoc(); + auto x_type = mlir::dyn_cast(op.getX().getType()); + // x input must have static shape. + if (!x_type || !x_type.hasStaticShape()) { + return failure(); + } + Type int_type = x_type.getElementType(); // Could be i32 or i64. + + auto result_type = x_type; + auto start = ConstOp::create(rewriter, loc, GetScalarOfType(int_type, 0)); + Value limit = ConstOp::create( + rewriter, loc, GetScalarOfType(int_type, x_type.getShape()[0])); + auto delta = ConstOp::create(rewriter, loc, GetScalarOfType(int_type, 1)); + // Construct a sequence of numbers [0, 1, ... len(x)-1]. + auto updates = + RangeOp::create(rewriter, loc, result_type, start, limit, delta); + + auto shape_type = + tensorflow::GetTypeFromTFTensorShape({2}, rewriter.getIntegerType(32)); + auto shape = ConstOp::create( + rewriter, loc, + DenseElementsAttr::get(shape_type, + {static_cast(x_type.getDimSize(0)), 1})); + auto indices = ReshapeOp::create(rewriter, loc, op.getX(), shape); + + rewriter.replaceOpWithNewOp( + op, result_type, op.getX(), indices, updates); + return success(); + } +}; + +// Approximates lgamma using Lanczos' approximation from +// "A Precision Approximation of the Gamma Function". SIAM Journal on Numerical +// Analysis series B. Vol. 1: +// lgamma(z + 1) = (log(2) + log(pi)) / 2 + (z + 1/2) * log(t(z)) - t(z) + A(z) +// t(z) = z + kLanczosGamma + 1/2 +// A(z) = kBaseLanczosCoeff +// + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k)) +// +// Coefficients for the Lanczos approximation of the gamma function. The +// coefficients are uniquely determined by the choice of g and n +// (kLanczosGamma and kLanczosCoefficients.size() + 1). The coefficients below +// correspond to [7, 9]. [5, 7], [7, 9], [9, 10], and [607/128.0, 15] were +// evaluated and [7, 9] seemed to be the least sensitive to the quality of the +// log function. In particular, [5, 7] is the only choice where -1.5e-5 <= +// lgamma(2) <= 1.5e-5 for a particularly inaccurate log function. +static constexpr double kLanczosGamma = 7; // aka g +static constexpr double kBaseLanczosCoeff = 0.99999999999980993227684700473478; +static constexpr std::array kLanczosCoefficients = { + 676.520368121885098567009190444019, -1259.13921672240287047156078755283, + 771.3234287776530788486528258894, -176.61502916214059906584551354, + 12.507343278686904814458936853, -0.13857109526572011689554707, + 9.984369578019570859563e-6, 1.50563273514931155834e-7}; + +class LowerLgammaOp : public RewritePattern { + public: + explicit LowerLgammaOp(MLIRContext *context) + : RewritePattern(LgammaOp::getOperationName(), 1, context, + { + CastOp::getOperationName(), + ConstOp::getOperationName(), + NegOp::getOperationName(), + SubOp::getOperationName(), + SelectV2Op::getOperationName(), + LessOp::getOperationName(), + AddV2Op::getOperationName(), + DivOp::getOperationName(), + SubOp::getOperationName(), + LogOp::getOperationName(), + Log1pOp::getOperationName(), + IsInfOp::getOperationName(), + MulOp::getOperationName(), + FloorOp::getOperationName(), + AbsOp::getOperationName(), + GreaterOp::getOperationName(), + SinOp::getOperationName(), + IsFiniteOp::getOperationName(), + }) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + Location loc = op.getLoc(); + Value input = op.getX(); + TensorType original_tensor_type = + mlir::cast(op.getX().getType()); + + // The approximation is not precise enough for float16. Do the computation + // in float32 for that case. + TensorType tensor_type = original_tensor_type; + FloatType float_type = mlir::cast(tensor_type.getElementType()); + bool needs_cast = float_type.getWidth() < 32; + if (needs_cast) { + MLIRContext *context = rewriter.getContext(); + float_type = Float32Type::get(context); + if (original_tensor_type.hasRank()) { + tensor_type = tensorflow::GetTypeFromTFTensorShape( + original_tensor_type.getShape(), float_type); + } else { + tensor_type = UnrankedTensorType::get(float_type); + } + input = CastOp::create(rewriter, loc, tensor_type, input); + } + + // Helper lambda function for creating a ConstOp for a tensor filled with + // the given constant float value. + auto create_const_op = [&rewriter, loc, tensor_type, + float_type](double value) { + return ConstOp::create( + rewriter, loc, + DenseElementsAttr::get(tensor_type, + FloatAttr::get(float_type, value))); + }; + + Value one_half = create_const_op(0.5); + Value one = create_const_op(1.0); + Value infinity = create_const_op(std::numeric_limits::infinity()); + Value pi = create_const_op(M_PI); + Value log_pi = create_const_op(std::log(M_PI)); + Value log_sqrt_two_pi = create_const_op((std::log(2) + std::log(M_PI)) / 2); + Value lanczos_gamma_plus_one_half = create_const_op(kLanczosGamma + 0.5); + Value log_lanczos_gamma_plus_one_half = + create_const_op(std::log(kLanczosGamma + 0.5)); + Value base_lanczos_coeff = create_const_op(kBaseLanczosCoeff); + + Value minus_input = NegOp::create(rewriter, loc, input); + Value input_minus_one = SubOp::create(rewriter, loc, input, one); + + // If the input is less than 0.5 use Euler's reflection formula: + // gamma(x) = pi / (sin(pi * x) * gamma(1 - x)) + Value need_to_reflect = LessOp::create(rewriter, loc, input, one_half); + Type tensor_bool_type = need_to_reflect.getType(); + Value z = SelectV2Op::create(rewriter, loc, need_to_reflect, minus_input, + input_minus_one); + + Value x = base_lanczos_coeff; + for (int i = 0, end = kLanczosCoefficients.size(); i < end; ++i) { + Value lanczos_coefficient = create_const_op(kLanczosCoefficients[i]); + Value index = create_const_op(static_cast(i)); + Value z_plus_index = AddV2Op::create(rewriter, loc, z, index); + Value z_plus_index_plus_one = + AddV2Op::create(rewriter, loc, z_plus_index, one); + Value incr = DivOp::create(rewriter, loc, lanczos_coefficient, + z_plus_index_plus_one); + x = AddV2Op::create(rewriter, loc, x, incr); + } + + // To improve accuracy on platforms with less-precise log implementations, + // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on + // the device. + // log(t) = log(kLanczosGamma + 0.5 + z) + // = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5)) + Value t = AddV2Op::create(rewriter, loc, lanczos_gamma_plus_one_half, z); + Value z_div_lanczos_gamma_plus_one_half = + DivOp::create(rewriter, loc, z, lanczos_gamma_plus_one_half); + Value log1p_z_div_lanczos_gamma_plus_one_half = + Log1pOp::create(rewriter, loc, z_div_lanczos_gamma_plus_one_half); + Value log_t = + AddV2Op::create(rewriter, loc, log_lanczos_gamma_plus_one_half, + log1p_z_div_lanczos_gamma_plus_one_half); + + // Compute the final result (modulo reflection). t(z) may be large, and we + // need to be careful not to overflow to infinity in the first term of + // + // (z + 1/2) * log(t(z)) - t(z). + // + // Therefore we compute this as + // + // (z + 1/2 - t(z) / log(t(z))) * log(t(z)). + // + // log_y = log_sqrt_two_pi + (z + one_half - t / log_t) * log_t + Log(x); + Value t_div_log_t = DivOp::create(rewriter, loc, t, log_t); + Value one_half_minus_t_div_log_t = + SubOp::create(rewriter, loc, one_half, t_div_log_t); + Value z_plus_one_half_minus_t_div_log_t = + AddV2Op::create(rewriter, loc, z, one_half_minus_t_div_log_t); + Value z_plus_one_half_minus_t_div_log_t_mul_log_t = + MulOp::create(rewriter, loc, z_plus_one_half_minus_t_div_log_t, log_t); + Value log_x = LogOp::create(rewriter, loc, x); + Value log_y_rhs = AddV2Op::create( + rewriter, loc, z_plus_one_half_minus_t_div_log_t_mul_log_t, log_x); + Value log_y = AddV2Op::create(rewriter, loc, log_sqrt_two_pi, log_y_rhs); + + // Compute the reflected value, used when x < 0.5: + // + // lgamma(x) = log(pi) - lgamma(1-x) - log(abs(sin(pi * x))). + // + // (The abs is because lgamma is the log of the absolute value of the gamma + // function.) + // + // We have to be careful when computing the final term above. gamma(x) goes + // to +/-inf at every integer x < 0, and this is controlled by the + // sin(pi * x) term. The slope is large, so precision is particularly + // important. + // + // Because abs(sin(pi * x)) has period 1, we can equivalently use + // abs(sin(pi * frac(x))), where frac(x) is the fractional part of x. This + // is more numerically accurate: It doesn't overflow to inf like pi * x can, + // and if x is an integer, it evaluates to 0 exactly, which is significant + // because we then take the log of this value, and log(0) is inf. + // + // We don't have a frac(x) primitive in XLA and computing it is tricky, but + // because abs(sin(pi * x)) = abs(sin(pi * abs(x))), it's good enough for + // our purposes to use abs(frac(x)) = abs(x) - floor(abs(x)). + // + // Furthermore, pi * abs(frac(x)) loses precision when abs(frac(x)) is close + // to 1. To remedy this, we can use the fact that sin(pi * x) in the domain + // [0, 1] is symmetric across the line Y=0.5. + Value abs_input = AbsOp::create(rewriter, loc, input); + Value abs_input_floor = FloorOp::create(rewriter, loc, abs_input); + Value abs_frac_input = + SubOp::create(rewriter, loc, abs_input, abs_input_floor); + + // Convert values of abs_frac_input > 0.5 to (1 - frac_input) to improve + // precision of pi * abs_frac_input for values of abs_frac_input close to 1. + Value one_minus_abs_frac_input = + SubOp::create(rewriter, loc, one, abs_frac_input); + Value abs_frac_input_gt_one_half = + GreaterOp::create(rewriter, loc, abs_frac_input, one_half); + Value reduced_frac_input = + SelectV2Op::create(rewriter, loc, abs_frac_input_gt_one_half, + one_minus_abs_frac_input, abs_frac_input); + Value pi_mul_reduced_frac_input = + MulOp::create(rewriter, loc, pi, reduced_frac_input); + Value sin_pi_mul_reduced_frac_input = + SinOp::create(rewriter, loc, pi_mul_reduced_frac_input); + Value reflection_denom = + LogOp::create(rewriter, loc, sin_pi_mul_reduced_frac_input); + + // Avoid computing -inf - inf, which is nan. If reflection_denom is +/-inf, + // then it "wins" and the result is +/-inf. + Value is_finite = + IsFiniteOp::create(rewriter, loc, tensor_bool_type, reflection_denom); + Value neg_reflection_denom = NegOp::create(rewriter, loc, reflection_denom); + Value log_pi_minus_reflection_denom = + SubOp::create(rewriter, loc, log_pi, reflection_denom); + Value reflection_if_finite = + SubOp::create(rewriter, loc, log_pi_minus_reflection_denom, log_y); + Value reflection = SelectV2Op::create( + rewriter, loc, is_finite, reflection_if_finite, neg_reflection_denom); + + Value result = + SelectV2Op::create(rewriter, loc, need_to_reflect, reflection, log_y); + + // lgamma(+/-inf) = +inf. + Value is_inf = IsInfOp::create(rewriter, loc, tensor_bool_type, input); + result = SelectV2Op::create(rewriter, loc, is_inf, infinity, result); + + if (needs_cast) { + result = CastOp::create(rewriter, loc, original_tensor_type, result); + } + + rewriter.replaceOp(op, result); + return success(); + } +}; + +// Lowers Pack op to ConcatV2 op after changing shape of the inputs with +// ExpandDims op. +// +// Sample result with 2 inputs to pack: +// +// %axis = "tf.Const"() {value = dense<1> : tensor} +// %inp0 = "tf.ExpandDims"(%operand0, %axis): tensor<2xf32> -> tensor<2x1xf32> +// %inp1 = "tf.ExpandDims"(%operand1, %axis): tensor<2xf32> -> tensor<2x1xf32> +// %result = "tf.ConcatV2"(%operand0, %operand1, %axis) { N = 2 : i64 }: +// +class LowerPackOp : public RewritePattern { + public: + explicit LowerPackOp(MLIRContext *context) + : RewritePattern( + PackOp::getOperationName(), 1, context, + {ConstOp::getOperationName(), ConcatV2Op::getOperationName(), + ExpandDimsOp::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + Location loc = op.getLoc(); + auto axis_value = ConstOp::create( + rewriter, loc, + DenseElementsAttr::get(tensorflow::GetTypeFromTFTensorShape( + {}, rewriter.getIntegerType(64)), + op.getAxis())); + int64_t axis = op.getAxis(); + + Type prev_input_ty, inferred_ty; + SmallVector expanded_inputs; + expanded_inputs.reserve(op.getN()); + for (Value input : op.getValues()) { + // If input type is different than the previous input type, infer the + // output type. Otherwise, use the already inferred output type from the + // previous iteration. + Type input_ty = input.getType(); + if (input_ty != prev_input_ty) { + inferred_ty = InferExpandDimsType(input_ty, axis, &rewriter); + prev_input_ty = input_ty; + } + expanded_inputs.push_back( + ExpandDimsOp::create(rewriter, loc, inferred_ty, input, axis_value)); + } + + rewriter.replaceOpWithNewOp(op, op.getType(), expanded_inputs, + axis_value); + return success(); + } +}; + +// Lowers SpaceToBatchND by reducing to reshape(transpose(reshape(pad(input)))). +// +// Before rewrite: +// output = SpaceToBatchND(input, block_shape, paddings) +// Let: +// [batch] + spatial_shape + remaining_shape = input.shape +// M = spatial_shape.rank +// After rewrite: +// padded = zero-pad input with paddings +// The spatial_shape component of input.shape pads with paddings[*, 0] +// before each dimension, and paddings[*, 1] after each dimension. +// reshaped = reshape padded to: +// [batch] +// + [padded.shape[1]/block_shape[0], block_shape[0], ..., +// padded.shape[M]/block_shape[M-1], block_shape[M-1]] +// + remaining_shape +// permuted = transpose reshaped to: +// block_shape +// + [batch] +// + [padded.shape[1]/block_shape[0], ..., padded.shape[M]/block_shape[M-1]] +// + remaining_shape +// result = reshape permuted to: +// [batch * product(block_shape)] +// + [padded.shape[1]/block_shape[0], ..., padded.shape[M]/block_shape[M-1]] +// + remaining_shape +class LowerSpaceToBatchNDOp : public RewritePattern { + public: + explicit LowerSpaceToBatchNDOp(MLIRContext *context) + : RewritePattern(SpaceToBatchNDOp::getOperationName(), 1, context, + { + CastOp::getOperationName(), + ConstOp::getOperationName(), + ConcatV2Op::getOperationName(), + AddV2Op::getOperationName(), + PadOp::getOperationName(), + SplitOp::getOperationName(), + UnpackOp::getOperationName(), + DivOp::getOperationName(), + MulOp::getOperationName(), + ReshapeOp::getOperationName(), + TransposeOp::getOperationName(), + }) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + Location loc = op.getLoc(); + auto input_type = mlir::cast(op.getInput().getType()); + auto element_type = input_type.getElementType(); + if (!input_type.hasStaticShape()) { + return failure(); + } + ArrayRef input_shape = input_type.getShape(); + auto block_shape_type = + mlir::cast(op.getBlockShape().getType()); + if (!block_shape_type.hasStaticShape()) { + return failure(); + } + auto paddings_type = mlir::cast(op.getPaddings().getType()); + if (!paddings_type.hasRank()) { + return failure(); + } + + int64_t input_rank = input_type.getRank(); + int64_t block_rank = block_shape_type.getNumElements(); + int64_t remaining_rank = input_rank - 1 - block_rank; + if (remaining_rank < 0) { + // TODO(b/157475606): Move this check to ::Verify + return failure(); + } + + auto block_shape_i64_type = tensorflow::GetTypeFromTFTensorShape( + block_shape_type.getShape(), rewriter.getIntegerType(64)); + auto block_shape_i64 = + CastOp::create(rewriter, loc, block_shape_i64_type, op.getBlockShape()); + + auto paddings_i64_type = tensorflow::GetTypeFromTFTensorShape( + paddings_type.getShape(), rewriter.getIntegerType(64)); + auto paddings_i64 = + CastOp::create(rewriter, loc, paddings_i64_type, op.getPaddings()); + + auto pad00 = ConstOp::create(rewriter, loc, + DenseElementsAttr::get( + tensorflow::GetTypeFromTFTensorShape( + {1, 2}, rewriter.getIntegerType(64)), + {0, 0})); + SmallVector full_paddings_list{pad00, paddings_i64}; + full_paddings_list.append(remaining_rank, pad00); + auto full_paddings_type = tensorflow::GetTypeFromTFTensorShape( + {input_rank, 2}, rewriter.getIntegerType(64)); + auto zero_i64 = ConstOp::create( + rewriter, loc, GetScalarOfType(rewriter.getIntegerType(64), 0)); + // Extends paddings to all dimensions of input by adding 0s to non-block + // dimensions. + auto full_paddings = ConcatV2Op::create(rewriter, loc, full_paddings_type, + full_paddings_list, zero_i64); + + // Compute the result type here instead of using shape inference because the + // full_paddings won't be available as a constant for shape inference. + ElementsAttr block_shape; + ElementsAttr paddings; + llvm::SmallVector block_shape_ints; + auto padded_shape = llvm::to_vector<4>(input_shape); + if (matchPattern(op.getBlockShape(), m_Constant(&block_shape)) && + matchPattern(op.getPaddings(), m_Constant(&paddings))) { + for (uint64_t i = 0; i < block_rank; i++) { + int64_t paddings_sum = + paddings.getValues()[{i, 0}].getSExtValue() + + paddings.getValues()[{i, 1}].getSExtValue(); + int64_t block_shape_i = + block_shape.getValues()[i].getSExtValue(); + padded_shape[i + 1] = (paddings_sum + input_shape[i + 1]); + block_shape_ints.push_back(block_shape_i); + } + } else { + for (int i = 0; i < block_rank; i++) { + padded_shape[i + 1] = ShapedType::kDynamic; + } + block_shape_ints.resize(block_shape_type.getNumElements(), -1); + } + + auto padded_type = + tensorflow::GetTypeFromTFTensorShape(padded_shape, element_type); + // padded = pad(input, full_paddings) + auto padded = + PadOp::create(rewriter, loc, padded_type, op.getInput(), full_paddings); + + auto paddings_sum_type = tensorflow::GetTypeFromTFTensorShape( + {input_rank}, rewriter.getIntegerType(64)); + // paddings_sum = paddings[*,0] + paddings[*,1] + auto paddings_split = UnpackOp::create( + rewriter, loc, TypeRange({paddings_sum_type, paddings_sum_type}), + full_paddings, rewriter.getI64IntegerAttr(1)); + auto paddings_sum = + AddV2Op::create(rewriter, loc, paddings_split.getResult(0), + paddings_split.getResult(1)); + + auto input_shape_tensor = ConstOp::create( + rewriter, loc, + DenseElementsAttr::get(tensorflow::GetTypeFromTFTensorShape( + {input_rank}, rewriter.getIntegerType(64)), + input_shape)); + + // padded_shape_tensor is the shape of padded. + auto padded_shape_tensor = + AddV2Op::create(rewriter, loc, paddings_sum, input_shape_tensor); + + auto zero_i32 = ConstOp::create( + rewriter, loc, GetScalarOfType(rewriter.getIntegerType(32), 0)); + SmallVector padded_shape_splits_types( + input_rank, + tensorflow::GetTypeFromTFTensorShape({1}, rewriter.getIntegerType(64))); + SmallVector padded_shape_splits( + SplitOp::create(rewriter, loc, padded_shape_splits_types, zero_i32, + padded_shape_tensor) + .getOutput()); + + SmallVector block_shape_splits_types( + block_rank, + tensorflow::GetTypeFromTFTensorShape({1}, rewriter.getIntegerType(64))); + SmallVector block_shape_splits( + SplitOp::create(rewriter, loc, block_shape_splits_types, zero_i32, + block_shape_i64) + .getOutput()); + + SmallVector outer_shape_ints; + SmallVector outer_shape_vals; + for (int64_t i = 0; i < block_rank; ++i) { + // TODO(b/157475606): Insert tf.Assert that the following division has + // remainder 0. + outer_shape_vals.push_back(DivOp::create( + rewriter, loc, padded_shape_splits[1 + i], block_shape_splits[i])); + + auto padded_shape_i = padded_shape[1 + i]; + auto block_shape_ints_i = block_shape_ints[i]; + + // Compute the outer_shape constant values to infer the reshape. + if (padded_shape_i == -1 || block_shape_ints_i == -1) { + outer_shape_ints.push_back(-1); + } else { + outer_shape_ints.push_back(padded_shape_i / block_shape_ints_i); + } + } + + SmallVector reshaped_shape_vals{padded_shape_splits[0]}; + SmallVector reshaped_shape_ints{padded_shape[0]}; + for (int64_t i = 0; i < block_rank; ++i) { + reshaped_shape_vals.push_back(outer_shape_vals[i]); + reshaped_shape_vals.push_back(block_shape_splits[i]); + + reshaped_shape_ints.push_back(outer_shape_ints[i]); + reshaped_shape_ints.push_back(block_shape_ints[i]); + } + for (int64_t i = 1 + block_rank; i < input_rank; ++i) { + reshaped_shape_vals.push_back(padded_shape_splits[i]); + reshaped_shape_ints.push_back(padded_shape[i]); + } + auto reshaped_shape = ValuesToRank1( + rewriter, loc, rewriter.getIntegerType(64), reshaped_shape_vals); + + auto reshaped = ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(reshaped_shape_ints, element_type), + padded, reshaped_shape); + + SmallVector permutation_vals; + for (int64_t i = 0; i < block_rank; ++i) { + permutation_vals.push_back(2 + 2 * i); + } + permutation_vals.push_back(0); + for (int64_t i = 0; i < block_rank; ++i) { + permutation_vals.push_back(1 + 2 * i); + } + for (int64_t i = 1 + block_rank; i < input_rank; ++i) { + permutation_vals.push_back(block_rank + i); + } + auto permutation = ConstOp::create( + rewriter, loc, GetI64ElementsAttr(permutation_vals, &rewriter)); + + auto permuted = TransposeOp::create(rewriter, loc, reshaped, permutation); + auto output_batch = padded_shape_splits[0]; + for (int64_t i = 0; i < block_rank; ++i) { + output_batch = + MulOp::create(rewriter, loc, output_batch, block_shape_splits[i]); + } + SmallVector output_shape_vals{output_batch}; + for (int64_t i = 0; i < block_rank; ++i) { + output_shape_vals.push_back(outer_shape_vals[i]); + } + for (int64_t i = 1 + block_rank; i < input_rank; ++i) { + output_shape_vals.push_back(padded_shape_splits[i]); + } + auto output_shape = ValuesToRank1( + rewriter, loc, rewriter.getIntegerType(64), output_shape_vals); + + // Sometimes the result type is more specific than what the reshape builder + // can infer. + auto result_type = op.getResult().getType(); + rewriter.replaceOpWithNewOp(op, result_type, permuted, + output_shape); + + return success(); + } +}; + +class LowerBatchToSpaceND : public RewritePattern { + public: + explicit LowerBatchToSpaceND(MLIRContext *context) + : RewritePattern(BatchToSpaceNDOp::getOperationName(), 1, context, + { + ConstOp::getOperationName(), + ReshapeOp::getOperationName(), + SliceOp::getOperationName(), + TransposeOp::getOperationName(), + }) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + auto input = op.getInput(); + auto input_ty = mlir::cast(input.getType()); + auto element_ty = input_ty.getElementType(); + if (!input_ty.hasStaticShape()) { + return failure(); + } + + const int input_rank = input_ty.getRank(); + auto input_shape = input_ty.getShape(); + + DenseIntElementsAttr block_shape; + DenseIntElementsAttr crops; + if (!matchPattern(op.getBlockShape(), m_Constant(&block_shape)) || + !matchPattern(op.getCrops(), m_Constant(&crops))) { + return failure(); + } + + auto block_shape_ty = block_shape.getType(); + if (!block_shape_ty.hasRank() || block_shape_ty.getRank() != 1) { + return failure(); + } + + const int block_rank = block_shape_ty.getShape().front(); + auto remainder_shape = input_shape.drop_front(1 + block_rank); + + const int64_t batch_size = input_shape[0]; + + // Compute the product of the block_shape values. + int64_t block_num_elems = 1; + + for (auto val : block_shape.getValues()) { + block_num_elems *= val.getSExtValue(); + } + + if (block_num_elems <= 0) { + op.emitOpError() + << "The product of the block dimensions must be positive"; + return failure(); + } + + // 1. Reshape `input` to `reshaped` of shape: + // [block_shape[0], ..., block_shape[M-1], + // batch / prod(block_shape), + // input_shape[1], ..., input_shape[N-1]] + SmallVector reshaped_shape; + reshaped_shape.reserve(block_shape.size()); + for (auto val : block_shape) { + reshaped_shape.push_back(val.getSExtValue()); + } + reshaped_shape.resize(input_rank + block_rank); + + reshaped_shape[block_rank] = batch_size / block_num_elems; + std::copy(input_shape.begin() + 1, input_shape.end(), + reshaped_shape.begin() + block_rank + 1); + + auto reshaped = TF::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(reshaped_shape, element_ty), input, + ConstOp::create(rewriter, op.getLoc(), + rewriter.getI64TensorAttr(reshaped_shape))); + + // 2. Permute dimensions of `reshaped` to produce `permuted` of shape + // [batch / prod(block_shape), + // + // input_shape[1], block_shape[0], + // ..., + // input_shape[M], block_shape[M-1], + // + // input_shape[M+1], ..., input_shape[N-1]] + SmallVector permutation(reshaped_shape.size()); + permutation[0] = block_rank; + for (int i = 0; i < block_rank; ++i) { + permutation[1 + 2 * i] = block_rank + 1 + i; + permutation[1 + 2 * i + 1] = i; + } + std::iota(permutation.begin() + 1 + block_rank * 2, permutation.end(), + 1 + block_rank * 2); + + SmallVector transpose_shape(permutation.size()); + for (auto it : llvm::enumerate(permutation)) { + transpose_shape[it.index()] = reshaped_shape[it.value()]; + } + + auto permuted = TF::TransposeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(transpose_shape, element_ty), + reshaped, + ConstOp::create(rewriter, op.getLoc(), + rewriter.getI64TensorAttr(permutation))); + + // 3. Reshape `permuted` to produce `reshaped_permuted` of shape + // [batch / prod(block_shape), + // + // input_shape[1] * block_shape[0], + // ..., + // input_shape[M] * block_shape[M-1], + // + // input_shape[M+1], + // ..., + // input_shape[N-1]] + SmallVector reshaped_permuted_shape(input_rank); + auto block_shape_values = + llvm::to_vector<4>(block_shape.getValues()); + reshaped_permuted_shape[0] = batch_size / block_num_elems; + for (int i = 0; i < block_rank; ++i) { + reshaped_permuted_shape[1 + i] = + block_shape_values[i].getSExtValue() * input_shape[1 + i]; + } + std::copy(remainder_shape.begin(), remainder_shape.end(), + reshaped_permuted_shape.begin() + 1 + block_rank); + + auto reshaped_permuted = TF::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(reshaped_permuted_shape, + element_ty), + permuted, + ConstOp::create(rewriter, op.getLoc(), + rewriter.getI64TensorAttr(reshaped_permuted_shape))); + + // 4. Crop the start and end of dimensions `[1, ..., M]` of + // `reshaped_permuted` according to `crops` to produce the output of + // shape: + // [batch / prod(block_shape), + // + // input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1], + // ..., + // input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1], + // + // input_shape[M+1], ..., input_shape[N-1]] + SmallVector start_indices(input_rank, 0); + SmallVector slice_sizes = reshaped_permuted_shape; + SmallVector strides(input_rank, 1); + auto crop_values = llvm::to_vector<4>(crops.getValues()); + for (int i = 0; i < block_rank; ++i) { + int64_t crop_start = crop_values[i * 2].getSExtValue(); + int64_t crop_end = crop_values[i * 2 + 1].getSExtValue(); + + if (crop_start < 0 || crop_end < 0) { + op.emitOpError() << "Crops must be non-negative"; + return failure(); + } + + start_indices[i + 1] = crop_start; + slice_sizes[i + 1] -= crop_start + crop_end; + + if (slice_sizes[i + 1] < 0) { + op.emitOpError() << "Cropped size must be non-negative: start: " + << crop_start << " end: " << crop_end << " size " + << reshaped_permuted_shape[1 + i]; + } + } + + rewriter.replaceOpWithNewOp( + op, tensorflow::GetTypeFromTFTensorShape(slice_sizes, element_ty), + reshaped_permuted, + ConstOp::create(rewriter, op.getLoc(), + rewriter.getI64TensorAttr(start_indices)), + ConstOp::create(rewriter, op.getLoc(), + rewriter.getI64TensorAttr(slice_sizes))); + return success(); + } +}; + +// Lowers `SparseMatMulOp` to `MatMulOp`, ignoring the sparseness hints, +// since we currently don't have an implementation that can use this +// information. Adds appropriate casts where necessary to align element types +// of operands and result for `MatMulOp`. +class LowerSparseMatMulOp : public RewritePattern { + public: + explicit LowerSparseMatMulOp(MLIRContext *context) + : RewritePattern( + SparseMatMulOp::getOperationName(), 1, context, + {CastOp::getOperationName(), MatMulOp::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + + // Result type must be f32 for applying the pattern (currently this is + // required by the op anyway but this might change). + if (!mlir::cast(op.getProduct().getType()) + .getElementType() + .isF32()) { + return failure(); + } + MLIRContext *context = rewriter.getContext(); + llvm::SmallVector operands{op.getA(), op.getB()}; + for (Value &operand : operands) { + TensorType tensor_type = mlir::cast(operand.getType()); + Type element_type = tensor_type.getElementType(); + if (element_type.isF32()) continue; + // Element type can either be f32 or bf16 for `SparseMatMulOp` so it + // must be bf16 here. + assert(element_type.isBF16()); + Type tensor_type_f32; + if (tensor_type.hasRank()) { + tensor_type_f32 = tensorflow::GetTypeFromTFTensorShape( + tensor_type.getShape(), Float32Type::get(context)); + } else { + tensor_type_f32 = UnrankedTensorType::get(Float32Type::get(context)); + } + // Add cast to f32 to conform with element type of result. + operand = CastOp::create(rewriter, op.getLoc(), tensor_type_f32, operand); + } + Value result = MatMulOp::create( + rewriter, op.getLoc(), op.getProduct().getType(), operands[0], + operands[1], op.getTransposeA(), op.getTransposeB()); + + rewriter.replaceOp(op, {result}); + return success(); + } +}; + +// Lowers _UnaryOpsComposition op as a series of original TensorFlow ops that +// were fused together. +class Lower_UnaryOpsComposition + : public OpRewritePattern<_UnaryOpsCompositionOp> { + public: + using OpRewritePattern<_UnaryOpsCompositionOp>::OpRewritePattern; + + LogicalResult matchAndRewrite(_UnaryOpsCompositionOp op, + PatternRewriter &rewriter) const override { + Value result = op.getX(); + for (StringRef op_name : op.getOpNames().getAsValueRange()) { + std::string full_name = "tf." + op_name.str(); + // All ops in the sequences have the same result type as the original + // result type. + OperationState state(op.getLoc(), full_name, /*operands=*/{result}, + /*types=*/{op.getType()}, /*attributes=*/{}); + Operation *op = rewriter.create(state); + result = op->getResult(0); + } + rewriter.replaceOp(op, {result}); + return success(); + } +}; + +// Lowers ResizeNearestNeighbor to an indices computations with a gather along +// the combined spatial dimensions. Generating the indices along the +// width/height index could be used to gather along each of W and H dimension +// of the input image array. To reduce to a single gather, these indices are +// combined, so a single gather can be performed along the combined spatial +// dimensions. +// +// Images must take the shape [b, h, w, c] and size is a rank-1 length-2 tensor +// containing the height and width values for the output tensor. This lowering +// should work with a dynamic images array. +// +// For example, a scaling with image shape [1, 3, 3, 1] to [2, 2] and unaligned +// corners would generate a [0, 1] lookup along both the x and y direction. +// Then when combined to form the 1-D spatial index the values would be +// [0, 1, 3, 4] which would gather along the reshape image tensor of shape +// [1, 9, 1], reshaped to the final [1, 3, 3, 1]. +class LowerResizeNearestNeighbor : public RewritePattern { + public: + explicit LowerResizeNearestNeighbor(MLIRContext *context) + : RewritePattern(ResizeNearestNeighborOp::getOperationName(), 1, context, + { + BroadcastToOp::getOperationName(), + ConstOp::getOperationName(), + DivOp::getOperationName(), + PackOp::getOperationName(), + RangeOp::getOperationName(), + ReshapeOp::getOperationName(), + ShapeOp::getOperationName(), + SplitOp::getOperationName(), + TransposeOp::getOperationName(), + }) {} + + LogicalResult matchAndRewrite(Operation *src_op, + PatternRewriter &rewriter) const override { + auto op = cast(src_op); + auto loc = op.getLoc(); + auto result_ty = mlir::cast(op.getType()); + + auto input = op.getImages(); + auto input_ty = mlir::cast(input.getType()); + auto input_element_ty = input_ty.getElementType(); + auto out_size = op.getSize(); + auto out_size_ty = mlir::cast(out_size.getType()); + auto out_size_element_ty = out_size_ty.getElementType(); + + // Input should be rank 4. + if (!input_ty.hasRank() || input_ty.getRank() != 4) { + return failure(); + } + + // Check that out_size is rank-1, length-2. Otherwise the size is not legal. + if (!out_size_ty.hasRank() || out_size_ty.getRank() != 1 || + out_size_ty.getShape()[0] != 2) { + return failure(); + } + + // Extract the output width / height dim size. + int out_height_constant = -1; + int out_width_constant = -1; + DenseIntElementsAttr out_size_cst; + if (matchPattern(out_size, m_Constant(&out_size_cst))) { + llvm::SmallVector cst_size; + for (auto val : out_size_cst.getValues()) { + cst_size.push_back(val.getSExtValue()); + } + + out_height_constant = cst_size[0]; + out_width_constant = cst_size[1]; + + if (out_height_constant < 0 || out_width_constant < 0) return failure(); + } + + int out_spatial_cst = out_height_constant < 0 || out_width_constant < 0 + ? -1 + : out_height_constant * out_width_constant; + + // Input rank should be 4. Might be able to drop this requirement entirely + // as its an input requirement. + if (!input_ty.hasRank() || input_ty.getRank() != 4) { + return failure(); + } + + int64_t batch_cst = input_ty.getShape()[0]; + int64_t channels_cst = input_ty.getShape()[3]; + + int64_t in_y_cst = input_ty.getShape()[1]; + int64_t in_x_cst = input_ty.getShape()[2]; + int64_t in_spatial_cst = + in_y_cst < 0 || in_x_cst < 0 ? -1 : in_y_cst * in_x_cst; + + // TODO(suderman): Add support for these optional parameters. + if (op.getAlignCorners() == true || op.getHalfPixelCenters() == true) { + return failure(); + } + + auto one = + ConstOp::create(rewriter, loc, GetScalarOfType(out_size_element_ty, 1)); + + // Extract the image shape. + Value input_shape = ShapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({4}, rewriter.getI64Type()), + input); + input_shape = CastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({4}, out_size_element_ty), + input_shape); + + auto scalar_dim_ty = + tensorflow::GetTypeFromTFTensorShape({}, out_size_element_ty); + auto split_image_shape = UnpackOp::create( + rewriter, loc, + TypeRange({scalar_dim_ty, scalar_dim_ty, scalar_dim_ty, scalar_dim_ty}), + input_shape); + + // Extract the separate components from the input shape. + auto batch = split_image_shape.getResult(0); + auto in_y = split_image_shape.getResult(1); + auto in_x = split_image_shape.getResult(2); + auto channels = split_image_shape.getResult(3); + + auto in_count = MulOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, out_size_element_ty), in_y, + in_x); + + // Unpack and separate the out width/height. + auto split_out_size = UnpackOp::create( + rewriter, loc, TypeRange({scalar_dim_ty, scalar_dim_ty}), out_size); + + auto out_y = split_out_size.getResult(0); + auto out_x = split_out_size.getResult(1); + + auto out_count = MulOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, out_size_element_ty), out_y, + out_x); + + // Generate what the final output shape will look like. + auto out_shape = PackOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({4}, out_size_element_ty), + ValueRange({batch, out_y, out_x, channels})); + + // Compute the indices along the vertical dimension. + auto in_y_f32 = CastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), in_y); + auto out_w_f32 = CastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), out_y); + + Value y_scale = DivOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), + in_y_f32, out_w_f32); + + Value zero_f32 = ConstOp::create( + rewriter, loc, GetScalarOfType(rewriter.getF32Type(), 0.0)); + Value one_f32 = ConstOp::create( + rewriter, loc, GetScalarOfType(rewriter.getF32Type(), 1.0)); + + Value y_range = + RangeOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_height_constant}, rewriter.getF32Type()), + zero_f32, out_w_f32, one_f32); + + y_range = MulOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_height_constant}, rewriter.getF32Type()), + y_range, y_scale); + + y_range = CastOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_height_constant}, out_size_element_ty), + y_range); + + y_range = ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({out_height_constant, 1}, + out_size_element_ty), + y_range, + PackOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({2}, out_size_element_ty), + ValueRange({out_y, one}))); + + Value y_indices = + MulOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_height_constant, 1}, out_size_element_ty), + y_range, in_x); + + // Compute the indices for the nearest neighbour lookup across the width + // dim. + auto in_x_f32 = CastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), in_x); + auto out_h_f32 = CastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), out_x); + + Value x_scale = DivOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getF32Type()), + in_x_f32, out_h_f32); + + Value x_range = + RangeOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_width_constant}, rewriter.getF32Type()), + zero_f32, out_h_f32, one_f32); + + x_range = MulOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_width_constant}, rewriter.getF32Type()), + x_range, x_scale); + + x_range = CastOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_width_constant}, out_size_element_ty), + x_range); + + Value x_indices = ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({1, out_width_constant}, + out_size_element_ty), + x_range, + PackOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({2}, out_size_element_ty), + ValueRange({one, out_x}))); + + // Generate the combined index array, reshape to be 1-D. + Value indices = AddV2Op::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {out_height_constant, out_width_constant}, out_size_element_ty), + y_indices, x_indices); + + indices = ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({out_spatial_cst}, + out_size_element_ty), + indices, + ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({1}, out_size_element_ty), + out_count, + ConstOp::create(rewriter, loc, rewriter.getI64TensorAttr({1})))); + + // Group the spatial indices and gather along that combined index. + Value input_collapsed_spatial = ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {batch_cst, in_spatial_cst, channels_cst}, input_element_ty), + input, + PackOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({3}, out_size_element_ty), + ValueRange({batch, in_count, channels}))); + + Value gathered_values = GatherV2Op::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {batch_cst, out_spatial_cst, channels_cst}, input_element_ty), + input_collapsed_spatial, indices, /*axis=*/one); + + gathered_values = + ReshapeOp::create(rewriter, loc, result_ty, gathered_values, out_shape); + + rewriter.replaceOp(op, gathered_values); + return success(); + } +}; + +struct LowerRollOp : public RewritePattern { + explicit LowerRollOp(MLIRContext *context) + : RewritePattern( + RollOp::getOperationName(), 1, context, + {ConstOp::getOperationName(), SliceOp::getOperationName(), + ConcatV2Op::getOperationName()}) {} + + LogicalResult matchAndRewrite(Operation *op, + PatternRewriter &rewriter) const override { + auto tf_roll_op = cast(op); + + auto input_ty = + mlir::dyn_cast(tf_roll_op.getInput().getType()); + if (!input_ty || !input_ty.hasStaticShape()) { + return rewriter.notifyMatchFailure( + op, "require the type of input to have static shapes"); + } + + DenseIntElementsAttr shift_attr; + Value shift = tf_roll_op.getShift(); + auto shift_ranked_attr_type = + mlir::dyn_cast(shift.getType()); + if (!shift_ranked_attr_type || + !matchPattern(shift, m_Constant(&shift_attr))) { + return failure(); + } + + DenseIntElementsAttr axis_attr; + Value axis = tf_roll_op.getAxis(); + auto axis_ranked_attr_type = + mlir::dyn_cast(axis.getType()); + if (!axis_ranked_attr_type || !matchPattern(axis, m_Constant(&axis_attr))) { + return failure(); + } + + // Combine duplicate axis and make sure they are in [0, rank(input)) range. + auto input_shape = input_ty.getShape(); + int input_rank = input_shape.size(); + SmallVector shift_map(input_rank, 0); + for (int i = 0; i < axis_attr.getNumElements(); ++i) { + int32_t axis_i = axis_attr.getValues()[i]; + if (axis_i < 0) axis_i += input_rank; + int32_t shift_i = shift_attr.getValues()[i]; + shift_map[axis_i] += shift_i; + } + + SmallVector adjusted_axis; + SmallVector adjusted_shift; + for (int i = 0; i < input_rank; ++i) { + int32_t input_dims_i = input_shape[i]; + int32_t shift_i = shift_map[i] % input_dims_i; + if (shift_i < 0) shift_i += input_dims_i; + if (shift_i == 0) continue; + adjusted_axis.push_back(i); + adjusted_shift.push_back(shift_i); + } + + // Convert rolling in each dimension to two Slice ops and one Concat op. + auto axis_type = tensorflow::GetTypeFromTFTensorShape( + {input_rank}, rewriter.getIntegerType(64)); + auto create_slice_op = [&](int32_t axis_i, int32_t begin_i, int32_t size_i, + Value input) { + SmallVector begin_values(input_rank, 0); + begin_values[axis_i] = begin_i; + auto begin_attr = DenseIntElementsAttr::get(axis_type, begin_values); + auto begin = + ConstOp::create(rewriter, op->getLoc(), axis_type, begin_attr); + + SmallVector output_shape; + output_shape.append(input_shape.begin(), input_shape.end()); + output_shape[axis_i] = size_i; + auto size_attr = DenseIntElementsAttr::get(axis_type, output_shape); + auto size = ConstOp::create(rewriter, op->getLoc(), axis_type, size_attr); + + auto slice_op_ty = tensorflow::GetTypeFromTFTensorShape( + output_shape, input_ty.getElementType()); + return SliceOp::create(rewriter, op->getLoc(), slice_op_ty, input, begin, + size); + }; + + auto result = tf_roll_op.getInput(); + auto scalar_type = + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getIntegerType(32)); + for (int i = 0; i < adjusted_axis.size(); ++i) { + int32_t axis_i = adjusted_axis[i]; + int32_t shift_i = adjusted_shift[i]; + auto slice_op_1 = create_slice_op(axis_i, input_shape[axis_i] - shift_i, + shift_i, result); + auto slice_op_2 = + create_slice_op(axis_i, 0, input_shape[axis_i] - shift_i, result); + + auto dim_attr = DenseIntElementsAttr::get(scalar_type, {axis_i}); + auto concat_dim = + ConstOp::create(rewriter, op->getLoc(), scalar_type, dim_attr); + auto concat_op = ConcatV2Op::create( + rewriter, op->getLoc(), input_ty, + ArrayRef({slice_op_1.getOutput(), slice_op_2.getOutput()}), + concat_dim); + result = concat_op.getResult(); + } + + rewriter.replaceOp(op, result); + return success(); + } +}; + +// Decomposes Softmax and LogSoftmax to primitive TF ops, using the following +// formulas: +// +// softmax = div(exp(logits), sum(exp(logits))) +// log_softmax = sub(logits, log(sum(exp(logits)))) +// +// TODO(jpienaar): Evaluate benefit of templating here. +template +class LowerSoftmaxOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + Value logits = op.getLogits(); + auto loc = op.getLoc(); + + // Note that the TensorFlow Softmax op verifies that the input rank is + // greater than or equal to one so the following sequence is valid. + auto reduce_dim = + TF::ConstOp::create(rewriter, loc, GetI64ElementsAttr({-1}, &rewriter)); + + // Exponential of input values and then their sum can be very large here. + // Division with large denominator is numerically unstable. To improve + // numerical stability, subtract each batch with their max element so that + // the maximum input value is zero. It can be shown that softmax computed + // after adding or subtracting all inputs in a batch using a common value + // gives mathematically equivalent result. + auto max_logits = + TF::MaxOp::create(rewriter, loc, logits, reduce_dim, + /*keep_dims=*/rewriter.getBoolAttr(true)); + auto shifted_logits = TF::SubOp::create(rewriter, loc, logits, max_logits); + + // Exponentiate the inputs. + Value exp = TF::ExpOp::create(rewriter, loc, shifted_logits); + + // Compute summation of the exponentials. + Value sum = TF::SumOp::create(rewriter, loc, exp, reduce_dim, + /*keep_dims=*/rewriter.getBoolAttr(true)); + + if (use_log) { + Value log = TF::LogOp::create(rewriter, loc, sum); + rewriter.replaceOpWithNewOp(op, shifted_logits, log); + } else { + rewriter.replaceOpWithNewOp(op, exp, sum); + } + return success(); + } +}; + +} // namespace + +void PopulateLoweringTFPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + // clang-format off + patterns->add< + LowerAddNOp, + LowerExp1mOp, + ConvertFakeQuantWithMinMaxVarsOp, + LowerDynamicStitchOp, + LowerDynamicStitchOp, + LowerInvertPermutationOp, + LowerLgammaOp, + LowerPackOp, + LowerBatchToSpaceND, + LowerSpaceToBatchNDOp, + LowerResizeNearestNeighbor, + LowerSparseMatMulOp, + Lower_UnaryOpsComposition, + LowerRollOp>(context); + // clang-format on + populateWithGenerated(*patterns); +} + +void PopulateTFLoweringBeforeHLOPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + // clang-format off + patterns->add< + ConvertFakeQuantWithMinMaxVarsOp, + LowerAddNOp, + LowerBatchToSpaceND, + LowerDynamicStitchOp, + LowerDynamicStitchOp, + LowerInvertPermutationOp, + LowerPackOp, + LowerResizeNearestNeighbor, + LowerSoftmaxOp, + LowerSoftmaxOp, + LowerSpaceToBatchNDOp, + LowerSparseMatMulOp, + Lower_UnaryOpsComposition, + LowerRollOp>(context); + // clang-format on + + // Populate the relevant generated patterns. + // clang-format off + patterns->add< + LowerAddOp, + LowerBiasAddGradOp, + LowerDivNoNanOp, + LowerEmptyOp, + LowerFakeQuantWithMinMaxArgs, + LowerFillOp, + LowerInv, + LowerIsNanOp, + LowerL2LossOp, + LowerMulNoNanOp, + LowerMatrixBandPartOp, + LowerPadOp, + LowerReciprocal, + LowerRintOp, + LowerRoundOpOnFloatTensor, + LowerRoundOpOnIntTensor, + LowerRsqrtGradOp, + LowerScatterNdOp, + LowerSeluOp, + LowerSeluGradOp, + LowerSizeOp, + LowerSoftmaxCrossEntropyWithLogitsOp, + LowerSparseSoftmaxCrossEntropyWithLogitsOp, + LowerSqrtGradOp, + LowerSquareOp, + LowerSquaredDifferenceOpOnRealTensors, + LowerSquaredDifferenceOpOneComplexTensors, + LowerTanhGradOp, + LowerTruncateDivOpOnIntTensors, + LowerTruncateDivOpOnFloatTensors, + LowerXdivyOp, + LowerXlog1pyOp, + LowerXlogyOp>(context); + // clang-format on +} + +void PopulateLoweringQuantizedPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + // clang-format off + patterns->add< + LowerDequantizeOp>(context); + // clang-format on +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h new file mode 100644 index 0000000000000000000000000000000000000000..ff879ddb715c55f7ec13719b3e0ec99f7985a513 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h @@ -0,0 +1,52 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_TF_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_TF_H_ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Populates TensorFlow lowering patterns to lower some of the TensorFlow +// operations that can be represented using other TensorFlow operations. +// TODO(laurenzo): For some reason, TFLite uses this pass and has exact +// requirements on what it can do. This is fragile and should be fixed (at a +// minimum, names should clearly convey scope). In the mean time, for a real +// compiler, use PopulateTFLoweringBeforeHLOPatterns. +void PopulateLoweringTFPatterns(MLIRContext *context, + RewritePatternSet *patterns); + +// Populates TensorFlow lowering patterns to lower some of the TensorFlow +// operations that can be represented by means of other TensorFlow operations. +// This pattern collection preserves those TensorFlow operations that will later +// be lowered to equivalent operations in CHLO or MHLO. This allows for +// HLO-specific lowerings. +void PopulateTFLoweringBeforeHLOPatterns(MLIRContext *context, + RewritePatternSet *patterns); + +// Populates TensorFlow lowering patterns to lower some of the TensorFlow +// operations that can be represented using other TensorFlow operations. +// Patterns are from ops with some inputs or outputs that are quantized types +// only to ops that allow non-quantized types on all inputs and outputs. +void PopulateLoweringQuantizedPatterns(MLIRContext *context, + RewritePatternSet *patterns); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_LOWER_TF_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.td new file mode 100644 index 0000000000000000000000000000000000000000..68abb32e757e22b70be46dc3e529d16ea924d8b7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.td @@ -0,0 +1,653 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/IR/OpBase.td" +include "mlir/Dialect/Func/IR/FuncOps.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" +include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td" + +class GetScalarOfFloatType : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0)," # value # ")">; + +def GetScalarInfOfType : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0), " + "std::numeric_limits::infinity())">; + +def GetScalarNanOfType : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0), " + "std::numeric_limits::quiet_NaN())">; + +class GetI64ScalarElementsAttr : + NativeCodeCall<"GetI64ElementsAttr({" # value # "}, &$_builder)">; + +class GetF32Scalar : + NativeCodeCall<"GetF32Scalar(&$_builder, " # value # ")">; + +def TrueBoolAttr : AttrConstraint($_self).getValue()">>; + +def CreateTFShapeOp : NativeCodeCall< + "TF::ShapeOp::create($_builder, $0.getLoc(), $1, $2)">; + +def IsI32 : NativeCodeCall< + "$_builder.getBoolAttr(getElementTypeOrSelf($0.getType()).isInteger(32))">; + +def CreateTFCastOpF32 : NativeCodeCall< + "CreateTFCastOpF32(&$_builder, $0.getLoc(), $1, $2)">; + +def CreateTFCastOpI32 : NativeCodeCall< + "CreateTFCastOpI32(&$_builder, $0.getLoc(), $1, $2)">; + +def CreateTensorScatterNdOp : NativeCodeCall< + "TF::ScatterNdOp::create($_builder, " + "$0.getLoc(), $0.getType(), $1, $2, $3, $4)">; + +def CreateTensorScatterUpdateOp : NativeCodeCall< + "TF::TensorScatterUpdateOp::create($_builder, " + "$0.getLoc(), $0.getType(), $0, $1, $2, $3)">; + +//===----------------------------------------------------------------------===// +// Add op patterns. +//===----------------------------------------------------------------------===// + +// We can lower AddOp to AddV2Op, except for string type. AddOp is defined for +// TF_NumberNotQuantizedOrStrTensor tensor types. TF_NumberNotQuantizedTensor +// is the same, except not containing TF_Str. + +def LowerAddOp : Pat<(TF_AddOp TF_NumberNotQuantizedTensor:$x, + TF_NumberNotQuantizedTensor:$y), + (TF_AddV2Op $x, $y)>; + +//===----------------------------------------------------------------------===// +// BiasAddGrad op patterns. +//===----------------------------------------------------------------------===// + +def GetBiasAddGradReductionIndices : NativeCodeCall< + "GetBiasAddGradReductionIndices(" + "llvm::cast($0.getType()).getRank(), $1, &$_builder)">; + +def LowerBiasAddGradOp : + Pat<(TF_BiasAddGradOp AnyRankedTensor:$out_backprop, $data_format), + (TF_SumOp $out_backprop, + (TF_ConstOp (GetBiasAddGradReductionIndices $out_backprop, + $data_format)), + /*keep_dims=*/ConstBoolAttrFalse)>; + +// Lowers SoftmaxCrossEntropyWithLogitsOp using simpler TensorFlow ops. The op +// computes loss and backprop of the loss with respect to 'features'. +// +// Softmax cross entropy loss is defined as follows: +// +// loss = Sum(-labels * Log(Exp(features) / Sum(Exp(features))) +// loss = Sum(-labels * LogSoftmax(features)) +// +// Computing gradient of the loss with respect to features gives us, +// +// backprop = (Exp(features) / Sum(Exp(features))) - labels +// backprop = Softmax(features) - labels +// +// Computation of the reduction axis for the Sum op depends on whether the +// input is a scalar or not. Restrict pattern to ranked inputs so that input to +// the Sum op is also ranked. + +// TODO(hinsu): Support scalar inputs by introducing reshape to 1D. +def NonScalarType : Type>, "Non scalar type">; + +def LowerSoftmaxCrossEntropyWithLogitsOp : Pattern< + (TF_SoftmaxCrossEntropyWithLogitsOp AnyRankedTensor:$features, + AnyRankedTensor:$labels), + [(TF_SumOp (TF_MulNoNanOp:$sum_input + (TF_LogSoftmaxOp $features), + (TF_NegOp $labels)), + (TF_ConstOp (GetI64ScalarElementsAttr<-1>)), + /*keep_dims=*/ConstBoolAttrFalse), + (TF_SubOp (TF_SoftmaxOp $features), $labels)], + [(NonScalarType $features), (NonScalarType $labels)]>; + +// Returns size of the specified dimension as scalar elements attribute of type +// $1. +// Requires $0 to be of RankedTensorType with rank greater than `dim` and the +// dimension should be known. +class GetDimSizeOfType : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($1), " + "llvm::cast($0.getType()).getDimSize(" # dim # "))">; + +// Same as the above with i32 element type. +class GetDimSizeAsI32 : NativeCodeCall< + "GetScalarOfType($_builder.getIntegerType(32), " + "llvm::cast($0.getType()).getDimSize(" # dim # "))">; + +// Sparse version of SoftmaxCrossEntropyWithLogits is lowered to dense by +// expanding the sparse labels using: +// +// labels = OneHotOp(sparse_labels, depth, 1.0, 0.0) +// +// If any of the indices are out of range, we must populate the labels with +// NaNs to follow the semantics of the op. +def LowerSparseSoftmaxCrossEntropyWithLogitsOp : Pattern< + (TF_SparseSoftmaxCrossEntropyWithLogitsOp:$src_op + AnyStaticShapeTensor:$features, $sparse_labels), + [(TF_OneHotOp:$labels $sparse_labels, + (TF_ConstOp (GetDimSizeAsI32<1> $features, $src_op__0)), + (TF_ConstOp (GetScalarOfType<1> $features)), + (TF_ConstOp (GetScalarOfType<0> $features)), + ConstantAttr), + (TF_SelectV2Op:$zero_or_nan + (TF_LogicalAndOp + (TF_LessEqualOp + (TF_ConstOp (GetScalarOfType<0> $sparse_labels)), $sparse_labels), + (TF_LessOp $sparse_labels, + (TF_ConstOp (GetDimSizeOfType<1> $features, $sparse_labels)))), + (TF_ConstOp (GetScalarOfType<0> $features)), + (TF_ConstOp (GetScalarNanOfType $labels))), + (TF_AddV2Op:$adjusted_labels $labels, + (TF_ExpandDimsOp $zero_or_nan, + (TF_ConstOp (GetI64ScalarElementsAttr<-1>)))), + (TF_SoftmaxCrossEntropyWithLogitsOp $features, $adjusted_labels)]>; + +//===----------------------------------------------------------------------===// +// Dequantize op patterns. +//===----------------------------------------------------------------------===// + +def DequantizeHalfRange : NativeCodeCall< + "DequantizeHalfRange(&$_builder, $0)">; + +// TODO(b/188530181): Generalize to more quantized input types, +// allow num_slices > 1, and allow non default arguments for $mode, +// $narrow_range, and $axis. +def LowerDequantizeOp : Pat< + (TF_DequantizeOp:$result + $input, $min_range, $max_range, + TF_AnyStrAttrOf<["MIN_COMBINED"]>:$mode, + ConstBoolAttrFalse:$narrow_range, + ConstantAttr:$axis), + (TF_CastOp + (TF_AddV2Op + (TF_MulOp + (TF_AddV2Op + (CreateTFCastOpF32 $result, $input, /*truncate=*/ConstBoolAttrFalse), + (TF_ConstOp (DequantizeHalfRange $input))), + (TF_DivOp + (TF_SubOp $max_range, $min_range), + (TF_ConstOp (GetF32Scalar<255>)))), + $min_range), + /*truncate=*/ConstBoolAttrFalse), + [(TensorOf<[TF_Qint8, TF_Quint8]> $input), + (TF_Float32Tensor $min_range), + (TF_Float32Tensor $max_range), + (TensorOf<[TF_Float32, TF_Bfloat16]> $result)]>; + +//===----------------------------------------------------------------------===// +// Difference op patterns. +//===----------------------------------------------------------------------===// + +def ComplexTensor : TensorOf<[AnyComplex]>; +def RealTensor : TensorOf<[AnySignlessInteger, AnyFloat]>; + +def LowerSquareOp : Pat<(TF_SquareOp $val), (TF_MulOp $val, $val)>; + +def LowerSquaredDifferenceOpOnRealTensors : Pat< + (TF_SquaredDifferenceOp RealTensor: $lhs, RealTensor:$rhs), + (TF_SquareOp (TF_SubOp $lhs, $rhs))>; + +def LowerSquaredDifferenceOpOneComplexTensors : Pat< + (TF_SquaredDifferenceOp ComplexTensor: $lhs, ComplexTensor:$rhs), + (TF_MulOp (TF_SubOp:$diff $lhs, $rhs), (TF_ConjOp $diff))>; + +//===----------------------------------------------------------------------===// +// DivNoNan and MulNonNan op patterns. +//===----------------------------------------------------------------------===// + +class BinaryNoNanPat + : Pat<(FromOp $l, $r), + (TF_SelectV2Op (TF_EqualOp $r, + (TF_ConstOp:$zero (GetScalarOfType<0> $r)), + /*incompatible_shape_error*/ConstBoolAttrTrue), + $zero, (ToOp $l, $r))>; + +def LowerDivNoNanOp : BinaryNoNanPat; + +def LowerMulNoNanOp : BinaryNoNanPat; + +//===----------------------------------------------------------------------===// +// TruncateDiv op patterns. +//===----------------------------------------------------------------------===// + +def LowerTruncateDivOpOnIntTensors : Pat< + (TF_TruncateDivOp TF_IntTensor:$lhs, $rhs), + (TF_DivOp $lhs, $rhs)>; + +// Note: truncation could also be implemented as sign(x) * floor(abs(x)) or +// (-1 & x) || floor(abs(x)), based on performance benchmarks. +def LowerTruncateDivOpOnFloatTensors : Pat< + (TF_TruncateDivOp TF_FloatTensor:$lhs, $rhs), + (TF_SelectV2Op + (TF_LessOp + (TF_DivOp:$div $lhs, $rhs), + (TF_ConstOp:$zero (GetScalarOfFloatType<"0.0"> $lhs)) + ), + (TF_CeilOp $div), + (TF_FloorOp $div) + )>; + +//===----------------------------------------------------------------------===// +// Fill op patterns. +//===----------------------------------------------------------------------===// + +def LowerFillOp : Pat<(TF_FillOp $dims, $value), + (TF_BroadcastToOp $value, $dims)>; + +//===----------------------------------------------------------------------===// +// Empty op patterns. +//===----------------------------------------------------------------------===// + +def LowerEmptyOp : Pat<(TF_EmptyOp:$result $dims, TrueBoolAttr:$init), + (TF_BroadcastToOp + (TF_ConstOp (GetScalarOfType<0> $result)), $dims), + [(TF_SintOrFpTensor $result)]>; + +//===----------------------------------------------------------------------===// +// Inv op patterns. +//===----------------------------------------------------------------------===// + +def LowerInv : Pat<(TF_InvOp $x), (TF_ReciprocalOp $x)>; + +//===----------------------------------------------------------------------===// +// Inf op patterns. +//===----------------------------------------------------------------------===// + +def LowerIsInfOp : Pat<(TF_IsInfOp $x), + (TF_EqualOp (TF_AbsOp:$abs $x), + (TF_ConstOp:$inf (GetScalarInfOfType $x)), + /*incompatible_shape_error*/ConstBoolAttrTrue)>; + +//===----------------------------------------------------------------------===// +// NaN op patterns. +//===----------------------------------------------------------------------===// + +def LowerIsNanOp : Pat<(TF_IsNanOp $x), + (TF_NotEqualOp $x, $x, + /*incompatible_shape_error*/ConstBoolAttrTrue)>; + +//===----------------------------------------------------------------------===// +// L2Loss op patterns. +//===----------------------------------------------------------------------===// + +def GetAllAxes : NativeCodeCall< + "GetI64ElementsAttrForSeq(" + "0, llvm::cast($0.getType()).getRank(), &$_builder)">; + +// L2Loss is lowered using the formula, +// L2Loss(input) = Sum(input * input) / 2 + +// TODO(hinsu): Support unranked tensors once b/144593778 is resolved to not +// cause result type mismatch. +def LowerL2LossOp : + Pat<(TF_L2LossOp AnyRankedTensor:$input), + (TF_DivOp + (TF_SumOp (TF_MulOp $input, $input), + (TF_ConstOp (GetAllAxes $input)), + /*keep_dims=*/ConstBoolAttrFalse), + (TF_ConstOp (GetScalarOfType<2> $input)))>; + +//===----------------------------------------------------------------------===// +// Pad op patterns. +//===----------------------------------------------------------------------===// + +def LowerPadOp : Pat< + (TF_PadOp TensorOf<[AnySignlessInteger, AnyFloat]>:$input, $paddings), + (TF_PadV2Op $input, $paddings, + (TF_ConstOp + (GetScalarOfType<0> $input) + ) + )>; + +//===----------------------------------------------------------------------===// +// Reciprocal op patterns. +//===----------------------------------------------------------------------===// + +def LowerReciprocal : Pat<(TF_ReciprocalOp $x), + (TF_DivOp (TF_ConstOp (GetScalarOfType<1> $x)), $x)>; + +//===----------------------------------------------------------------------===// +// Round op patterns. +//===----------------------------------------------------------------------===// + +// Rint is specified as RoundHalfToEven, which happens to be the same behavior +// as TF_RoundOp, so lower to TF_RoundOp. +def LowerRintOp : Pat<(TF_RintOp:$res TF_FloatTensor:$input), (TF_RoundOp $input)>; + +// Rounds on integers should just be bypassed. +def LowerRoundOpOnIntTensor : Pat< + (TF_RoundOp:$res TF_IntTensor:$input), + (TF_IdentityOp $input)>; + +// Implements TF Round on floats using basic operations. TF Round is specified +// as RoundHalfToEven to be compatible with Numpy. +def LowerRoundOpOnFloatTensor : Pat< + (TF_RoundOp:$res TF_FloatTensor:$input), + (TF_SelectV2Op + (TF_EqualOp + (TF_ConstOp:$zero (GetScalarOfFloatType<"0.0"> $input)), + (TF_SelectV2Op:$rounded + (TF_LogicalOrOp + (TF_GreaterOp + (TF_SubOp:$fraction $input, (TF_FloorOp:$round_val $input)), + (TF_ConstOp:$half (GetScalarOfFloatType<"0.5"> $input)) + ), + (TF_LogicalAndOp + // If the input is equal to 0.5 ... + (TF_EqualOp + $fraction, + $half, + /*incompatible_shape_error*/ConstBoolAttrTrue + ), + // ... check if the value is odd, so we can round up in that case. + (TF_EqualOp + (TF_SubOp + $round_val, + (TF_MulOp + (TF_ConstOp (GetScalarOfType<2> $input)), + (TF_FloorOp (TF_MulOp $half, $input)) + ) + ), + (TF_ConstOp:$one (GetScalarOfType<1> $input)), + /*incompatible_shape_error*/ConstBoolAttrTrue + ) + ) + ), + (TF_AddV2Op $round_val, $one), + $round_val + ), + /*incompatible_shape_error*/ConstBoolAttrTrue + ), + $zero, + $rounded + )>; + +//===----------------------------------------------------------------------===// +// Rsqrt op patterns. +//===----------------------------------------------------------------------===// + +// RsqrtGrad(lhs, rhs) = (lhs * lhs * lhs) * (rhs / -2) +def LowerRsqrtGradOp : Pat< + (TF_RsqrtGradOp $lhs, $rhs), + (TF_MulOp + (TF_MulOp + (TF_MulOp $lhs, $lhs), + $lhs + ), + (TF_DivOp + $rhs, + (TF_ConstOp + (GetScalarOfType<-2> $rhs) + ) + ) + )>; + +//===----------------------------------------------------------------------===// +// Size op patterns. +//===----------------------------------------------------------------------===// + +// Size(x) = Prod(Shape(x), reduction_indices=0, keep_dims=false) +def LowerSizeOp : Pat< + (TF_SizeOp:$res $arg), + (TF_ProdOp + (CreateTFShapeOp + $res, + $arg, + (IsI32 $res) + ), + /*reduction_indices=*/ + (TF_ConstOp + (GetScalarOfType<0> $res) + ), + /*keep_dims=*/ + ConstBoolAttrFalse + )>; + +//===----------------------------------------------------------------------===// +// Sqrt op patterns. +//===----------------------------------------------------------------------===// + +// SqrtGrad(y, dy) = dy * 0.5 / y +def LowerSqrtGradOp : Pat< + (TF_SqrtGradOp $y, $dy), + (TF_DivOp + (TF_MulOp $dy, (TF_ConstOp (GetScalarOfFloatType<"0.5"> $dy))), + $y + )>; + +//===----------------------------------------------------------------------===// +// TanhGrad op patterns. +//===----------------------------------------------------------------------===// + +// grad = dy * (1 - y**2) + +def LowerTanhGradOp : + Pat<(TF_TanhGradOp $y, $dy), + (TF_MulOp $dy, + (TF_SubOp (TF_ConstOp (GetScalarOfType<1> $y)), + (TF_SquareOp $y)))>; + + +//===----------------------------------------------------------------------===// +// LowerFakeQuantWithMinMaxArgs op patterns. +//===----------------------------------------------------------------------===// + +def LowerFakeQuantWithMinMaxArgs : + Pat<(TF_FakeQuantWithMinMaxArgsOp TF_FloatTensor: $input, + $min, $max, $bits, $narrow_range), + (TF_FakeQuantWithMinMaxVarsOp $input, + (TF_ConstOp $min), (TF_ConstOp $max), $bits, $narrow_range)>; + +// ZerosLike op patterns. +//===----------------------------------------------------------------------===// + + +class LowerInitializationOp + : Pat<(FromOp:$src_op + TensorOf<[AnyInteger, AnyFloat, AnyComplex]>:$input), + (TF_BroadcastToOp (TF_ConstOp (GetScalarOfType $input)), + (CreateTFShapeOp $src_op, $input, + /*use 32bit*/ConstBoolAttrFalse))>; + +def LowerZerosLikeOp : LowerInitializationOp; +def LowerOnesLikeOp : LowerInitializationOp; + +def LowerScatterNdOp : + Pat<(TF_ScatterNdOp $indices, + TensorOf<[AnyInteger, AnyFloat, AnyComplex]>: + $updates, + $shape, + $bad_indices_policy), + (TF_TensorScatterAddOp + (TF_FillOp $shape, (TF_ConstOp (GetScalarOfType<0> $updates))), + $indices, $updates, $bad_indices_policy)>; + +//===----------------------------------------------------------------------===// +// Xdivy, Xlog1p and Xlogy op patterns. +//===----------------------------------------------------------------------===// + +class BinaryXopyPat : Pat< + From, + (TF_SelectV2Op + (TF_EqualOp + $x, + (TF_ConstOp + (GetScalarOfType<0> $x) + ), + /*incompatible_shape_error*/ConstBoolAttrTrue + ), + $x, + To + )>; + +def LowerXdivyOp : BinaryXopyPat< + (TF_XdivyOp $x, $y), + (TF_DivOp $x, $y)>; + +def LowerXlog1pyOp : BinaryXopyPat< + (TF_Xlog1pyOp $x, $y), + (TF_MulOp $x, (TF_Log1pOp $y))>; + +def LowerXlogyOp : BinaryXopyPat< + (TF_XlogyOp $x, $y), + (TF_MulOp $x, (TF_LogOp $y))>; + +//===----------------------------------------------------------------------===// +// IsFinite op patterns. +//===----------------------------------------------------------------------===// + +def LowerIsFiniteOp : Pat<(TF_IsFiniteOp $x), + (TF_EqualOp + (TF_SubOp $x, $x), + (TF_ConstOp (GetScalarOfType<0> $x)), + /*incompatible_shape_error*/ConstBoolAttrTrue)>; + +//===----------------------------------------------------------------------===// +// TensorScatterUpdate op patterns. +//===----------------------------------------------------------------------===// + +def LowerTensorScatterUpdate_1 : Pat<(TF_TensorScatterUpdateOp$input, $indices, $updates, $bad_indices_policy), + (TF_CastOp + (CreateTensorScatterUpdateOp + (CreateTFCastOpI32 + $input, + $input, + /*truncate=*/ConstBoolAttrFalse), + $indices, + (CreateTFCastOpI32 + $updates, + $updates, + /*truncate=*/ConstBoolAttrFalse), + $bad_indices_policy), + /*truncate=*/ConstBoolAttrFalse), + [(TensorOf<[TF_Bool]> $input), (TensorOf<[TF_Bool]> $updates)] >; + +def LowerTensorScatterUpdate_2 : Pat<(TF_TensorScatterUpdateOp $input, $indices, $updates, $bad_indices_policy), + (TF_AddV2Op + (TF_MulOp + (TF_SubOp (TF_ConstOp (GetScalarOfType<1> $updates)), + (CreateTensorScatterNdOp $input, $indices, + (TF_OnesLikeOp $updates), + (CreateTFShapeOp + $input, + $input, + ConstBoolAttrTrue), + $bad_indices_policy)), + $input), + (CreateTensorScatterNdOp $input, $indices, $updates, + (CreateTFShapeOp + $input, + $input, + ConstBoolAttrTrue), + $bad_indices_policy)), + [(TensorOf<[TF_Int, TF_Float, TF_Complex]> $updates)]>; + +//===----------------------------------------------------------------------===// +// Selu op patterns. +//===----------------------------------------------------------------------===// + +def getScale : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0), 1.0507009873554804934193349852946)" + >; + +def getScaledAlpha : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0), 1.7580993408473768599402175208123)" + >; + +// Selu(x) = scale * x if x > 0 else scaledAlpha * (exp(x) - 1). Here +// scale is taken as a constant 1.0507009873554804934193349852946 and +// scaledAlpha is taken as a constant 1.7580993408473768599402175208123. + +def LowerSeluOp : Pat<(TF_SeluOp TF_FloatTensor:$features), + (TF_SelectV2Op + (TF_GreaterOp + $features, + (TF_ConstOp (GetScalarOfType<0> $features))), + (TF_MulOp + (TF_ConstOp (getScale $features)), + $features), + (TF_MulOp + (TF_ConstOp (getScaledAlpha $features)), + (TF_Expm1Op $features)))>; + +// SeluGradOp(gradients, outputs) = gradients * scale if outputs > 0 +// else gradients * (outputs + scaledAlpha). Here scale is taken as a +// constant 1.0507009873554804934193349852946 and scaledAlpha is taken +// as a constant 1.7580993408473768599402175208123. + +def LowerSeluGradOp : Pat<(TF_SeluGradOp AnyStaticShapeTensor:$gradients, + TF_FloatTensor:$features), + (TF_SelectV2Op + (TF_GreaterOp + $features, + (TF_ConstOp (GetScalarOfType<0> $features))), + (TF_MulOp + $gradients, + (TF_ConstOp (getScale $features))), + (TF_MulOp + $gradients, + (TF_AddV2Op + $features, + (TF_ConstOp (getScaledAlpha $features)))))>; + + + +//===----------------------------------------------------------------------===// +// Expm1 op patterns. +//===----------------------------------------------------------------------===// + +// Expm1(x) = Exp(x) - 1 +def LowerExp1mOp : Pat< + (TF_Expm1Op $x), + (TF_SubOp + (TF_ExpOp $x), + (TF_ConstOp (GetScalarOfType<1> $x)) + )>; + +//===----------------------------------------------------------------------===// +// MatrixBandPart op patterns. +//===----------------------------------------------------------------------===// + +class GetDimensionSize : NativeCodeCall< + "GetDimensionSize(&$_builder, $_loc, $0, " # idx # ", $1)">; + +// MatrixBandPart(input, num_lower, num_upper) { +// m = Shape(input)[-2] +// n = Shape(input)[-1] +// num_lower_or_m = Select(num_lower < 0, m, num_lower) +// num_upper_or_n = Select(num_upper < 0, n, num_upper) +// offset = ExpandDims(Range(0, m), -1) - Range(0, n) +// indicator = LogicalAnd(offset <= num_lower_or_m, offset >= -num_upper_or_n) +// return Select(indicator, input, ZerosLike(Input)) +// } +def LowerMatrixBandPartOp : Pattern< + (TF_MatrixBandPartOp $input, $num_lower, $num_upper), + [(TF_ConstOp:$zero (GetScalarOfType<0> $num_lower)), + (TF_ConstOp:$one (GetScalarOfType<1> $num_lower)), + (TF_ConstOp:$neg_one (GetScalarOfType<-1> $num_lower)), + (GetDimensionSize<-2>:$m $input, (IsI32 $num_lower)), + (GetDimensionSize<-1>:$n $input, (IsI32 $num_upper)), + (TF_SelectV2Op:$num_lower_or_m (TF_LessOp $num_lower, $zero), + $m, $num_lower), + (TF_SelectV2Op:$num_upper_or_n (TF_LessOp $num_upper, $zero), + $n, $num_upper), + (TF_RangeOp:$range_m $zero, $m, $one), + (TF_RangeOp:$range_n $zero, $n, $one), + (TF_SubOp:$offset (TF_ExpandDimsOp $range_m, $neg_one), $range_n), + (TF_LogicalAndOp:$indicator (TF_LessEqualOp $offset, $num_lower_or_m), + (TF_GreaterEqualOp $offset, + (TF_NegOp $num_upper_or_n))), + (TF_SelectV2Op $indicator, $input, (TF_ZerosLikeOp $input))]>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..98532a54ee8fd79538b19af5d2f4b8407a372631 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf_test_pass.cc @@ -0,0 +1,56 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" + +namespace mlir { +namespace tf_test { +namespace { + +#define GEN_PASS_DEF_TESTTENSORFLOWLOWERTFPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +// Lowers some of the TensorFlow operations that can be represented using other +// TensorFlow operations. +struct LowerTF : public impl::TestTensorFlowLowerTFPassBase { + void runOnOperation() override { + // Add lowering patterns to the list. + RewritePatternSet patterns(&getContext()); + if (default_patterns_) { + mlir::TF::PopulateLoweringTFPatterns(&getContext(), &patterns); + } + if (pre_hlo_patterns_) { + mlir::TF::PopulateTFLoweringBeforeHLOPatterns(&getContext(), &patterns); + } + + (void)applyPatternsGreedily(getOperation(), std::move(patterns)); + } +}; + +} // namespace + +std::unique_ptr> CreateTestTFLowerTFPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_variable_ops_to_ml_program.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_variable_ops_to_ml_program.cc new file mode 100644 index 0000000000000000000000000000000000000000..8f75a63d7e11295ddf4341a0bf6c666836efae4f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/lower_variable_ops_to_ml_program.cc @@ -0,0 +1,209 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/Support/Casting.h" +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" // from @llvm-project +#include "mlir/Analysis/DataFlowFramework.h" // from @llvm-project +#include "mlir/Dialect/MLProgram/IR/MLProgram.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_dataflow.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +std::string GetVariableName(Operation* op) { + if (auto handle = dyn_cast(op)) { + std::string container = handle.getContainer().str(); + std::string shared_name = handle.getSharedName().str(); + if (container.empty()) { + return absl::StrCat("vars.", shared_name); + } else { + return absl::StrCat("vars.", container, ".", shared_name); + } + } else if (auto global = dyn_cast(op)) { + return absl::StrCat("vars.", global.getSymName().str()); + } + return ""; +} + +Operation* GetHandleSource(Operation* op, DataFlowSolver& solver) { + Value resource; + if (auto read = llvm::dyn_cast(op)) { + resource = read.getResource(); + } else if (auto write = llvm::dyn_cast(op)) { + resource = write.getResource(); + } + const TF::ResourceDataflowState* state = + solver.lookupState(resource); + if (!state) { + return nullptr; + } + auto ops = state->getValue().ops; + if (ops.size() != 1) { + return nullptr; + } + Operation* source = *ops.begin(); + return source; +} + +Attribute GetInitialValue(Operation* source) { + if (auto global = dyn_cast(source)) { + if (global.getValue()) { + return *global.getValue(); + } + } + return nullptr; +} + +Type GetGlobalType(Operation* source) { + if (auto var_handle_op = dyn_cast(source)) { + // Resources are represented as tensor>>, so + // unwrap until we get to the inner tensor<...>. + auto tensor = + llvm::dyn_cast(var_handle_op.getResource().getType()); + if (!tensor) return nullptr; + TF::ResourceType resource = + llvm::dyn_cast(tensor.getElementType()); + if (!resource || resource.getSubtypes().size() != 1) return nullptr; + return resource.getSubtypes().front(); + } else if (auto global_tensor_op = + dyn_cast(source)) { + return global_tensor_op.getType(); + } + // Likely can't actually happen, assuming tf_saved_model.semantics checks + // already ran. + return nullptr; +} + +ml_program::GlobalOp CreateGlobalOpFromOp(Operation* source, OpBuilder& builder, + SymbolTable& symbol_table) { + Type type = GetGlobalType(source); + std::string name = GetVariableName(source); + if (auto existing = symbol_table.lookup(name)) { + // This might be of a different type, but we'll do a Cast later. + return existing; + } + + Attribute initial_value = GetInitialValue(source); + if (!initial_value) { + initial_value = builder.getZeroAttr(type); + if (!initial_value) { + initial_value = builder.getArrayAttr({}); + } + } + + if (!type) return nullptr; + + auto globalOp = ml_program::GlobalOp::create( + builder, builder.getBlock()->getParentOp()->getLoc(), name, type, false, + initial_value, nullptr); + symbol_table.insert(globalOp); + + return globalOp; +} + +} // namespace + +#define GEN_PASS_DEF_LOWERVARIABLEOPSTOMLPROGRAMPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +struct LowerVariableOpsToMlProgramPass + : public impl::LowerVariableOpsToMlProgramPassBase< + LowerVariableOpsToMlProgramPass> { + explicit LowerVariableOpsToMlProgramPass() = default; + void getDependentDialects(DialectRegistry& registry) const override { + registry.insert(); + } + void runOnOperation() override { + auto module = getOperation(); + + DataFlowSolver solver; + solver.load(); + solver.load(); + TF::LoadResourceDataflowAnalysis(solver); + if (failed(solver.initializeAndRun(module))) return signalPassFailure(); + + SymbolTable symbol_table(module); + + OpBuilder globalBuilder(module.getBodyRegion()); + + module.walk([&](TF::ReadVariableOp op) { + Operation* source = GetHandleSource(op, solver); + if (!source) return; + ml_program::GlobalOp globalOp = + CreateGlobalOpFromOp(source, globalBuilder, symbol_table); + if (!globalOp) return; + OpBuilder builder(op); + Operation* load = mlir::ml_program::GlobalLoadOp::create( + builder, op.getLoc(), globalOp.getType(), + SymbolRefAttr::get(op->getContext(), globalOp.getSymName())); + if (globalOp.getType() != op.getValue().getType()) { + load = TF::CastOp::create(builder, op.getLoc(), op.getValue().getType(), + load->getResult(0)); + } + op.getResult().replaceAllUsesWith(load->getResult(0)); + op.erase(); + }); + + module.walk([&](TF::AssignVariableOp op) { + Operation* source = GetHandleSource(op, solver); + if (!source) return; + ml_program::GlobalOp globalOp = + CreateGlobalOpFromOp(source, globalBuilder, symbol_table); + if (!globalOp) return; + symbol_table.insert(globalOp); + OpBuilder builder(op); + globalOp.setIsMutableAttr(builder.getUnitAttr()); + Value value_to_store = op.getValue(); + if (globalOp.getType() != op.getValue().getType()) { + value_to_store = TF::CastOp::create(builder, op.getLoc(), + globalOp.getType(), value_to_store); + } + mlir::ml_program::GlobalStoreOp::create( + builder, op.getLoc(), + SymbolRefAttr::get(op->getContext(), globalOp.getSymName()), + value_to_store); + op.erase(); + }); + } +}; + +std::unique_ptr> +CreateLowerVariableOpsToMlProgramPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.cc new file mode 100644 index 0000000000000000000000000000000000000000..58990d0dbcadc4665f5910078199dc18578838d6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.cc @@ -0,0 +1,100 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h" + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Threading.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/session_utils.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/device_base.h" +#include "tensorflow/core/framework/rendezvous.h" +#include "tensorflow/core/framework/resource_var.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Returns true if the variable 'var_handle_op' is initialized in 'session'. +bool IsVariableInitialized(mlir::TF::VarHandleOp var_handle_op, + llvm::StringRef device_name, + const tensorflow::DeviceMgr* mgr, + tensorflow::Session* session) { + auto* var_ptr = GetVariableFromSession(var_handle_op, device_name, mgr); + if (!var_ptr) return false; + auto* tensor = var_ptr->tensor(); + bool is_initialized = tensor && tensor->IsInitialized(); + var_ptr->Unref(); + return is_initialized; +} + +LogicalResult MarkInitializedVariablesInFunction(func::FuncOp function, + tensorflow::Session* session) { + if (!session || !llvm::hasSingleElement(function)) return success(); + Block& block = function.front(); + + const tensorflow::DeviceMgr* mgr = nullptr; + auto status = session->LocalDeviceManager(&mgr); + if (!status.ok()) + return function->emitError( + absl::StrCat("failed to fetch device manager: ", status.message())); + + // Fetch all varHandleOp in the function. + llvm::SmallVector var_ops; + for (auto var_handle_op : block.getOps()) + var_ops.emplace_back(var_handle_op); + + // Get resources from Session. + auto resource_tensors_or = GetResourcesFromSession(var_ops, session); + if (!resource_tensors_or.ok()) + return function->emitError(resource_tensors_or.status().message().data()); + + MLIRContext* context = function.getContext(); + for (auto var_and_tensor : llvm::zip(var_ops, resource_tensors_or.value())) { + auto& var_op = std::get<0>(var_and_tensor); + auto& resource_tensor = std::get<1>(var_and_tensor); + bool is_variable_initialized = false; + if (resource_tensor.dtype() != tensorflow::DT_RESOURCE) { + is_variable_initialized = true; + } else { + auto handle = resource_tensor.scalar()(); + is_variable_initialized = + IsVariableInitialized(var_op, handle.device(), mgr, session); + } + var_op->setAttr("_is_initialized", + BoolAttr::get(context, is_variable_initialized)); + } + return success(); +} + +LogicalResult MarkInitializedVariablesInFunction(ModuleOp module, + tensorflow::Session* session) { + auto functions_range = module.getOps(); + return mlir::failableParallelForEach( + module.getContext(), functions_range.begin(), functions_range.end(), + [&](func::FuncOp function) { + return MarkInitializedVariablesInFunction(function, session); + }); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h new file mode 100644 index 0000000000000000000000000000000000000000..34b59435a5428cbc6ccdd947063897482ce49b76 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h @@ -0,0 +1,41 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MARK_INITIALIZED_VARIABLES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MARK_INITIALIZED_VARIABLES_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { +// Marks all variables in 'function' whether they are initialized +// in 'session' or not by setting an attribute named 'is_initialized' +// on each variable op with value true/false based on variable is initialized +// in the session or not. +// If 'session' is NULL the function is no-op. +// Returns failure in case fetching variables from session failed, success +// otherwise. +LogicalResult MarkInitializedVariablesInFunction(func::FuncOp function, + tensorflow::Session* session); +// Apply `MarkInitializedVariablesInFunction` to every non-empty function in the +// module. +LogicalResult MarkInitializedVariablesInFunction(ModuleOp module, + tensorflow::Session* session); + +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MARK_INITIALIZED_VARIABLES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..48912dac860403130da03225049f53d0756d35da --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables_test_pass.cc @@ -0,0 +1,83 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/fake_session.h" + +namespace mlir { +namespace { + +// This pass is only available in the tf-opt binary for testing. +class MarkInitializedVariablesTestPass + : public PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(MarkInitializedVariablesTestPass) + + StringRef getArgument() const final { + return "tf-saved-model-mark-initialized-variables-test"; + } + + StringRef getDescription() const final { + return "Mark variables as initialized or not."; + } + + void runOnOperation() override { + TF::test_util::FakeSession session; + if (failed(mlir::tf_saved_model::MarkInitializedVariablesInFunction( + getOperation(), &session))) + return signalPassFailure(); + } +}; + +// This pass is only available in the tf-opt binary for testing. +class MarkInitializedVariablesInvalidSessionTestPass + : public PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + MarkInitializedVariablesInvalidSessionTestPass) + + StringRef getArgument() const final { + return "tf-saved-model-mark-initialized-variables-invalid-session-test"; + } + + StringRef getDescription() const final { + return "Mark variables as initialized or not, but with invalid session."; + } + + void runOnOperation() override { + // Pass an invalid session argument, which is a nullptr. + if (failed(mlir::tf_saved_model::MarkInitializedVariablesInFunction( + getOperation(), /*session=*/nullptr))) + return signalPassFailure(); + } +}; + +} // namespace + +namespace tf_saved_model { + +static PassRegistration + mark_initialized_variables_test_pass; + +static PassRegistration + mark_initialized_variables_invalid_session_test_pass; + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_input_output_aliases.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_input_output_aliases.cc new file mode 100644 index 0000000000000000000000000000000000000000..41883218d2c0084cc5e3d058d65a3388a1e46646 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mark_input_output_aliases.cc @@ -0,0 +1,202 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +#define DEBUG_TYPE "tf-device-mark-input-output-aliases" + +namespace mlir { +namespace TFDevice { + +#define GEN_PASS_DEF_MARKINPUTOUTPUTALIASESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +namespace { +struct MarkInputOutputAliasesPass + : public impl::MarkInputOutputAliasesPassBase { + void runOnOperation() override; +}; + +constexpr char kAliasingAttr[] = "tf.aliasing_output"; +constexpr int kUnassigned = -1; + +struct AliasInfo { + AliasInfo() : input_index(kUnassigned), output_index(kUnassigned) {} + int input_index; + int output_index; +}; + +// Idenitfy tf_device.cluster_func input-output alias pairs. +// This is currently conservative, primarily handling the following base case: +// ``` +// %value = tf.ReadVariableOp(%resource_var) +// %output:N = tf_device.cluster_func(..., /*input index = a*/ %value, ...) +// tf.AssignVariableOp(%resource_var, %output#b) // write output #b to resource +// ``` +// where `%value` and `%output#b` have only one use. (a, b) would be added as +// input-output alias pair for `%resource_var`. +// +// This pass also handles the case where instead +// ``` +// %output:N = tf_device.parallel_execute(...) +// ``` +// with a `cluster_func` nested within the `parallel_execute`. This occurs when +// there are outside-compiled ops are in the middle of an XLA cluster, which are +// extracted to be run on CPU host by the `ExtractOutsideCompilation` pass and +// wrapped in `parallel_execute` alongside the device `cluster_func`. + +TF::AssignVariableOp FindAssignVariableOp(OpResult result) { + auto use = result.use_begin(); + + Operation* device_return = use->getOwner(); + if (!device_return) return nullptr; + Operation* parent = device_return->getParentOp(); + + if (parent) { + if (llvm::isa(parent)) { + if (auto execute = + llvm::dyn_cast_or_null(parent)) { + int operand_idx = use->getOperandNumber(); + auto execute_results = execute.GetRegionOutputs( + device_return->getParentRegion()->getRegionNumber()); + if (operand_idx >= execute_results.size()) return nullptr; + + auto result_from_use = execute_results[operand_idx]; + if (!result_from_use.hasOneUse()) return nullptr; + + device_return = result_from_use.use_begin()->getOwner(); + if (!device_return) return nullptr; + } + } else { + LLVM_DEBUG(llvm::dbgs() + << "No supported special parent found, using " + "tf_device.cluster_func outputs directly to search for " + "consuming AssignVariableOp.\n"); + } + } + return llvm::dyn_cast_or_null(device_return); +} + +// TODO(b/184420848): Explore relaxing these constraints. +LogicalResult BuildAliasingInfo( + tf_device::ClusterFuncOp cluster_func, + llvm::DenseMap& resource_alias_info_map) { + for (auto result : cluster_func.getResults()) { + if (!result.hasOneUse()) continue; + + TF::AssignVariableOp assign_op = FindAssignVariableOp(result); + if (!assign_op) continue; + + AliasInfo& alias_info = resource_alias_info_map[assign_op.getResource()]; + // TODO(b/184420848): We may not need to skip aliasing for entire function + // in case of multiple assigns. + if (alias_info.output_index != kUnassigned) { + LLVM_DEBUG(llvm::dbgs() + << "Skip adding aliasing information because of multiple " + "assigns to " + "the same resource from tf_device.cluster_func outputs. " + "This can " + "lead to poor memory management on device.\n"); + + return failure(); + } + alias_info.output_index = result.getResultNumber(); + } + + for (auto& operand : cluster_func->getOpOperands()) { + auto read_op = llvm::dyn_cast_or_null( + operand.get().getDefiningOp()); + if (!read_op) continue; + if (!read_op->hasOneUse()) continue; + auto it = resource_alias_info_map.find(read_op.getResource()); + if (it == resource_alias_info_map.end()) continue; + AliasInfo& alias_info = it->getSecond(); + // TODO(b/184420848): We may not need to skip aliasing for entire function + // in case of multiple reads from same resource variable. + if (alias_info.input_index != kUnassigned) { + LLVM_DEBUG( + llvm::dbgs() + << "Skip adding aliasing information because of multiple reads of " + "the same resource in tf_device.cluster_func inputs. This can " + "lead to poor memory management on device.\n"); + return failure(); + } + + alias_info.input_index = operand.getOperandNumber(); + } + return success(); +} + +void AddAliasingAttributeToDeviceFunc( + func::FuncOp device_func, + llvm::DenseMap& resource_alias_info_map) { + OpBuilder builder(device_func.getContext()); + for (const auto& resource_alias_entry : resource_alias_info_map) { + const AliasInfo& alias_info = resource_alias_entry.second; + if (alias_info.input_index == kUnassigned || + alias_info.output_index == kUnassigned) + continue; + auto aliasing_attr = device_func.getArgAttrOfType( + alias_info.input_index, kAliasingAttr); + + // Set only if aliasing attribute does not exist. + if (!aliasing_attr) { + device_func.setArgAttr( + alias_info.input_index, kAliasingAttr, + builder.getI64IntegerAttr(alias_info.output_index)); + continue; + } + // If aliasing attribute already exists, it must match the new value. + assert(aliasing_attr.getInt() == alias_info.output_index); + } +} + +void MarkInputOutputAliasesPass::runOnOperation() { + SmallVector cluster_funcs; + ModuleOp module = getOperation(); + module.walk([&](tf_device::ClusterFuncOp cluster_func) { + // Map resource values to pair of input-output indices. + llvm::DenseMap resource_alias_info_map; + if (failed(BuildAliasingInfo(cluster_func, resource_alias_info_map)) || + resource_alias_info_map.empty()) { + return; + } + + FlatSymbolRefAttr func_attr = cluster_func.getFuncAttr(); + func::FuncOp device_func = + module.lookupSymbol(func_attr.getValue()); + AddAliasingAttributeToDeviceFunc(device_func, resource_alias_info_map); + }); +} + +} // namespace + +std::unique_ptr> CreateMarkInputOutputAliasesPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/materialize_mlir_passthrough_op.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/materialize_mlir_passthrough_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..56dab8e2064b4fdbc040feab6ae798ac704b97a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/materialize_mlir_passthrough_op.cc @@ -0,0 +1,112 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +#define DEBUG_TYPE "tf-materialize-passthrough-op" + +namespace mlir { +namespace { + +#define GEN_PASS_DEF_MATERIALIZEPASSTHROUGHOP +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class MaterializePassthroughOpPass + : public impl::MaterializePassthroughOpBase { + public: + void runOnOperation() override; +}; + +void MaterializePassthroughOpPass::runOnOperation() { + getOperation().walk([](TF::MlirPassthroughOp op) { + std::string module_string(op.getMlirModule()); + // Parse the module. + auto nested_module = + parseSourceString(module_string, op.getContext()); + if (!nested_module) { + op->emitError() << "could not parse attached MLIR module"; + return; + } + func::FuncOp main = + dyn_cast(nested_module->lookupSymbol("main")); + if (!main) { + op->emitError() << "MLIR Opaque Op expects a main() entry point\n"; + return; + } + if (main.getNumArguments() != op->getNumOperands()) { + op->emitError() << "mismatch between MLIR Opaque Op number of operands (" + << op->getNumOperands() + << ") and main() entry point in the module (" + << main.getNumArguments() << " args)\n"; + return; + } + if (main.getFunctionType().getNumResults() != op->getNumResults()) { + op->emitError() << "mismatch between MLIR Opaque Op number of results (" + << op->getNumResults() + << ") and main() entry point in the module (" + << main.getFunctionType().getNumResults() + << " results)\n"; + return; + } + Region &body = main.getBody(); + if (!llvm::hasSingleElement(body)) { + op->emitError() << "MLIR Opaque Op expects a main() entry point with a " + "single block\n"; + return; + } + Block &block = body.front(); + for (const auto &arg_mapping : + llvm::zip(block.getArguments(), op->getOperands())) { + std::get<0>(arg_mapping).replaceAllUsesWith(std::get<1>(arg_mapping)); + } + op->getBlock()->getOperations().splice(op->getIterator(), + block.getOperations(), block.begin(), + std::prev(block.end())); + Operation &return_op = block.front(); + for (auto ret_mapping : + llvm::zip(op->getResults(), return_op.getOperands())) { + std::get<0>(ret_mapping).replaceAllUsesWith(std::get<1>(ret_mapping)); + } + op->erase(); + }); +} + +} // namespace + +namespace TF { +std::unique_ptr> +CreateMaterializePassthroughOpPass() { + return std::make_unique(); +} +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/merge_control_flow.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/merge_control_flow.cc new file mode 100644 index 0000000000000000000000000000000000000000..28ecbfc8613a3ecee4445b756bf44b9549e58812 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/merge_control_flow.cc @@ -0,0 +1,662 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +// For the 2d vector type aliases defined below, the first dimension represents +// the class of the IfRegion group and the second dimension represents the +// segments of the IfRegion group. +// For example, if we want to merge the following six IfRegions +// which share the same if_cond (regionA) +// ````````````` +// IfRegionA(1) +// IfRegionA(2) +// IfRegionA(3) +// IfRegionA(4) +// IfRegionA(5) +// IfRegionA(6) +// `````````````` +// After the analysis, we consider IfRegionA(1), IfRegionA(2) and IfRegionA(3) +// can be merged, IfRegionA(4) is standalone, IfRegionA(5) and IfRegionA(6) +// can be merged. Then the defined 2D vector is +// [[IfRegionA(1), IfRegionA(2), IfRegionA(3)], +// [IfRegionA(4)], +// [IfRegionA(5), IfRegionA(6)]] +using RegionVec2D = llvm::SmallVector, 8>; +using OperationVec2D = llvm::SmallVector, 8>; +using MapToRegionVec2D = llvm::SmallDenseMap; +using MapToOperationVec2D = llvm::SmallDenseMap; +using IfOpIterConst = + llvm::SmallVectorTemplateCommon::const_iterator; + +#define GEN_PASS_DEF_MERGECONTROLFLOWPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct MergeControlFlowPass + : public impl::MergeControlFlowPassBase { + void runOnOperation() override; +}; + +// Gets the IfRegion op and all of ops in the then and else branches. +llvm::SmallSetVector GetAllOpsFromIf(TF::IfRegionOp if_op) { + llvm::SmallSetVector all_ops; + all_ops.insert(if_op); + for (Operation& op : if_op.getThenBranch().front()) { + all_ops.insert(&op); + } + for (Operation& op : if_op.getElseBranch().front()) { + all_ops.insert(&op); + } + return all_ops; +} + +// Returns whether it is safe to merge `second_if` IfRegion into `first_if` +// IfRegion. `second if` must come after `first_if`. +// Note that `downstream_if_ops` means the ops in IfRegions except`first_if`. +bool SafeToMerge(TF::IfRegionOp first_if, TF::IfRegionOp second_if, + llvm::SmallSetVector& downstream_if_ops, + const TF::SideEffectAnalysis::Info& side_effect_analysis) { + // IfRegion ops must be in the same block. + if (second_if.getOperation()->getBlock() != + first_if.getOperation()->getBlock()) { + return false; + } + assert(first_if.getOperation()->isBeforeInBlock(second_if.getOperation())); + + llvm::SmallSetVector destination_ops = + GetAllOpsFromIf(first_if); + + // If there is an intermediate data or side effect dependency between the + // ops in first_if and the ops in second_if, it's not safe to merge + // them. + std::vector dependencies; + for (auto* user : first_if.getOperation()->getUsers()) { + if (!downstream_if_ops.contains(user)) { + dependencies.push_back(user); + } + } + for (auto* successor : + side_effect_analysis.DirectControlSuccessors(first_if.getOperation())) { + if (!downstream_if_ops.contains(successor)) { + dependencies.push_back(successor); + } + } + for (Operation& op : first_if.getThenBranch().front()) { + for (auto* successor : side_effect_analysis.DirectControlSuccessors(&op)) { + if (!downstream_if_ops.contains(successor) && + !destination_ops.contains(successor)) + dependencies.push_back(successor); + } + } + for (Operation& op : first_if.getElseBranch().front()) { + for (auto* successor : side_effect_analysis.DirectControlSuccessors(&op)) { + if (!downstream_if_ops.contains(successor) && + !destination_ops.contains(successor)) + dependencies.push_back(successor); + } + } + + bool safe_to_merge = true; + + llvm::SmallPtrSet visited; + while (!dependencies.empty()) { + Operation* dependency = dependencies.back(); + dependencies.pop_back(); + if (visited.count(dependency)) continue; + visited.insert(dependency); + for (auto* user : dependency->getUsers()) { + if (downstream_if_ops.contains(user)) { + safe_to_merge = false; + break; + } else { + dependencies.push_back(user); + } + } + for (auto* successor : + side_effect_analysis.DirectControlSuccessors(dependency)) { + if (downstream_if_ops.contains(successor)) { + safe_to_merge = false; + break; + } else { + dependencies.push_back(successor); + } + } + // If the op is nested, then also consider the users and successors of the + // parent op. + if (dependency->getBlock() != first_if.getOperation()->getBlock()) + dependencies.push_back(dependency->getParentOp()); + if (!safe_to_merge) break; + } + return safe_to_merge; +} + +// Move the body excluding the terminators of else and then regions from +// 'second_if' to 'first_if'. +void MoveBranches(TF::IfRegionOp first_if, TF::IfRegionOp second_if) { + Block& first_if_then_block = first_if.getThenBranch().front(); + auto& second_if_then_body = second_if.getThenBranch().front().getOperations(); + first_if_then_block.getOperations().splice( + first_if_then_block.without_terminator().end(), second_if_then_body, + second_if_then_body.begin(), std::prev(second_if_then_body.end())); + + Block& first_if_else_block = first_if.getElseBranch().front(); + auto& second_if_else_body = second_if.getElseBranch().front().getOperations(); + first_if_else_block.getOperations().splice( + first_if_else_block.without_terminator().end(), second_if_else_body, + second_if_else_body.begin(), std::prev(second_if_else_body.end())); +} + +// Check if the `last` IfRegion can be added to the segment of +// IfRegion start with `first` IfRegion. +bool CanAddToIfSegment( + IfOpIterConst first, IfOpIterConst last, + const llvm::SmallVector& if_ops, + const std::unique_ptr& side_effect_analysis) { + if (last == if_ops.end()) { + return false; + } + // downstream_if_ops contain ops in those IfRegions between first IfRegion + // and last IfRegion plus the ops in the last IfRegion. + llvm::SmallSetVector downstream_if_ops; + + TF::IfRegionOp second_if_op = *last; + + for (auto iter = std::prev(last); std::next(iter) != first; iter--) { + TF::IfRegionOp first_if_op = *iter; + func::FuncOp func = first_if_op->getParentOfType(); + const TF::SideEffectAnalysis::Info& analysis = + side_effect_analysis->GetAnalysisForFunc(func); + auto all_ops = GetAllOpsFromIf(*(std::next(iter))); + downstream_if_ops.insert(all_ops.begin(), all_ops.end()); + if (!SafeToMerge(first_if_op, second_if_op, downstream_if_ops, analysis)) { + return false; + } + } + return true; +} + +// Return the iterator of the IfRegion Op. This is the last IfRegion +// in the segment. +// For example, we have the following sequence of IfRegions +// ````` +// 1 2 3 4 5 +// IfRegionA, IfRegionA, IfRegionA, IfRegionA, IfRegionA +// ````` +// The first three IfRegionA are in one group and the last two are in another +// group. Then when we call FindLastIfInSegment for the first segment, it +// will return iterator of the 3rd IfRegionA. +// In the same way, when we call it for the second segment, it will return +// iterator of the 5th IfRegionA. +IfOpIterConst FindLastIfInSegment( + IfOpIterConst first_if, + const llvm::SmallVector& if_ops, + const std::unique_ptr& side_effect_analysis) { + IfOpIterConst last_if = first_if; + for (; CanAddToIfSegment(first_if, last_if, if_ops, side_effect_analysis); + last_if = std::next(last_if)) { + } + return std::prev(last_if); +} + +// Returns a set of ops to be moved after merged IfRegion between two IfRegions. +absl::flat_hash_set GetMoveOpsBetweenTwoIfRegions( + Operation* result_op, Operation* after_op, + llvm::SmallSetVector middle_if_ops, + const std::unique_ptr& side_effect_analysis) { + Block* block = after_op->getBlock(); + std::queue queue; + absl::flat_hash_set visited; + absl::flat_hash_set moved_ops; + + func::FuncOp func = result_op->getParentOfType(); + const TF::SideEffectAnalysis::Info& analysis = + side_effect_analysis->GetAnalysisForFunc(func); + + // Enqueue dependencies of source_op into queue. + auto enqueue_deps = [&](Operation* source_op) { + for (Operation* user : source_op->getUsers()) { + if (!visited.count(user) && !middle_if_ops.count(user)) { + visited.insert(user); + queue.push(user); + } + } + source_op->walk([&](Operation* walked_op) { + for (Operation* successor : analysis.DirectControlSuccessors(walked_op)) { + if (!source_op->isProperAncestor(successor)) { + if (!visited.count(successor) && !middle_if_ops.count(successor)) { + visited.insert(successor); + queue.push(successor); + } + } + } + }); + }; + enqueue_deps(result_op); + + while (!queue.empty()) { + auto* op = queue.front(); + queue.pop(); + while (op->getBlock() != block) op = op->getParentOp(); + if (op->isBeforeInBlock(after_op)) { + moved_ops.insert(op); + enqueue_deps(op); + } + } + return moved_ops; +} + +// Returns a vector that contains the ops to be moved after merged IfRegion. +// `sub_if_group` refers to a segment of IfRegions. +// The returned vector preserves op order. +llvm::SmallVector GetMoveOpList( + llvm::SmallVector& sub_if_group, + const std::unique_ptr& side_effect_analysis) { + absl::flat_hash_set all_moved_ops; + Operation* last_if_op = sub_if_group.back().getOperation(); + llvm::SmallSetVector middle_if_ops; + + // reversely calculate the all ops need to be moved because in this way, + // ops in the middle IfRegions can be easily obtained by simply adding to the + // current set. + for (auto it = std::prev(std::prev(sub_if_group.end())); + std::next(it) != sub_if_group.begin(); --it) { + auto op_list = GetMoveOpsBetweenTwoIfRegions( + it->getOperation(), last_if_op, middle_if_ops, side_effect_analysis); + all_moved_ops.insert(op_list.begin(), op_list.end()); + auto first_if_ops = GetAllOpsFromIf(*it); + middle_if_ops.insert(first_if_ops.begin(), first_if_ops.end()); + } + + llvm::SmallVector moved_ops_ordered; + moved_ops_ordered.reserve(all_moved_ops.size()); + for (Operation& op : *last_if_op->getBlock()) { + if (all_moved_ops.count(&op)) { + moved_ops_ordered.push_back(&op); + } + } + + return moved_ops_ordered; +} + +// Generate the segments for each IfRegion groups. Each element in the segments +// are supposed to can be merged into one new IfRegion.`if_cond` refers to the +// if condition of the segment of IfRegions. `if_ops` refers to the segment of +// IfRegions. `merged_groups` refers to all segments of IfRegions. +// `moved_ops_groups` refers to the ops need to be moved after new merged +// IfRegions associated with each segment of IfRegions. +void GenerateSegmentsPerIfGroups( + const mlir::Value& if_cond, + const llvm::SmallVector& if_ops, + const std::unique_ptr& side_effect_analysis, + MapToRegionVec2D& merged_groups, MapToOperationVec2D& moved_ops_groups) { + auto it_merged = merged_groups.try_emplace(if_cond); + auto it_moved = moved_ops_groups.try_emplace(if_cond); + llvm::SmallVector sub_merged_groups; + auto begin_if_op_iter = if_ops.begin(); + + while (begin_if_op_iter != if_ops.end()) { + auto current_last_if_op_iter = + FindLastIfInSegment(begin_if_op_iter, if_ops, side_effect_analysis); + assert(current_last_if_op_iter != if_ops.end()); + llvm::SmallVector sub_if_group; + for (auto it = begin_if_op_iter; it != std::next(current_last_if_op_iter); + ++it) { + sub_if_group.push_back(*it); + } + it_merged.first->getSecond().push_back(sub_if_group); + it_moved.first->getSecond().push_back( + GetMoveOpList(sub_if_group, side_effect_analysis)); + begin_if_op_iter = std::next(current_last_if_op_iter); + } +} + +// Checks whether a return index should be kept for `current_if_op` by checking +// for results in `if_op_segment`. +llvm::SmallVector GetReturnIndicesToKeep( + TF::IfRegionOp current_if_op, + const llvm::SmallVector& if_op_segment) { + llvm::SmallVector return_indices_to_keep; + auto is_op_inside_IfRegions = [&](Operation* op) { + for (auto& if_op : if_op_segment) { + if (if_op == current_if_op) { + continue; + } + if (if_op->isProperAncestor(op)) { + return true; + } + } + return false; + }; + for (const auto& index_and_value : + llvm::enumerate(current_if_op.getResults())) { + if (!llvm::all_of(index_and_value.value().getUsers(), + is_op_inside_IfRegions)) { + return_indices_to_keep.push_back(index_and_value.index()); + } + } + return return_indices_to_keep; +} + +// Return a vector of the return indices. +llvm::SmallVector> GetReturnIndicesVec( + const llvm::SmallVector& if_op_segment) { + llvm::SmallVector> return_indices_vec; + for (auto it = if_op_segment.begin(); it != if_op_segment.end(); ++it) { + llvm::SmallVector indices_to_keep_vec = + GetReturnIndicesToKeep(*it, if_op_segment); + return_indices_vec.push_back(indices_to_keep_vec); + } + return return_indices_vec; +} + +// Replace the internal usage in each pair of IfRegions from top to bottom for +// both then branch and else branch. +void ReplaceInternalUsage(llvm::SmallVector& if_op_segment) { + for (auto it = if_op_segment.begin(); it != if_op_segment.end(); ++it) { + for (auto it2 = std::next(it); it2 != if_op_segment.end(); ++it2) { + for (OpResult result : it->getResults()) { + replaceAllUsesInRegionWith( + result, + it->getThenBranch().front().getTerminator()->getOperand( + result.getResultNumber()), + it2->getThenBranch()); + replaceAllUsesInRegionWith( + result, + it->getElseBranch().front().getTerminator()->getOperand( + result.getResultNumber()), + it2->getElseBranch()); + } + } + } +} + +// Move ops in the `moved_ops_ordered` after `last_op`. +void MoveOpsAfter(Operation* last_op, + llvm::SmallVector& moved_ops_ordered) { + auto block = last_op->getBlock(); + absl::flat_hash_set all_moved_ops(moved_ops_ordered.begin(), + moved_ops_ordered.end()); + moved_ops_ordered.clear(); + for (Operation& op : *block) { + // There are no mutations in the loop. So each call of `isBeforeInBlock` + // is O(1). + if (all_moved_ops.count(&op) && op.isBeforeInBlock(last_op)) { + moved_ops_ordered.push_back(&op); + } + } + // Move ops in order. + for (Operation* op : moved_ops_ordered) { + op->moveAfter(last_op); + last_op = op; + } +} + +// Replace all external usage for each IfRegion in the segment of IfRegions. +// `if_op_segment` refers to the segment of IfRegions, `new_if_op` refers to the +// new merged IfRegion, `return_indices` refers to the indices to be kept in new +// merged IfRegion. +void ReplaceExternalUsage( + llvm::SmallVector& if_op_segment, + TF::IfRegionOp new_if_op, + llvm::SmallVector>& return_indices) { + int new_return_index = 0; + for (const auto& index_and_value : llvm::enumerate(if_op_segment)) { + auto old_if_op = index_and_value.value(); + for (int i : return_indices[index_and_value.index()]) { + old_if_op.getResult(i).replaceAllUsesWith( + new_if_op.getResult(new_return_index++)); + } + } +} + +// Update the moved op list to remove old IfRegions from the list and add new +// merged IfRegions. `old_to_new_IfRegions_map` refers to a map from old +// IfRegion to new merged IfRegion. `moved_ops_list` refers to the list of ops +// to be moved after new merged IfRegion. +void UpdateMovedOpList( + llvm::SmallDenseMap& old_to_new_IfRegion_map, + llvm::SmallVector& moved_ops_list) { + llvm::SmallDenseSet new_if_ops; + bool need_add_new_if_op = false; + for (auto iter = moved_ops_list.begin(); iter != moved_ops_list.end(); + iter++) { + if (old_to_new_IfRegion_map.count(*iter)) { + need_add_new_if_op = true; + auto new_if_op = old_to_new_IfRegion_map[*iter]; + new_if_ops.insert(new_if_op); + moved_ops_list.erase(iter--); + } + } + if (need_add_new_if_op) { + for (auto& new_if_op : new_if_ops) { + moved_ops_list.push_back(new_if_op.getOperation()); + } + } +} + +// Create the Yield ops for both branches with merged results. +// `builder` is the OpBuilder. +// `if_op_segment` refers to the segment of IfRegions to be merged. +// `return_indices` refers to the return indices to be kept in merged IfRegion +// `new_if_op` refers to the created new IfRegion +void CreateYieldOps( + OpBuilder& builder, llvm::SmallVector& if_op_segment, + llvm::SmallVector>& return_indices, + TF::IfRegionOp new_if_op, TF::IfRegionOp first_if) { + llvm::SmallVector merged_then_yield_values; + for (const auto& index_and_value : llvm::enumerate(if_op_segment)) { + auto if_op = index_and_value.value(); + for (auto i : return_indices[index_and_value.index()]) { + merged_then_yield_values.push_back( + if_op.getThenBranch().front().getTerminator()->getOperand(i)); + } + } + builder.setInsertionPointToEnd(&new_if_op.getThenBranch().front()); + TF::YieldOp::create( + builder, first_if.getThenBranch().front().getTerminator()->getLoc(), + /*operands=*/merged_then_yield_values); + + llvm::SmallVector merged_else_yield_values; + for (const auto& index_and_value : llvm::enumerate(if_op_segment)) { + auto if_op = index_and_value.value(); + for (auto i : return_indices[index_and_value.index()]) { + merged_else_yield_values.push_back( + if_op.getElseBranch().front().getTerminator()->getOperand(i)); + } + } + builder.setInsertionPointToEnd(&new_if_op.getElseBranch().front()); + TF::YieldOp::create( + builder, first_if.getElseBranch().front().getTerminator()->getLoc(), + /*operands=*/merged_else_yield_values); +} + +// Merge the IfRegions in each segment. In the meantime, the old IfRegions in +// the segment will be added to `regions_to_remove`. They will be erased in the +// end. +// `if_op_segment` refers to segments of IfRegions. `moved_op_list` refers to +// the ops to be moved after new merged IfRegion. `regions_to_remove` refers to +// the regions to be removed from the `moved_ops_list`. +// `old_to_new_IfRegion_map` refers to a map from old IfRegion to new merged +// IfRegion. +void MergeIfPerSegment( + llvm::SmallVector& if_op_segment, + llvm::SmallVector& moved_ops_list, + llvm::SmallSetVector& regions_to_remove, + llvm::SmallDenseMap& old_to_new_IfRegion_map) { + TF::IfRegionOp first_if = if_op_segment[0]; + llvm::SmallVector merged_return_types; + llvm::SmallVector sources_if_ops( + std::next(if_op_segment.begin()), if_op_segment.end()); + + // Create new IfRegion's merged results. + auto return_indices = GetReturnIndicesVec(if_op_segment); + for (const auto& index_and_value : llvm::enumerate(return_indices)) { + TF::IfRegionOp if_op = if_op_segment[index_and_value.index()]; + for (auto i : index_and_value.value()) { + merged_return_types.push_back(if_op.getResult(i).getType()); + } + } + + // Create new IfRegion for merged all IfRegions in if_op_segmemt. + OpBuilder builder(first_if); + builder.setInsertionPoint(if_op_segment.back().getOperation()); + + auto new_if_op = TF::IfRegionOp::create( + builder, first_if.getLoc(), merged_return_types, first_if.getCond(), + llvm::all_of(if_op_segment, + [&](TF::IfRegionOp op) { return op.getIsStateless(); }), + first_if.get_thenFuncNameAttr(), first_if.get_elseFuncNameAttr()); + new_if_op.getThenBranch().push_back(new Block); + new_if_op.getElseBranch().push_back(new Block); + + // Replace internal usages of merged if ops. + ReplaceInternalUsage(if_op_segment); + + // Replace external usages of merged if ops. + ReplaceExternalUsage(if_op_segment, new_if_op, return_indices); + + // Move ops after the new merged If region. + MoveOpsAfter(new_if_op.getOperation(), moved_ops_list); + + // Create the Yield ops for both branches with merged results. + CreateYieldOps(builder, if_op_segment, return_indices, new_if_op, first_if); + + for (auto& old_if_op : if_op_segment) { + MoveBranches(/*first_if=*/new_if_op, /*second_if=*/old_if_op); + } + + for (auto& old_if_op : if_op_segment) { + old_to_new_IfRegion_map[old_if_op.getOperation()] = new_if_op; + regions_to_remove.insert(old_if_op); + } +} + +// Merge IfRegions for each IfRegion group. Each IfRegion group contains +// several segments of IfRegions and each segment of IfRegions can be merged +// into one IfRegion. +// `if_cond` refers to the if condition of the segments of IfRegions. +// `planned_merged_groups` refers to the groups of IfRegions to be merged +// `moved_ops_groups` refers to the ops need to be moved after new merged +// IfRegions associated with each segment of IfRegions. +// `regions_to_remove` refers to the regions to be removed +// `old_to_new_IfRegion_map` refers to a map from old IfRegion to new merged +// IfRegion. +void MergeIfPerIfGroups( + const Value& if_cond, MapToRegionVec2D& planned_merged_groups, + MapToOperationVec2D& moved_ops_groups, + llvm::SmallSetVector& regions_to_remove, + llvm::SmallDenseMap& old_to_new_IfRegion_map) { + OperationVec2D& moved_ops_group = moved_ops_groups[if_cond]; + RegionVec2D& segments = planned_merged_groups[if_cond]; + + for (auto i = 0; i < segments.size(); ++i) { + if (segments[i].size() >= 2) { + UpdateMovedOpList(old_to_new_IfRegion_map, moved_ops_group[i]); + MergeIfPerSegment(segments[i], moved_ops_group[i], regions_to_remove, + old_to_new_IfRegion_map); + } + } +} + +// Groups IfRegions by common predicate and attemps to merge them. +void OptimizeIfRegions(Block* block, ModuleOp module) { + // Do side effect analysis only one time in the beginning + auto side_effect_analysis = std::make_unique(module); + + // Determine IfRegions with the same predicate. + llvm::SmallDenseMap, 8> + grouped_if_ops; + llvm::SmallVector if_cond_order; + block->walk([&](TF::IfRegionOp if_op) { + auto it = grouped_if_ops.try_emplace(if_op.getCond()); + if (it.second) { + if_cond_order.push_back(if_op.getCond()); + } + it.first->getSecond().push_back(if_op); + }); + + MapToRegionVec2D planned_merged_groups; + MapToOperationVec2D moved_ops_groups; + llvm::SmallSetVector regions_to_remove; + llvm::SmallDenseMap old_to_new_IfRegion_map; + + // For each if group, determine the segments of each if groups + // that can be merged and their related ops to be moved after + // the new generated IfRegions + // We cache the infomation into two maps: + // planned_merged_groups and moved_ops_groups + for (const auto& if_cond : if_cond_order) { + GenerateSegmentsPerIfGroups(if_cond, grouped_if_ops[if_cond], + side_effect_analysis, planned_merged_groups, + moved_ops_groups); + } + + // Merge IfRegions for each IfRegion groups. + for (const auto& if_cond : if_cond_order) { + MergeIfPerIfGroups(if_cond, planned_merged_groups, moved_ops_groups, + regions_to_remove, old_to_new_IfRegion_map); + } + + // Remove all old IfRegions that already been merged. + for (auto old_if_region : regions_to_remove) { + old_if_region.erase(); + } +} + +void MergeControlFlowPass::runOnOperation() { + ModuleOp module = getOperation(); + auto result = module.walk([&](tf_device::ClusterOp cluster) { + OptimizeIfRegions(&cluster.GetBody(), module); + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return signalPassFailure(); +} + +} // namespace + +std::unique_ptr> CreateMergeControlFlowPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.cc new file mode 100644 index 0000000000000000000000000000000000000000..9901949da52767046ab013a7a6529239b091912a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.cc @@ -0,0 +1,81 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.h" + +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/Twine.h" +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/bridge.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "xla/mlir_hlo/mhlo/transforms/passes.h" + +namespace tensorflow { + +void PopulateLowerToMlProgramAndHloPipeline(mlir::OpPassManager& pm) { + tensorflow::tf2xla::internal::AddNonReplicatedBridgeClusteringPipelinePasses( + pm); + + // Remove unused global tensors, or make then immutable if possible. + pm.addPass(mlir::tf_saved_model::CreateOptimizeGlobalTensorsPass()); + + pm.addPass( + mlir::tf_saved_model::CreateConvertSessionInitializerToFunctionPass()); + pm.addNestedPass( + mlir::TFDevice::CreateDecomposeResourceOpsPass()); + pm.addPass(mlir::TF::CreateNameAnonymousIteratorsPass()); + + // This will add regions to IfOp/WhileOp (turning them into IfRegionOp + // and WhileRegionOp), but be aware that those regions will still contain + // calls. + pm.addPass(mlir::TF::CreateTFFunctionalControlFlowToRegions()); + + pm.addPass(mlir::tf_saved_model::CreateLowerVariableOpsToMlProgramPass()); + pm.addPass(mlir::tf_saved_model::CreateLowerGlobalsToMlProgramPass()); + pm.addPass(mlir::TF::CreateLocalizeVarHandlesPass()); + pm.addPass(mlir::tf_saved_model::CreateAddFunctionsForExportedNamesPass()); + pm.addPass(mlir::tf_saved_model::CreateStripSavedModuleMetadataPass()); + + pm.addPass(mlir::TF::CreateRemoveUnusedArgumentsPass()); + pm.addNestedPass( + mlir::TF::CreateRemoveUnusedWhileResultsPass()); + + pm.addPass(mlir::createInlinerPass()); + pm.addPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::createSymbolDCEPass()); + + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + + llvm::StringRef tf2xla_fallback_device_type = "XLA_CPU_JIT"; + pm.addPass(mlir::mhlo::createLegalizeTFPass( + /*legalize_chlo=*/true, tf2xla_fallback_device_type, + /*prefer_tf2xla=*/false)); + + pm.addPass(mlir::TF::CreateStripTfAttributesPass()); + + pm.addPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::createInlinerPass()); + pm.addPass(mlir::createSymbolDCEPass()); + pm.addPass(mlir::createCanonicalizerPass()); + + pm.addPass(mlir::TF::CreateOrderByDialectPass()); + + pm.addPass(mlir::mhlo::createHloLegalizeToStablehloPass()); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.h new file mode 100644 index 0000000000000000000000000000000000000000..5945276ffbdb64eb38e4634ccf2fe88a96d21dbc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.h @@ -0,0 +1,32 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MLPROGRAM_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MLPROGRAM_H_ + +#include "llvm/ADT/STLFunctionalExtras.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project + +namespace tensorflow { + +void PopulateLowerToMlProgramAndHloPipeline(mlir::OpPassManager& pm); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_MLPROGRAM_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/move_tpu_compile_to_front.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/move_tpu_compile_to_front.cc new file mode 100644 index 0000000000000000000000000000000000000000..3b17f1cd4face305ee2abba3c87eaa9430571c38 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/move_tpu_compile_to_front.cc @@ -0,0 +1,99 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_remaining_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_MOVETPUCOMPILETOFRONTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct MoveTpuCompileToFrontPass + : public impl::MoveTpuCompileToFrontPassBase { + void runOnOperation() override; +}; + +void MarkCompilationOps(Operation* func) { + func->walk([&](Operation* op) { + if (llvm::isa(op)) { + op->setAttr("_is_compilation", UnitAttr::get(func->getContext())); + op = op->getParentOp(); + while (op && op != func) { + op->setAttr("_wraps_compilation", UnitAttr::get(func->getContext())); + op = op->getParentOp(); + } + } + }); +} + +void UnmarkCompilationOps(Operation* func) { + func->walk([&](Operation* op) { + while (op && op != func) { + op->removeAttr("_is_compilation"); + op->removeAttr("_wraps_compilation"); + op = op->getParentOp(); + } + }); +} + +int OutsideCompilationOrdering(Operation* predecessor, Operation* op) { + // Actual compilations go first. + if (op->hasAttr("_is_compilation")) return 2; + // Followed by nested ops that contain compilations. + if (op->hasAttr("_wraps_compilation")) return 1; + // Followed by everything else. + return 0; +} + +void MoveTpuCompileToFrontPass::runOnOperation() { + MarkCompilationOps(getOperation()); + getOperation().walk([](Operation* op) { + for (Region& region : op->getRegions()) { + for (Block& block : region.getBlocks()) { + if (block.empty()) continue; + auto ops = SortBlockTopologically(block, OutsideCompilationOrdering); + // Replace the block with the reordered block. + for (Operation* o : ops) { + o->remove(); + block.push_back(o); + } + } + } + }); + UnmarkCompilationOps(getOperation()); +} + +} // namespace + +std::unique_ptr> CreateMoveTpuCompileToFrontPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/name_anonymous_iterators.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/name_anonymous_iterators.cc new file mode 100644 index 0000000000000000000000000000000000000000..eb0fcc6c1ce5ad484620dc8877b437f20851f79e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/name_anonymous_iterators.cc @@ -0,0 +1,78 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/strings/str_format.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_NAMEANONYMOUSITERATORSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct NameAnonymousIteratorsPass + : public impl::NameAnonymousIteratorsPassBase { + void runOnOperation() override; +}; + +template +int replace(OP op, int count) { + OpBuilder builder(op); + std::string name = absl::StrFormat("_iterator%d", count++); + + auto new_op = TF::IteratorOp::create( + builder, op->getLoc(), op->getResultTypes()[0], name, /*container=*/"", + op.getOutputTypes(), op.getOutputShapes()); + op->getResults()[0].replaceAllUsesWith(new_op->getResults()[0]); + if (op->use_empty()) op->erase(); + return count; +} + +void NameAnonymousIteratorsPass::runOnOperation() { + int count = 1; + getOperation().walk( + [&](TF::AnonymousIteratorOp op) { count = replace(op, count); }); + getOperation().walk( + [&](TF::AnonymousIteratorV2Op op) { count = replace(op, count); }); + getOperation().walk( + [&](TF::AnonymousIteratorV3Op op) { count = replace(op, count); }); +} + +} // namespace + +std::unique_ptr> CreateNameAnonymousIteratorsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.cc new file mode 100644 index 0000000000000000000000000000000000000000..fd94d1c42434df8e09fcc6a742564b4150d45edc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.cc @@ -0,0 +1,203 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h" +#include "tensorflow/compiler/mlir/utils/validators.h" // IWYU pragma: keep + +namespace mlir { +namespace TF { +namespace { + +#include "tensorflow/compiler/mlir/tensorflow/transforms/generated_optimize.inc" + +// Returns a TF Constant tensor with the passed in values. +TF::ConstOp GetI64ConstantTensor(PatternRewriter& rewriter, + ArrayRef values, Location location) { + auto cst_attr = rewriter.getI64TensorAttr(values); + return TF::ConstOp::create(rewriter, location, cst_attr.getType(), cst_attr); +} + +// Rewrites broadcast->reshape to a reshape->broadcast that reduces +// the rank of the input and output of the broadcast. +class SimplifyBroadcastReshape : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BroadcastToOp op, + PatternRewriter& rewriter) const override { + // Only rewrite if the Broadcast has only one consumer. + if (!op.getOutput().hasOneUse()) return failure(); + + Operation* user = *op.getOutput().getUsers().begin(); + + auto reshape_op = llvm::dyn_cast_or_null(user); + if (!reshape_op) return failure(); + + auto reshape_type = + mlir::cast(reshape_op.getOutput().getType()); + + if (!reshape_type.hasStaticShape()) return failure(); + ArrayRef reshape_shape = reshape_type.getShape(); + + auto input_type = mlir::cast(op.getInput().getType()); + auto output_type = mlir::cast(op.getOutput().getType()); + + if (!input_type.hasRank() || !output_type.hasRank()) return failure(); + + // The pattern attempts to reduce the rank of the input to BroadcastTo. + // Thus, we fail to match if the consuming reshape rank is larger. + ArrayRef input_shape = input_type.getShape(); + if (reshape_shape.size() > input_shape.size()) return failure(); + + // Extend the input shape with leading 1s to match the broadcast shape. + ArrayRef broadcast_shape = output_type.getShape(); + SmallVector input_shape_extended; + input_shape_extended.append(broadcast_shape.size() - input_shape.size(), 1); + input_shape_extended.append(input_shape.begin(), input_shape.end()); + + // Collect non-unit dims and corresponding dim in the input shape. + SmallVector input_carryover_dims; + SmallVector non_unit_dims; + + for (int i = 0; i < input_shape_extended.size(); i++) { + int64_t dim = broadcast_shape[i]; + if (dim != 1) { + non_unit_dims.push_back(dim); + input_carryover_dims.push_back(input_shape_extended[i]); + } + } + + // If the reshape rank is less than the number of non-unit dimensions + // of the broadcast, then the reshape collapses non-unit dimensions. + // TODO(rahulsp) : Handle this case with more careful checks. + if (reshape_shape.size() < non_unit_dims.size()) return failure(); + + SmallVector old_reshape_non_unit_dims; + SmallVector new_reshape_dims; + int new_reshape_dim_idx = 0; + for (int64_t dim : reshape_shape) { + int new_reshape_dim = 1; + if (dim != 1) { + old_reshape_non_unit_dims.push_back(dim); + if (new_reshape_dim_idx < input_carryover_dims.size()) { + new_reshape_dim = input_carryover_dims[new_reshape_dim_idx]; + new_reshape_dim_idx++; + } + } + new_reshape_dims.push_back(new_reshape_dim); + } + + if (non_unit_dims != old_reshape_non_unit_dims) return failure(); + + if (failed(VerifyShapeOfReshapeOp(new_reshape_dims))) return failure(); + + Type el_ty = getElementTypeOrSelf(op.getType()); + TF::ConstOp new_reshape_shape = GetI64ConstantTensor( + rewriter, ArrayRef(new_reshape_dims), op.getLoc()); + auto new_reshape_type = RankedTensorType::get(new_reshape_dims, el_ty); + ReshapeOp new_reshape = + ReshapeOp::create(rewriter, new_reshape_shape.getLoc(), + new_reshape_type, op.getInput(), new_reshape_shape); + TF::ConstOp new_broadcast_shape = + GetI64ConstantTensor(rewriter, reshape_shape, op.getLoc()); + rewriter.replaceOpWithNewOp( + reshape_op, reshape_op.getOutput().getType(), new_reshape, + new_broadcast_shape); + return success(); + } +}; + +#define GEN_PASS_DEF_TENSORFLOWOPTIMIZEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Canonicalize operations in functions. +struct TensorFlowOptimizePass + : public impl::TensorFlowOptimizePassBase { + LogicalResult initialize(MLIRContext* context) override { + RewritePatternSet pattern_list(context); + populateWithGenerated(pattern_list); + pattern_list.add(context); + patterns = std::move(pattern_list); + return success(); + } + + void runOnOperation() override { + auto func = getOperation(); + if (failed(applyPatternsGreedily(func, patterns))) signalPassFailure(); + } + + FrozenRewritePatternSet patterns; +}; + +} // namespace + +void CreateTFStandardPipeline(OpPassManager& pm, + const StandardPipelineOptions& options) { + OpPassManager& func_pm = pm.nest(); + + // First operates on the executor dialect: + // - remove dead islands. + // - fuse islands as much as possible. + // - materialize the eventual "pass-through" ops by inlining their content. + func_pm.addPass( + tf_executor::CreateTFExecutorGraphPruningPass(options.ops_to_preserve)); + func_pm.addPass(tf_executor::CreateTFExecutorIslandCoarseningPass()); + func_pm.addPass(CreateMaterializePassthroughOpPass()); + if (options.form_clusters) pm.addPass(TFDevice::CreateClusterFormationPass()); + + // Hopefully there is a single island left, or there wasn't any to begin with. + // We now run the optimizer which operates mostly inside islands. + func_pm.addPass(createCanonicalizerPass()); + pm.addPass(CreateTFShapeInferencePass( + {}, options.enable_stablehlo_shape_propagation)); + if (options.enable_inliner) { + pm.addPass(createInlinerPass()); + } + pm.addPass(createSymbolDCEPass()); + pm.addNestedPass(CreateTFOptimizePass()); + pm.addNestedPass(createCSEPass()); +} + +std::unique_ptr> CreateTFOptimizePass() { + return std::make_unique(); +} + +void RegisterTFOptimizePassPipeline() { + // Registers a pipeline builder function for the default + // canonicalize/optimizer. + static mlir::PassPipelineRegistration pipeline( + "tf-standard-pipeline", + "Run all the passes involved in transforming/optimizing the graph after " + "importing into MLIR, without any target specialization.", + CreateTFStandardPipeline); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.td new file mode 100644 index 0000000000000000000000000000000000000000..2e81814e66118799d8c82233f119066eb377c511 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize.td @@ -0,0 +1,113 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/IR/OpBase.td" +include "mlir/IR/PatternBase.td" +include "mlir/Dialect/Arith/IR/ArithOps.td" +include "mlir/Dialect/Func/IR/FuncOps.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" + +def IsDataFormatNHWC : ConstantAttr; + +// Get the last dimension size as a 1-d single element attr. +def GetLastDimSizeAsI32 : NativeCodeCall< + "DenseElementsAttr::get(RankedTensorType::get({1}, $_builder.getIntegerType(32)), " + "static_cast(llvm::cast($0.getType()).getDimSize( " + " llvm::cast($0.getType()).getRank() - 1)))">; + +// Check whether the tensor is ranked and whether its last dim is static. +def IsRankedShapeLastDimStatic : Constraint($0.getType())">, + CPred<"!llvm::cast($0.getType()).isDynamicDim( " + " llvm::cast($0.getType()).getRank() - 1)">]>>; + +def IsNotComplexType : Constraint($0.getType())">, + CPred<"!llvm::isa(llvm::cast($0.getType()).getElementType())"> +]>>; + +// Only fuse multiplier if all dimensions other than the channel dimension +// are equal to 1. +def CanFuseMulAndConv2D : + Constraint>; + +def F32ElementsAttr : ElementsAttrBase< + CPred<"llvm::cast($_self).getShapedType().getElementType().isF32()">, "float constant tensor">; +def DefinedByConv2D : Constraint($0.getDefiningOp())">>; +// Checks if the value has only one user. +def HasOneUse : Constraint>; + +// If we see a Conv2D op followed by Mul, then multiply the filter +// with the value in Mul. +def FuseMulAndConv2D : + Pat<(TF_MulOp:$mul (TF_Conv2DOp:$conv $input, + (Arith_ConstantOp:$filter F32ElementsAttr:$filter_value), + $strides, $use_cudnn, $padding, $explicit_padding, + IsDataFormatNHWC:$data_format, $dilations), + (Arith_ConstantOp:$multiplier F32ElementsAttr:$mul_value)), +// TODO(karimnosseir): Add check for $conv is of rank 4. + (TF_Conv2DOp $input, + (TF_MulOp (Arith_ConstantOp $filter_value, (location $filter)), + (Arith_ConstantOp $mul_value, (location $multiplier)), + (location $mul)), + $strides, $use_cudnn, $padding, $explicit_padding, $data_format, + $dilations, (location $conv)), + [(CanFuseMulAndConv2D $filter_value, $mul_value), (HasOneUse $conv)]>; + +// This rule does the following pattern match and rewrite: +// +// input bias input value bias value +// \ / => \ / \ / +// BiasAdd value Mul Mul +// \ / \ / +// Mul AddV2 +// This is to enable the FuseMulAndConv2D pattern. +// Here, root of the result is AddV2 instead of BiasAdd because the value may +// not have rank one and therefore the second operand may not have rank one +// that is required by the BiasAdd. BiasAdd with 'NHWC' data format equivalent +// to AddV2 op. +def PassthroughMulAndBiasAdd : + Pat<(TF_MulOp + (TF_BiasAddOp:$output $input, + (Arith_ConstantOp F32ElementsAttr:$bias), IsDataFormatNHWC:$format), + (Arith_ConstantOp F32ElementsAttr:$value)), + (TF_AddV2Op + (TF_MulOp $input, (Arith_ConstantOp $value)), + (TF_MulOp (Arith_ConstantOp $bias), (Arith_ConstantOp $value))), + [(DefinedByConv2D $input), (HasOneUse $output)]>; + + +// This rule does the following pattern match and rewrite: +// +// input bias input value bias value +// \ / => \ / \ / +// AddV2 value Mul Mul +// \ / \ / +// Mul AddV2 +// This is to enable the FuseMulAndConv2D pattern. +def PassthroughMulAndAddV2 : + Pat<(TF_MulOp + (TF_AddV2Op:$output $input, (Arith_ConstantOp F32ElementsAttr:$bias)), + (Arith_ConstantOp F32ElementsAttr:$value)), + (TF_AddV2Op + (TF_MulOp $input, (Arith_ConstantOp $value)), + (TF_MulOp (Arith_ConstantOp $bias), (Arith_ConstantOp $value))), + [(DefinedByConv2D $input), (HasOneUse $output)]>; + +// input -> cast -> FFT => input -> RFFT +def ConvertCastComplexFFTToRFFT: Pat< + (TF_FFTOp (TF_CastOp $input, ConstBoolAttrFalse)), + (TF_RFFTOp $input, + (Arith_ConstantOp (GetLastDimSizeAsI32 $input))), + [(IsRankedShapeLastDimStatic $input), (IsNotComplexType $input)]>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize_global_tensors.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize_global_tensors.cc new file mode 100644 index 0000000000000000000000000000000000000000..e381f002648c152ee2573616cda2eeb627f936b7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/optimize_global_tensors.cc @@ -0,0 +1,183 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass optimizes tf_saved_model.global_tensor ops. + +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_value_typed_analyzer.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +#define GEN_PASS_DEF_OPTIMIZEGLOBALTENSORSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" +struct OptimizeGlobalTensorsPass + : public impl::OptimizeGlobalTensorsPassBase { + void runOnOperation() override; +}; + +// A global tensor is bound to arguments of multiple funcs. +// This struct tracks which funcs (and which argument to that func) the global +// tensor is bound to. +struct GlobalTensorUse { + mutable func::FuncOp func; + size_t arg_index; +}; + +using GlobalTensorUsesMap = + std::map>; + +bool IsImmutable(GlobalTensorOp global_tensor, + ArrayRef global_tensor_uses, + const TF::ResourceAnalyzer& resource_analyzer) { + // Global tensor is already known to be immutable. + if (!global_tensor.getIsMutable()) { + return false; + } + // An exported global tensor that is not already known to be immutable might + // be externally mutated. + if (IsExported(global_tensor)) { + return false; + } + + // A global tensor is immutable if the resource analyzer deems it so. + for (auto& global_tensor_use : global_tensor_uses) { + auto arg = global_tensor_use.func.getArgument(global_tensor_use.arg_index); + if (resource_analyzer.IsPotentiallyWritten(arg)) { + return false; + } + } + return true; +} + +GlobalTensorUsesMap CreateGlobalTensorUsesMap(ModuleOp module) { + GlobalTensorUsesMap global_tensor_uses; + + SymbolTable symbol_table(module); + for (auto func : module.getOps()) { + for (size_t i = 0, e = func.getNumArguments(); i < e; i++) { + auto sym = + func.getArgAttrOfType(i, "tf_saved_model.bound_input"); + if (!sym) { + continue; + } + auto global_tensor = symbol_table.lookup( + mlir::cast(sym).getValue()); + if (!global_tensor) { + continue; + } + global_tensor_uses[global_tensor].push_back({func, i}); + } + } + + return global_tensor_uses; +} + +// Removes `is_mutable` attribute from tf_saved_model.global_tensor ops where we +// can prove it is safe to do so. +void MarkGlobalTensorsImmutable( + ModuleOp module, const GlobalTensorUsesMap& global_tensor_uses_map, + const TF::ResourceAnalyzer& resource_analyzer) { + for (const auto& kv : global_tensor_uses_map) { + auto global_tensor = kv.first; + const auto& global_tensor_uses = kv.second; + if (IsImmutable(global_tensor, global_tensor_uses, resource_analyzer)) { + global_tensor->removeAttr("is_mutable"); + } + } +} + +void EraseUnusedGlobalTensors(ModuleOp module, + const GlobalTensorUsesMap& global_tensor_uses) { + for (auto global_tensor : + llvm::make_early_inc_range(module.getOps())) { + // If the tensor is exported, then it is used. + if (IsExported(global_tensor)) { + continue; + } + // If the tensor is bound to an argument, then it is used. + if (global_tensor_uses.find(global_tensor) != global_tensor_uses.end()) { + continue; + } + // Erase it. + global_tensor.erase(); + } +} + +LogicalResult EraseUnusedBoundInputs(ModuleOp module) { + for (auto func : module.getOps()) { + llvm::BitVector args_to_erase(func.getNumArguments()); + for (int i = 0, e = func.getNumArguments(); i < e; i++) { + if (func.getArgAttr(i, "tf_saved_model.bound_input") && + func.getArgument(i).use_empty()) { + args_to_erase.set(i); + } + } + + if (failed(func.eraseArguments(args_to_erase))) { + return failure(); + } + } + return success(); +} + +void OptimizeGlobalTensorsPass::runOnOperation() { + auto module = getOperation(); + if (!tf_saved_model::HasTfSavedModelSemantics(module)) { + return; + } + + if (failed(EraseUnusedBoundInputs(module))) { + return signalPassFailure(); + } + + TF::ResourceAnalyzer resource_analyzer(module); + + GlobalTensorUsesMap global_tensor_uses = CreateGlobalTensorUsesMap(module); + + MarkGlobalTensorsImmutable(module, global_tensor_uses, resource_analyzer); + + EraseUnusedGlobalTensors(module, global_tensor_uses); +} + +} // namespace + +std::unique_ptr> CreateOptimizeGlobalTensorsPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.cc new file mode 100644 index 0000000000000000000000000000000000000000..e1510d6760012ed1acdaa2d4a12f55ce650b74d1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.cc @@ -0,0 +1,96 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h" + +namespace mlir { +namespace TF { +namespace { + +std::vector groupOperationsByDialect(Block& block); + +#define GEN_PASS_DEF_ORDERBYDIALECTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Reorder operations so that consecutive ops stay in the same dialect, as far +// as possible. This is to optimize the op order for the group-by-dialect pass, +// which factors consecutive same-dialect ops into functions. +class OrderByDialectPass + : public impl::OrderByDialectPassBase { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OrderByDialectPass) + void runOnOperation() override; +}; + +int DialectOrdering(Operation* predecessor, Operation* op) { + return predecessor && predecessor->getName().getDialectNamespace() == + op->getName().getDialectNamespace(); +} + +void OrderByDialectPass::runOnOperation() { + ModuleOp module = getOperation(); + for (func::FuncOp func : module.getOps()) { + std::vector> side_effect_data; + const detail::SideEffectAnalysisInfo* info = nullptr; + auto extra_dependencies = + [&](Operation* op, + bool incoming) -> llvm::SmallVector const& { + return incoming ? info->DirectControlPredecessors(op) + : info->DirectControlSuccessors(op); + }; + // Some tests have recursive calls and other shenanigans, so allow + // them to skip side effect analysis. + if (!func->hasAttr("ignore_side_effects_for_testing")) { + info = + &getAnalysis().GetAnalysisForFunc(func); + } + func->walk([&](Operation* function) { + for (Region& region : function->getRegions()) { + for (Block& block : region.getBlocks()) { + if (block.empty()) continue; + auto ops = SortBlockTopologically( + block, DialectOrdering, + info ? extra_dependencies : no_extra_dependencies); + // Replace the block with the reordered block. + for (Operation* op : ops) { + op->remove(); + block.push_back(op); + } + } + } + }); + } +} + +} // namespace + +std::unique_ptr CreateOrderByDialectPass() { + return std::make_unique(); +} + +void RegisterOrderByDialectPass() { registerPass(CreateOrderByDialectPass); } + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.h new file mode 100644 index 0000000000000000000000000000000000000000..c9a724de47fa935076bcb1fb489e895d40e04f8e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_by_dialect.h @@ -0,0 +1,36 @@ +/* Copyright 2022 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_ORDER_BY_DIALECT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_ORDER_BY_DIALECT_H_ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Create an instance of a pass that reorders ops so ops of the same dialect are +// next to each other. +std::unique_ptr CreateOrderByDialectPass(); + +// Register this pass in the global registry of MLIR. +void RegisterOrderByDialectPass(); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_ORDER_BY_DIALECT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_for_program_key.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_for_program_key.cc new file mode 100644 index 0000000000000000000000000000000000000000..7fb460aa575696b650f82710c26d6449fb8d81f8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/order_for_program_key.cc @@ -0,0 +1,94 @@ +/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_ORDERFORPROGRAMKEYPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct OrderForProgramKeyPass + : public impl::OrderForProgramKeyPassBase { + void runOnOperation() override; +}; + +int DialectOrdering(Operation* predecessor, Operation* op) { + // Emit TPUCompileMlirOps first. + if (auto launch = llvm::dyn_cast(op)) { + for (Operation* _ : launch.getOps()) { + return 1; + } + } else if (llvm::isa(op)) { + return 1; + } + return 0; +} + +void OrderForProgramKeyPass::runOnOperation() { + ModuleOp module = getOperation(); + for (func::FuncOp func : module.getOps()) { + std::vector> side_effect_data; + const detail::SideEffectAnalysisInfo* info = + &getAnalysis().GetAnalysisForFunc(func); + auto extra_dependencies = + [&](Operation* op, + bool incoming) -> llvm::SmallVector const& { + return incoming ? info->DirectControlPredecessors(op) + : info->DirectControlSuccessors(op); + }; + func->walk([&](Operation* function) { + for (Region& region : function->getRegions()) { + for (Block& block : region.getBlocks()) { + if (block.empty()) continue; + auto ops = SortBlockTopologically(block, DialectOrdering, + extra_dependencies); + // Replace the block with the reordered block. + for (Operation* op : ops) { + op->remove(); + block.push_back(op); + } + } + } + }); + } +} +} // namespace + +std::unique_ptr> CreateOrderForProgramKeyPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/parallel_execute_to_islands.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/parallel_execute_to_islands.cc new file mode 100644 index 0000000000000000000000000000000000000000..3e5b58c82f31726210ed9015a44f763230d7fe88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/parallel_execute_to_islands.cc @@ -0,0 +1,279 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass forms `tf_executor.island` per region of +// `tf_device.parallel_execute`. +// +// For example, the following: +// +// %0 = tf_executor.island { +// tf_executor.yield +// } +// %1:2 = tf_executor.island { +// %2 = "tf.opA"(%arg0) : (tensor) -> tensor +// tf_executor.yield %2 : tensor +// } +// %3:2 = tf_executor.island(%0) { +// %4 = "tf_device.parallel_execute"() ({ +// %5 = "tf.opB"() : () -> tensor +// tf_device.return %5 : tensor +// }, { +// %5 = "tf.opC"(%1#0) : (tensor) -> tensor +// tf_device.return +// }) {} : () -> (tensor) +// tf_executor.yield %4 : tensor +// } +// tf_executor.fetch %3#0 : tensor +// +// gets lowered to: +// +// %0 = tf_executor.island { +// tf_executor.yield +// } +// %1:2 = tf_executor.island { +// %2 = "tf.opA"(%arg0) : (tensor) -> tensor +// tf_executor.yield %2 : tensor +// } +// +// // Island for the first region of above parallel_execute. +// %3:2 = tf_executor.island(%0) { +// %4 = "tf.opB"() : () -> tensor +// tf_executor.yield %4 : tensor +// } +// +// // Island for the second region of above parallel_execute. +// %5 = tf_executor.island(%0) { +// %6 = "tf.opC"(%1#0) : (tensor) -> tensor +// tf_executor.yield +// } +// +// tf_executor.fetch %3#0, %5 : tensor, !tf_executor.control +// +// When tf_device.parallel_execute op is enclosed after tf_device.replicate, +// then this pass will run following `replicate-to-island` pass and +// `tf-executor-break-up-islands` pass. + +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h" + +namespace mlir { +namespace TFDevice { +namespace { + +#define GEN_PASS_DEF_PARALLELEXECUTETOISLANDSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ParallelExecuteToIslandsPass + : public impl::ParallelExecuteToIslandsPassBase< + ParallelExecuteToIslandsPass> { + explicit ParallelExecuteToIslandsPass(bool legacy_graph_export) { + legacy_graph_export_ = legacy_graph_export; + } + void runOnOperation() override; +}; + +// Convert parallel_execute op to a set of islands where each region of +// parallel_execute op becomes a separate island. This ensures that the regions +// of the parallel_execute op gets executed concurrently. +void ExpandParallelExecuteToIslands( + tf_executor::IslandOp island_op, + tf_device::ParallelExecuteOp parallel_execute_op, OpBuilder* builder, + llvm::SmallVectorImpl& executes, + bool legacy_graph_export, int parallel_group_idx) { + const int num_regions = parallel_execute_op.getOperation()->getNumRegions(); + executes.reserve(num_regions); + + for (int i : llvm::seq(0, num_regions)) { + Block& execute_block = parallel_execute_op.GetRegionBlockWithIndex(i); + + // Replace terminator with tf_executor.YieldOp. + Operation* terminator = execute_block.getTerminator(); + builder->setInsertionPoint(terminator); + auto yield = tf_executor::YieldOp::create(*builder, terminator->getLoc(), + terminator->getOperands()); + terminator->erase(); + + // Create new island for each region. + builder->setInsertionPoint(island_op); + auto execute_island = tf_executor::IslandOp::create( + *builder, island_op.getLoc(), yield.getOperandTypes(), + island_op.getControl().getType(), island_op.getControlInputs()); + + // Move over tf_device.parallel_execute body region into newly the created + // island. + execute_island.getBody().takeBody(*execute_block.getParent()); + + // In new graph export pipeline, we will update control dependencies in the + // end of the pipeline. Mostly, it will rely on side effect analysis by + // considering accessing resource only. However, for branches under parallel + // group, there should not be any control deps between them even side effect + // analysis indicate some control deps. Therefore, we will mark parallel + // group and branch information here so that `UpdateControlDependenciesPass` + // can fetch the related information later. + if (!legacy_graph_export) { + std::string group_annotation = absl::StrCat( + "p", std::to_string(parallel_group_idx), ":", std::to_string(i)); + if (auto parallel_group_attr = + parallel_execute_op->getAttrOfType( + TF::kParallelExecAnnotation)) { + // Extend the existing attribute so that nested parallel execution + // structure is supported. + group_annotation = absl::StrCat(parallel_group_attr.getValue().str(), + ",", group_annotation); + } + for (auto& op : execute_island.GetBody()) { + op.setAttr(TF::kParallelExecAnnotation, + builder->getStringAttr(group_annotation)); + } + } + + executes.push_back(execute_island); + } +} + +void CreateIslandsFromParallelExecute( + tf_executor::IslandOp island_op, + tf_device::ParallelExecuteOp parallel_execute_op, bool legacy_graph_export, + int parallel_group_idx) { + OpBuilder builder(island_op); + + // Create islands for each region of the parallel_execute op. + llvm::SmallVector executes; + ExpandParallelExecuteToIslands(island_op, parallel_execute_op, &builder, + executes, legacy_graph_export, + parallel_group_idx); + + // Remap all results of parallel_execute op with outputs from newly created + // islands. + llvm::SmallVector parallel_execute_outputs; + parallel_execute_outputs.reserve( + parallel_execute_op.getOperation()->getNumResults()); + + for (auto& execute : executes) + parallel_execute_outputs.append(execute.getOutputs().begin(), + execute.getOutputs().end()); + + for (auto result : + llvm::zip(island_op.getOutputs(), parallel_execute_outputs)) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + // Add sink island to pin all islands as a control dependency if there is a + // control dependency leading from the parallel_execute originally. + if (!island_op.getControl().use_empty()) { + llvm::SmallVector island_operands; + for (auto& execute : executes) + island_operands.push_back(execute.getControl()); + + builder.setInsertionPoint(island_op); + auto island_sink = tf_executor::IslandOp::create( + builder, island_op.getLoc(), llvm::ArrayRef{}, + island_op.getControl().getType(), island_operands); + island_sink.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&island_sink.GetBody()); + tf_executor::YieldOp::create(builder, island_op.getLoc(), + llvm::ArrayRef{}); + island_op.getControl().replaceAllUsesWith(island_sink.getControl()); + } + + if (legacy_graph_export) { + // Islands with no uses should be pinned to a graph fetch so they still + // execute. + llvm::SmallVector unused_execute_controls; + for (auto& execute : executes) + if (execute.use_empty()) + unused_execute_controls.push_back(execute.getControl()); + + if (!unused_execute_controls.empty()) { + auto graph_op = island_op->getParentOfType(); + tf_executor::FetchOp fetch = graph_op.GetFetch(); + auto fetches = llvm::to_vector<8>(fetch.getOperands()); + fetches.append(unused_execute_controls.begin(), + unused_execute_controls.end()); + builder.setInsertionPoint(fetch); + tf_executor::FetchOp::create(builder, fetch.getLoc(), fetches); + fetch.erase(); + } + } else { + // Now, finally, we need to maintain the invariant expected to be maintained + // throughout the graph export pipeline that all islands always perfectly + // wrap a single op. So we'll split all islands which wrap multiple ops. + auto control_type = tf_executor::ControlType::get(island_op.getContext()); + for (auto& execute : executes) { + if (execute.GetBody().getOperations().size() > 1) { + mlir::TF::SplitIsland(execute, control_type); + } + } + } + + island_op.erase(); +} + +void ParallelExecuteToIslandsPass::runOnOperation() { + // Find islands with a single `tf_device.parallel_execute` and create + // individual islands per execute region of the parallel_execute. + llvm::SmallVector parallel_execute_op_islands; + getOperation().walk([&](tf_executor::GraphOp graph_op) { + for (auto island_op : graph_op.getOps()) { + if (!island_op.WrapsSingleOp()) { + island_op.emitError( + "tf_executor.island must perfectly wrap a single op"); + signalPassFailure(); + } + + if (isa(&island_op.GetBody().front())) + parallel_execute_op_islands.push_back(island_op); + } + }); + + // This number is unique within each function which is sufficient for + // `UpdateControlDependenciesPass` which consumes the related attributes. + // However, this assumes that we don't inline functions between this pass + // and `UpdateControlDependenciesPass`. + // If we need globally unique parallel group IDs in the future, + // we can either make this pass a module pass (using a global counter) + // or use an atomic counter. + int parallel_group_idx = 0; + for (tf_executor::IslandOp island_op : parallel_execute_op_islands) { + auto parallel_execute_op = + cast(island_op.GetBody().front()); + CreateIslandsFromParallelExecute(island_op, parallel_execute_op, + legacy_graph_export_, parallel_group_idx); + } +} +} // anonymous namespace + +std::unique_ptr> CreateParallelExecuteToIslandsPass( + bool legacy_graph_export) { + return std::make_unique(legacy_graph_export); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/passes.h new file mode 100644 index 0000000000000000000000000000000000000000..9be5083528dcbcabe5ade8e1a395af586e6e835a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/passes.h @@ -0,0 +1,730 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_PASSES_H_ + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassOptions.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "shardy/dialect/sdy/ir/dialect.h" // from @shardy // IWYU pragma: keep +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" + +namespace mlir { + +// Creates a pass that breaks up an island with multiple ops into multiple +// islands, each with a single op. +std::unique_ptr> CreateBreakUpIslandsPass(); + +// Creates a pass that converts mlir functions consisting of mlir ops into a +// tf_executor dialect as a single island. +std::unique_ptr> +CreateFunctionalToExecutorDialectConversionPass(); + +// Creates a pass that lifts inner ops of tf_executor.island ops in +// tf_executor.graph into the same block as the tf_executor.graph. +std::unique_ptr> +CreateExecutorDialectToFunctionalConversionPass(); + +namespace TF { +// Creates a pass that canonicalizes legacy compilation and replication +// attributes. +std::unique_ptr> +CreateCanonicalizeCompileAndReplicateAttributesPass(); + +// Creates a pass that drops `shape_invariant` attribute from While/WhileRegion +// ops. +std::unique_ptr> +CreateDropWhileShapeInvariantPass(); + +// Creates a pass that drops `shape_invariant` attribute from While/WhileRegion +// ops within device cluster. +std::unique_ptr> +CreateDropWhileShapeInvariantInDeviceClusterPass(); + +// Creates a pass that moves writes to replicate invariant resource variables +// outside tf_device.replicate op. +std::unique_ptr> +CreateHoistReplicateInvariantResourceWritesPass(); + +// Transforms functional control flow operations in the TensorFlow dialect to +// MLIR Control Flow Graph (CFG) form. +std::unique_ptr> +CreateTFFunctionalControlFlowToCFG(); + +// Transforms functional control flow operations in the TensorFlow dialect to +// their region based counterparts. +std::unique_ptr> +CreateTFFunctionalControlFlowToRegions(); +std::unique_ptr> CreateTFFunctionalControlFlowToRegions( + bool allow_passthrough_args); + +// Transforms region bases control flow operations in the TensorFlow dialect to +// their functional counterparts. +std::unique_ptr> +CreateTFRegionControlFlowToFunctional(); + +// Materialize the MlirPassthroughOp by replacing it with the MLIR module +// attached as an attribute. +std::unique_ptr> +CreateMaterializePassthroughOpPass(); + +// Replicates the TensorList init op by undoing some CSE needed for correct +// shape assignment in shape_inference. +std::unique_ptr> +CreateReplicateTensorListInitOpsPass(); + +// Performs Shape Inference on the TensorFlow dialect using the global registry. +std::unique_ptr> CreateTFShapeInferencePass( + ArrayRef> input_shapes = {}, + bool enable_stablehlo_propagation = false); + +// Performs TF.data optimizations. +std::unique_ptr> CreateTFDataOptimizationPass(); + +std::unique_ptr> CreateMoveTransposesPass(); +std::unique_ptr> CreateLayoutAssignmentPass(); + +// Guarantee that all FuncOp's have a single use. +std::unique_ptr> CreateGuaranteeAllFuncsOneUsePass(); + +// Optional pass which will unroll BatchMatMul and use only MatMul +std::unique_ptr> CreateUnrollBatchMatMulPassPass(); + +// Optional pass which will map TF BatchMatMul to TF Einsum +std::unique_ptr> CreateBatchMatMulToEinsumPass(); + +// Pass that transform Einsum to other TF Ops for the supported variants. +std::unique_ptr> CreateTransformEinsumPass(); + +// Optimizes Tensorflow graph. +std::unique_ptr> CreateTFOptimizePass(); +void RegisterTFOptimizePassPipeline(); + +// Creates pass to rewrite RecvTPUEmbeddingActivationsOp and +// SendTPUEmbeddingGradients ops to internal variants. +std::unique_ptr> CreateRewriteTPUEmbeddingOpsPass(); + +// Performs specific fusion for GPU targets. +std::unique_ptr> CreateGpuOpFusionPass(); + +// Creates a pass that decomposes to be compiled ReduceDataset ops into a while +// loop that iterates the dataset and calls the reduction function. +std::unique_ptr> CreateDecomposeReduceDatasetPass(); + +// Create a pass that convert ops that copy tensors between devices, e.g. +// tf.Identity. +std::unique_ptr> +CreateTensorDeviceCopyConversionPass(); + +// Returns a pass that folds tf.BroadcastTo nodes with subsequent nodes if they +// have built in broadcasting support. +std::unique_ptr> CreateBroadcastFoldPass(); + +void populateTfControlFlowToScfPatterns(MLIRContext* context, + RewritePatternSet* patterns); +// Create a pass to convert TensorFlow control flow to SCF. +std::unique_ptr> createConvertTfControlFlowToScfPass(); + +struct LayoutOptimizationPipelineOptions + : public PassPipelineOptions { + Option force_data_format{ + *this, "force-data-format", + llvm::cl::desc("Force data format for all layout sensitive ops")}; + Option skip_fold_transpose_in_ops{ + *this, "skip-fold-transpose-in-ops", + llvm::cl::desc("Skip folding transpose operands in Ops which can support " + "different layouts.")}; +}; + +// Layout optimization assigns optimal data layout for layout sensitive +// operations, and cancels all redundant transposes. +void CreateLayoutOptimizationPipeline( + OpPassManager& pm, // NOLINT - MLIR contract is pass by mutable reference. + const LayoutOptimizationPipelineOptions& options); + +struct StandardPipelineOptions + : public PassPipelineOptions { + Option enable_inliner{*this, "enable-inliner", + llvm::cl::desc("Enable inliner."), + llvm::cl::init(false)}; + Option form_clusters{*this, "form-clusters", + llvm::cl::desc("Enable Cluster Formation pass."), + llvm::cl::init(false)}; + Option enable_stablehlo_shape_propagation{ + *this, "enable-stablehlo-shape-propagation", + llvm::cl::desc( + "Enable StableHLO shape propagation in the TF shape inference pass."), + llvm::cl::init(false)}; + ListOption ops_to_preserve{ + *this, "ops-to-preserve", + llvm::cl::desc( + "list of ops to preserve during graph pruning. This is " + "useful for keeping ops with side effects, e.g. DebugIdentityOp.")}; +}; + +// Propagates the pass manager with the passes involved in transforming or +// optimizing an MLIR graph without any target specialization. +// NOLINTNEXTLINE - MLIR contract is pass by mutable reference. +void CreateTFStandardPipeline(OpPassManager& pm, + const StandardPipelineOptions& options); + +// Propagates device attributes of resources from callers to callees. +std::unique_ptr> CreateResourceDeviceInferencePass(); + +// Creates a pass that promotes resource reads/writes in `functions` to inputs +// and outputs of `functions`, assuming that resource operations have already +// been decomposed and function calls have already been inlined. If `functions` +// is empty, the pass is applied to the main function by default. The pass also +// annotates the input arguments for resources with the indices of their +// aliasing output arguments. +std::unique_ptr> CreatePromoteResourcesToArgsPass( + llvm::ArrayRef functions = {}); + +// Creates a pass that promotes tf.VarHandleOp to resource arguments for all +// functions. +std::unique_ptr> CreatePromoteVarHandlesToArgsPass(); + +// Creates a pass that converts readonly reference variables to the +// corresponding resource variables. +std::unique_ptr> +CreateConvertReadonlyReferenceVariablesToResourceVariablesPass(); + +// Creates a simple device assignment pass on TF dialect for CoreRT use case. +std::unique_ptr> CreateSimpleTFDeviceAssignmentPass( + llvm::StringRef default_device = "cpu"); + +// Creates a pass to perform device assignment for TF dialect ops that do not +// have device assignment, by using the device attribute of the function. +std::unique_ptr> +CreateTFDeviceAssignmentByFuncAttrPass(); + +// Performs resource lifting on the function body to hoist resource variable +// accesses outside all control flow statements. +LogicalResult ResourceLiftingForFunctionalControlFlow(func::FuncOp function); + +// Converts stack ops into operations on local variables, which can later be +// removed by resource lifting. Requires known maximum sizes of stacks and +// known element shapes of push ops. +std::unique_ptr> CreateStackOpsDecompositionPass(); + +// Creates a pass to strip the "tf._noinline" attribute from the functions in +// the module. +std::unique_ptr> CreateStripNoinlineAttributePass(); + +// Converts tensor list operations into operations on buffers and sizes. Needs +// static shapes and known max element count. +std::unique_ptr> CreateTensorListOpsDecompositionPass(); + +// Converts tensor array ops into operations on local variables, which can later +// be removed by resource lifting. Requires known sizes and known element shapes +// (either defined in TensorArrayV3 or implied in the first write). +std::unique_ptr> +CreateTensorArrayOpsDecompositionPass(); + +// Create a pass that legalize TFG to TF dialect. +std::unique_ptr CreateLegalizeTFGToTFEPass(); + +// Matches sequence of ops to TensorFlow fused kernels. This pass should not be +// generally used beyond exporting to runtimes that supports these ops. In the +// future these fusions may be codegen'd automatically. +std::unique_ptr> CreateFusedKernelMatcherPass(); + +// Creates function pass to select device index/fold tf.DeviceIndex. +std::unique_ptr> CreateDeviceIndexSelectorPass(); + +// Creates function pass to replace InitializeTableFromTextFileV2Ops with +// LookupTableImportV2Op ops. +std::unique_ptr> CreateInitTextFileToImportPass( + std::string saved_model_dir = ""); + +// Creates function pass to cluster TensorFlow ops by host. The program +// generated by this pass will have one function per host where all operations +// in the same function are placed on the same host. Each result of the per-host +// function will have a "tf.device" attribute which specifies the device +// assignment of the result. +std::unique_ptr> CreateClusterTFOpsByHostPass(); + +// Creates a pass to insert tf_device.send and tf_device.receive ops to make +// sure any argument of any op is on the same host of the op itself. +std::unique_ptr> CreateCrossHostTransferPass(); + +// Creates a pass that adds the device attribute to every tf.Const op based on +// the device attribute of the operations that read its result. If the result of +// a tf.Const op is read by operations placed on multiple devices, then the pass +// will replicate the tf.Const op once for each device. +std::unique_ptr> CreateConstantOpDeviceAssignmentPass(); + +// Returns pass that verifies whether all functions in module are of single +// tf_executor.graph and each tf_executor.island in tf_executor.graph only has a +// single op. +std::unique_ptr> CreateVerifySuitableForExportPass(); + +// Creates an op ordering favorable for the EmbeddingProgramKey pass. +std::unique_ptr> CreateOrderForProgramKeyPass(); + +// Returns pass that prepares TPU computation to be legal for export to +// TensorFlow. +std::unique_ptr> +CreatePrepareTpuComputationForTfExportPass(); + +// Rewrites ops that require quantized inputs or outputs to ops that allow +// non-quantized inputs and outputs. +std::unique_ptr> CreateLowerQuantizedPass(); + +// Reorders ops so ops of the same dialect are next to each other. +std::unique_ptr CreateOrderByDialectPass(); + +// Groups ops into functions that only contain one dialect. +std::unique_ptr CreateGroupByDialectPass(); + +// Removes unused parameters from functions & their callers. +std::unique_ptr> CreateRemoveUnusedArgumentsPass(); + +// Removes unused results from WhileRegion ops. +std::unique_ptr> +CreateRemoveUnusedWhileResultsPass(); + +// Hoists loop invariant ops to the outside of the loop. +std::unique_ptr> CreateHoistLoopInvariantPass(); + +// Creates VarHandleOps right next to the operations that use them. +std::unique_ptr> CreateLocalizeVarHandlesPass(); + +// Removes all TF attributes +std::unique_ptr> CreateStripTfAttributesPass(); + +// Converts AnonymousIteratorOps to (named) IteratorOps. +std::unique_ptr> CreateNameAnonymousIteratorsPass(); + +// Creates a pass that breaks up an island with multiple ops into multiple +// islands, each with a single op. This pass intentionally does not propagate +// control dependencies across newly created islands and is handled by +// CreateTFExecutorUpdateControlDependenciesPass. +std::unique_ptr> CreateSplitIntoIslandPerOpPass(); + +// Prints, but otherwise pipes through without changes, the current module. +std::unique_ptr> CreatePrintPass( + raw_ostream* os = nullptr); + +// Moves TPUCompileMlir ops as far to the front as possible. +std::unique_ptr> CreateMoveTpuCompileToFrontPass(); + +// Decomposes OptionalFromValue, OptionalGetValue, OptionalNone, +// and OptionalHasValue +std::unique_ptr> CreateDecomposeOptionalsPass(); + +//===----------------------------------------------------------------------===// +// XlaCallModule +//===----------------------------------------------------------------------===// + +// Creates a pass that deserializes functions in the StableHLO modules from +// `tf.XlaCallModule` to the top-level module. +std::unique_ptr> +CreateXlaCallModuleDeserializationPass(); + +// Creates a pass that serializes StableHLO functions referenced by +// `tf.XlaCallModule` from the top-level module to `tf.XlaCallModule`'s +// `module` attribute. +std::unique_ptr> CreateXlaCallModuleSerializationPass(); + +} // namespace TF + +namespace tf_executor { + +// Creates a pass to chain control outputs of while loop body. +std::unique_ptr> +CreateTFExecutorConvertControlToDataOutputsPass(); +std::unique_ptr> +CreateTFExecutorConvertControlToDataOutputsPass( + bool composite_tpuexecute_side_effects); + +std::unique_ptr> +CreateTFExecutorCheckControlDependenciesPass(); + +// Creates a pass to merge IslandOps from TFExecutor dialect. +std::unique_ptr> +CreateTFExecutorIslandCoarseningPass(); + +// Creates a pass to merge IslandOps for operation marked for execution on TPU. +// This is a V1 backward compatibility. +std::unique_ptr> +CreateTFExecutorTPUV1IslandCoarseningPass(); + +// Creates a pass to outlining TPU clusters from single IslandOp into a nested +// module suitable for being processed as-if it was a V2 module. +// This is a V1 backward compatibility. +std::unique_ptr> +CreateTFExecutorTPUV1IslandOutliningPass(); + +// Creates a pass to inline calls to the nested TPU module, this reverses the +// effect of the `TFExecutorTPUV1IslandOutlining` pass above. +// This is a V1 backward compatibility. +std::unique_ptr> +CreateTFExecutorTPUV1IslandInliningPass(); + +// Creates a pass to prune tf_executor.graph from dead nodes. +std::unique_ptr> CreateTFExecutorGraphPruningPass( + llvm::ArrayRef ops_to_preserve = {}); + +// Creates a pass to update control dependencies. +std::unique_ptr> +CreateTFExecutorUpdateControlDependenciesPass(); + +} // namespace tf_executor + +namespace TFDevice { +// Creates a pass that forms clusters from instructions that are assigned to +// same device. +std::unique_ptr> CreateClusterFormationPass(); + +// Sinks `tf.Const` operations in the ClusterOp region using them. This is +// performed in order to limit the number of values implicitly captured in this +// region before outlining. +std::unique_ptr> CreateClusterConstantSinkingPass( + llvm::function_ref filter = {}); + +// Creates a pass that outlines regions of tf_device.cluster operations. +std::unique_ptr> CreateClusterOutliningPass(); + +// Creates a pass that outlines regions of tf_device.launch operations. +std::unique_ptr> CreateLaunchOutliningPass(); + +// Creates a pass that converts tf_device::LaunchFuncOp into +// TF::PartitionedCallOp. +std::unique_ptr> CreateConvertLaunchFuncToTFCallPass(); + +// A pass that decomposes composite resource operations into primitive ones like +// ReadVariableOp, AssignVariableOp and other computations to facilitate +// transformations like resource op lifting. +std::unique_ptr> CreateDecomposeResourceOpsPass(); + +// A pass that decomposes composite resource operations in device cluster +// (tf_device.cluster op) into primitive ones like ReadVariableOp, +// AssignVariableOp and other computations to facilitate transformations like +// resource op lifting. +std::unique_ptr> +CreateDecomposeResourceOpsInClusterPass(); + +// Creates a pass that marks TPU cluster input-output pairs reading and writing +// to same resource variable as aliases. +std::unique_ptr> CreateMarkInputOutputAliasesPass(); + +// Creates a pass that lifts operations on external resource variables from +// device computation nested in `tf_device::LaunchOp` out so that resource +// variable load operations are all before device computation while resource +// variable store operations are all after device computation. After this pass, +// device computation no longer interacts with external resource variables. +std::unique_ptr> CreateResourceOpLiftingPass(); + +// Creates a pass that lifts operations from the main function. +std::unique_ptr> +CreateResourceOpLiftingForMainFunctionPass(); + +// Lifts resource operations from tf_device.launch_func ops nested in `op` +// outside. Returns a failure if there are remaining resource-type values that +// can not be lifted. +LogicalResult LiftResourceOps(Operation* op); + +// Creates a pass that hoists invariant operations in a `tf_device.replicate`. +std::unique_ptr> +CreateReplicateInvariantOpHoistingPass(); + +// Creates a pass that forms replica `tf_executor.island` from a single +// `tf_device.replicate` island. +std::unique_ptr> CreateReplicateToIslandPass( + bool legacy_graph_export = true); + +// Creates a pass that sets the device ordinal attribute of the required op +// using the replica id attribute. +std::unique_ptr> +CreateReplicaIDToDeviceOrdinalPass(); + +// Creates a pass that creates `tf_executor.island` from a single +// `tf_device.parallel_execute` island. +std::unique_ptr> CreateParallelExecuteToIslandsPass( + bool legacy_graph_export = true); + +// Creates a pass that annotates whether a LaunchFuncOp's parameters have the +// same data across replicas. +std::unique_ptr> +CreateAnnotateParameterReplicationPass(); + +// Creates a pass that merges control flow with similar predicates. +std::unique_ptr> CreateMergeControlFlowPass(); + +// Creates a pass that wraps each TensorFlow dialect with `device` attribute +// in a `tf_device.launch` op with the same `device` attribute. +std::unique_ptr> +CreateDeviceAttributeToLaunchPass(); + +// Creates a pass that hoists a `tf_device.launch` body and assigns a `device` +// attribute to each TensorFlow dialect op in the body based on the `device` +// attribute on the `tf_device.launch`. +std::unique_ptr> CreateLaunchToDeviceAttributePass( + bool legacy_graph_export = true); + +// Creates a pass to ensure that the `_xla_outside_compilation` and +// tf_device.launch op no longer exist after Outside Compilation is complete. +std::unique_ptr> +CreateVerifyNoOutsideCompilationMarkersPass(); + +// Create a pass that inlines the StatefulPartitionedCallOp op based in the +// parent region. +std::unique_ptr> CreateXlaInlineDeviceOpsPass(); + +// Creates a pass that rewrites partitioned calls with `_xla_compile_device +// type` with `tf.XlaLaunch` ops. +std::unique_ptr> CreateXlaRewritePass(); + +// Create a pass that validates the input graph to the CPU/GPU bridge. +std::unique_ptr> CreateXlaValidateInputsPass(); +} // namespace TFDevice + +namespace TFTPU { +// Creates a pass that converts unified compilation and replication +// attributes back to legacy attributes. +std::unique_ptr> +CreateConvertToLegacyCompileAndReplicateAttributesPass(); + +// Creates a pass that converts all TPUPartitionedInput to TPUPartitionedInputV2 +std::unique_ptr> +CreateTPUPartitionedOpConversionPass(); + +// Creates a pass that cleans up `_replication_info` attribute on operations +// that are inside a cluster. +std::unique_ptr> +CreateTPUClusterCleanupAttributesPass(); + +// Creates a pass that removes Identity/IdentityN ops from a cluster. +std::unique_ptr> CreateTPUIdentityPruningPass(); + +// Creates a pass that allows TPU program inputs to have layouts determined at +// run time. +std::unique_ptr> CreateTPUDynamicLayoutPass(); + +// Creates a pass that adds `tf.ReadVariableOp` to a TPU cluster for resources +// the cluster only writes to. +std::unique_ptr> CreateTPUResourceReadForWritePass(); + +// Creates a pass that reorders partitiioned resource reads and replicated +// inputs. +std::unique_ptr> +CreateTPUReorderReplicateAndPartitionedInputsPass(); + +// Creates a pass that partitions unpartitioned resource read/write to +// partitioned resource variables. +std::unique_ptr> +CreateTPUResourceReadsWritesPartitioningPass(); + +// Creates a pass that looks for usage of the result of +// TPUCopyWithDynamicShapeOp and annotate these values to be dynamic shape. This +// ensures that the generated tpu program has the correct inputs annotation. +std::unique_ptr> +CreateTPUAnnotateDynamicShapeInputsPass(); + +// Creates a pass that moves `tf.AssignVariableOp` into a +// `tf_device.parallel_execute` region if the `tf.AssignVariableOp` is the +// only consumer of a `tf_device.parallel_execute` result. +std::unique_ptr> +CreateTPUParallelExecuteSinkResourceWritePass(); + +// Create a pass that extract TPUCopyWithDynamicShapeOp from the host launch op +// and wrap them in device launch op. This allows this op executed on TPU while +// still compiled on host. +std::unique_ptr> +CreateExtractTPUCopyWithDynamicShapeOpPass(); + +// Creates a pass that wraps ReadVariableOp/AssignVariable op that consumes a +// packed tensor to have same device placement as underlying TPU device. +std::unique_ptr> +CreateTPUColocateCompositeResourceOps(); + +// Creates a pass that expands outside compilation cluster at the head/tail of +// TPU computation by adding outside compilation attribute to identity/cast ops +// that are only used for host computation. +std::unique_ptr> +CreateTPUHostComputationExpansionPass(); + +// Creates a pass that updates inputs to TPU embedding layer enqueue ops so that +// correct ops are invoked during training and evaluation. +std::unique_ptr> +CreateTPUUpdateEmbeddingEnqueueOpInputsPass(); + +// Creates a pass that propagates TPU devices to users. +std::unique_ptr> CreateTPUDevicePropagationPass(); + +// Create a pass that colocates each `Split` with its predecessor. +std::unique_ptr> CreateTPUColocateSplitsPass(); + +// Creates a pass that replicates the tf._TPUCompileMlir op on each host that +// needs the compiled program. It helps avoid transferring the compiled binary +// between hosts. +std::unique_ptr> +CreateTPUCompileOpReplicationPass(); + +// Creates a pass that applies space to depth transform +// for the first or frontier convolutions consume host inputs on TPU. +std::unique_ptr> CreateTPUSpaceToDepthPass(); + +// Adjusts the device on TPUCopyWithDynamicShape ops. +std::unique_ptr> +CreateColocateTPUCopyWithDynamicShapePass(); + +} // namespace TFTPU + +// Define the registrations in a detail namespace, just so that we can overload +// the main entry point `registerTensorFlowPasses` to inject +// RegisterTFOptimizePassPipeline. +namespace detail { + +// Direction in which to move transposes in MoveTransposePass. +enum MoveTransposeDirection { kBegin, kEnd }; + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_BATCHMATMULTOEINSUMPASS +#define GEN_PASS_DECL_BREAKUPISLANDSPASS +#define GEN_PASS_DECL_BROADCASTFOLDPASS +#define GEN_PASS_DECL_CANONICALIZECOMPILEANDREPLICATEATTRIBUTESPASS +#define GEN_PASS_DECL_CLUSTERCONSTANTSINKINGPASS +#define GEN_PASS_DECL_CLUSTERFORMATIONPASS +#define GEN_PASS_DECL_CLUSTEROUTLININGPASS +#define GEN_PASS_DECL_CLUSTERTFOPSBYHOSTPASS +#define GEN_PASS_DECL_CONSTANTOPDEVICEASSIGNMENTPASS +#define GEN_PASS_DECL_CONVERTLAUNCHFUNCTOTFCALLPASS +#define GEN_PASS_DECL_CONVERTREADONLYREFERENCEVARIABLESTORESOURCEVARIABLESPASS +#define GEN_PASS_DECL_CONVERTTFCONTROLFLOWTOSCFPASS +#define GEN_PASS_DECL_CONVERTTOLEGACYCOMPILEANDREPLICATEATTRIBUTESPASS +#define GEN_PASS_DECL_DECOMPOSEREDUCEDATASETPASS +#define GEN_PASS_DECL_DEVICEINDEXSELECTORPASS +#define GEN_PASS_DECL_DROPWHILESHAPEINVARIANTINDEVICECLUSTERPASS +#define GEN_PASS_DECL_DROPWHILESHAPEINVARIANTPASS +#define GEN_PASS_DECL_EXECUTORCHECKCONTROLDEPENDENCIESPASS +#define GEN_PASS_DECL_EXECUTORCONVERTCONTROLTODATAOUTPUTSPASS +#define GEN_PASS_DECL_EXECUTORDIALECTTOFUNCTIONALPASS +#define GEN_PASS_DECL_EXECUTORGRAPHPRUNINGPASS +#define GEN_PASS_DECL_EXECUTORISLANDCOARSENINGPASS +#define GEN_PASS_DECL_EXECUTORTPUV1ISLANDINLININGPASS +#define GEN_PASS_DECL_EXECUTORUPDATECONTROLDEPENDENCIESPASS +#define GEN_PASS_DECL_FUNCTIONALCONTROLFLOWTOCFGPASS +#define GEN_PASS_DECL_FUNCTIONALCONTROLFLOWTOREGIONSPASS +#define GEN_PASS_DECL_FUNCTIONALTOEXECUTORDIALECTCONVERSIONPASS +#define GEN_PASS_DECL_FUSEDKERNELMATCHERPASS +#define GEN_PASS_DECL_GROUPBYDIALECTPASS +#define GEN_PASS_DECL_GUARANTEEALLFUNCSONEUSEPASS +#define GEN_PASS_DECL_HOISTREPLICATEINVARIANTRESOURCEWRITESPASS +#define GEN_PASS_DECL_INITTEXTFILETOIMPORTPASS +#define GEN_PASS_DECL_LAUNCHOUTLININGPASS +#define GEN_PASS_DECL_LAYOUTASSIGNMENTPASS +#define GEN_PASS_DECL_LEGALIZEHLOTOTFPASS +#define GEN_PASS_DECL_LEGALIZETFGTOTFPASS +#define GEN_PASS_DECL_LOCALIZEVARHANDLESPASS +#define GEN_PASS_DECL_LOWERQUANTIZEDPASS +#define GEN_PASS_DECL_MARKINPUTOUTPUTALIASESPASS +#define GEN_PASS_DECL_MATERIALIZEPASSTHROUGHOP +#define GEN_PASS_DECL_MERGECONTROLFLOWPASS +#define GEN_PASS_DECL_MOVETRANSPOSESPASS +#define GEN_PASS_DECL_ORDERBYDIALECTPASS +#define GEN_PASS_DECL_PARALLELEXECUTETOISLANDSPASS +#define GEN_PASS_DECL_PREPARETPUCOMPUTATIONFORTFEXPORTPASS +#define GEN_PASS_DECL_PROMOTERESOURCESTOARGSPASS +#define GEN_PASS_DECL_PROMOTEVARHANDLESTOARGSPASS +#define GEN_PASS_DECL_REGIONCONTROLFLOWTOFUNCTIONALPASS +#define GEN_PASS_DECL_REMOVEUNUSEDARGUMENTSPASS +#define GEN_PASS_DECL_REMOVEUNUSEDWHILERESULTSPASS +#define GEN_PASS_DECL_REPLICAIDTODEVICEORDINALPASS +#define GEN_PASS_DECL_REPLICATEINVARIANTOPHOISTINGPASS +#define GEN_PASS_DECL_REPLICATETOISLANDPASS +#define GEN_PASS_DECL_RESOURCEDEVICEINFERENCEPASS +#define GEN_PASS_DECL_REWRITETPUEMBEDDINGOPSPASS +#define GEN_PASS_DECL_SIMPLETFDEVICEASSIGNMENTPASS +#define GEN_PASS_DECL_SPLITINTOISLANDPEROPPASS +#define GEN_PASS_DECL_STACKOPSDECOMPOSITIONPASS +#define GEN_PASS_DECL_STRIPNOINLINEATTRIBUTEPASS +#define GEN_PASS_DECL_TFDATAOPTIMIZATIONPASS +#define GEN_PASS_DECL_TFDEVICEASSIGNMENTBYFUNCATTRPASS +#define GEN_PASS_DECL_TPUBRIDGEEXECUTORISLANDOUTLININGPASS +#define GEN_PASS_DECL_TPUCLEANUPCLUSTERATTRIBUTESPASS +#define GEN_PASS_DECL_TPUCLUSTERFORMATIONPASS +#define GEN_PASS_DECL_TPUCOLOCATECOMPOSITERESOURCEOPSPASS +#define GEN_PASS_DECL_TPUDEVICEPROPAGATIONPASS +#define GEN_PASS_DECL_TPUDYNAMICLAYOUTPASS +#define GEN_PASS_DECL_TPUHOSTCOMPUTATIONEXPANSIONPASS +#define GEN_PASS_DECL_TPUIDENTITYPRUNINGPASS +#define GEN_PASS_DECL_EXTRACTTPUCOPYWITHDYNAMICSHAPEOPPASS +#define GEN_PASS_DECL_TPUPARALLELEXECUTESINKRESOURCEWRITEPASS +#define GEN_PASS_DECL_TPUREORDERREPLICATEANDPARTITIONEDINPUTSPASS +#define GEN_PASS_DECL_TPURESOURCEREADFORWRITEPASS +#define GEN_PASS_DECL_TPURESOURCEREADSWRITESPARTITIONINGPASS +#define GEN_PASS_DECL_TPUSPACETODEPTHPASS +#define GEN_PASS_DECL_TPUUPDATEEMBEDDINGENQUEUEOPINPUTSPASS +#define GEN_PASS_DECL_TENSORARRAYOPSDECOMPOSITIONPASS +#define GEN_PASS_DECL_TENSORDEVICECOPYCONVERSIONPASS +#define GEN_PASS_DECL_TENSORFLOWOPTIMIZEPASS +#define GEN_PASS_DECL_TENSORFLOWSHAPEINFERENCEPASS +#define GEN_PASS_DECL_TENSORLISTOPSDECOMPOSITIONPASS +#define GEN_PASS_DECL_TENSORFLOWGPUFUSION +#define GEN_PASS_DECL_TPUV1BRIDGEEXECUTORISLANDCOARSENINGPASS +#define GEN_PASS_DECL_TRANSFORMEINSUMPASS +#define GEN_PASS_DECL_UNROLLBATCHMATMULPASS +#define GEN_PASS_DECL_VERIFYSUITABLEFOREXPORTPASS +#define GEN_PASS_DECL_XLACALLMODULEDESERIALIZATIONPASS +#define GEN_PASS_DECL_XLACALLMODULESERIALIZATIONPASS +#define GEN_PASS_DECL_XLACALLMODULECUSTOMCALLTFFUNCTIONRENAMINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" +} // namespace detail +using namespace detail; // NOLINT +inline void registerTensorFlowPasses() { + detail::registerTensorFlowPasses(); + TF::RegisterTFOptimizePassPipeline(); +} + +namespace TFDevice { +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_ANNOTATEPARAMETERREPLICATIONPASS +#define GEN_PASS_DECL_DECOMPOSERESOURCEOPSINCLUSTERPASS +#define GEN_PASS_DECL_DECOMPOSERESOURCEOPSPASS +#define GEN_PASS_DECL_DEVICEATTRIBUTETOLAUNCHPASS +#define GEN_PASS_DECL_HOSTLAUNCHTOOUTSIDECOMPILEDPASS +#define GEN_PASS_DECL_LAUNCHTODEVICEATTRIBUTEPASS +#define GEN_PASS_DECL_OUTSIDECOMPILEDTOHOSTLAUNCHPASS +#define GEN_PASS_DECL_RESOURCEOPLIFTINGFORMAINFUNCTIONPASS +#define GEN_PASS_DECL_RESOURCEOPLIFTINGPASS +#define GEN_PASS_DECL_VERIFYNOOUTSIDECOMPILATIONMARKERSPASS +#define GEN_PASS_DECL_XLACLUSTERFORMATIONPASS +#define GEN_PASS_DECL_XLAINLINEDEVICEOPSPASS +#define GEN_PASS_DECL_XLAREWRITEPASS +#define GEN_PASS_DECL_XLAREWRITEV2PASS +#define GEN_PASS_DECL_XLAVALIDATEINPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" +} // namespace TFDevice + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/prepare_tpu_computation_for_tf_export.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/prepare_tpu_computation_for_tf_export.cc new file mode 100644 index 0000000000000000000000000000000000000000..c67a56e50372674a513170b17d288871754724da --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/prepare_tpu_computation_for_tf_export.cc @@ -0,0 +1,300 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_remaining_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/tf2xla/side_effect_util.h" + +namespace mlir { +namespace TF { +namespace { + +// Returns true if the given op is TF/XLA communication op in the old bridge. +bool IsCommunicationOp(Operation* op) { + return isa( + op); +} + +// Returns true if the given op is one of ops supported to have communication +// subcomputation in the TF/XLA bridge. +bool SupportsCommunicationComputation(Operation* op) { + return isa(op); +} + +#define GEN_PASS_DEF_PREPARETPUCOMPUTATIONFORTFEXPORTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class PrepareTpuComputationForTfExportPass + : public impl::PrepareTpuComputationForTfExportPassBase< + PrepareTpuComputationForTfExportPass> { + void runOnOperation() override; +}; + +class RewriteXlaHostComputeMlir + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::_XlaHostComputeMlirOp op, + PatternRewriter& rewriter) const override { + if (op.getManualSharding()) { + // This rewrite does not support manual_sharding. It is expected that the + // _XlaHostComputeMlirOp registered as an MlirXlaOpKernel will handle this + // case later once the XlaBuilder graph reaches it. + return failure(); + } + + llvm::SmallVector shape_attrs; + shape_attrs.reserve(op.getNumResults()); + for (Type ty : op.getResultTypes()) { + shape_attrs.push_back(TF::ShapeAttr::get(rewriter.getContext(), + mlir::cast(ty))); + } + + // Clone the `host_func` in the `host_mlir_module` attribute if it exists + // and use it for `shape_inference_graph` attribute on XlaHostCompute. + func::FuncOp cloned_func; + SymbolTable manager(op->getParentOfType()); + StringRef host_module = op.getHostMlirModule(); + if (!host_module.empty()) { + mlir::OwningOpRef module_for_func; + + func::FuncOp func = op.GetHostFunc(&module_for_func); + + OpBuilder::InsertionGuard guard(rewriter); + rewriter.setInsertionPointAfter(op->getParentOfType()); + cloned_func = llvm::dyn_cast_or_null( + rewriter.clone(*func.getOperation())); + manager.insert(cloned_func); + rewriter.setInsertionPointToStart(&cloned_func.getBody().front()); + auto result_type = + RankedTensorType::get({3}, rewriter.getType()); + auto dynamic_key = TF::_XlaCompileMlirPlaceholderProgramKeyOp::create( + rewriter, func.getLoc(), /*program=*/result_type, + llvm::ArrayRef{}); + + auto recv_at_host = TF::_XlaRecvAtHostOp::create( + rewriter, func.getLoc(), op.getOperandTypes(), + /*dynamic_key=*/dynamic_key, op.getSendKeyAttr(), + /*device_ordinal=*/rewriter.getI64IntegerAttr(0), + rewriter.getStringAttr("TPU")); + for (auto result : + llvm::zip(cloned_func.getArguments(), recv_at_host->getResults())) { + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + } + + rewriter.setInsertionPoint(cloned_func.getBody().front().getTerminator()); + TF::_XlaSendFromHostOp::create( + rewriter, func.getLoc(), + cloned_func.getBody().front().getTerminator()->getOperands(), + /*dynamic_key=*/dynamic_key, op.getRecvKeyAttr(), + /*device_ordinal=*/rewriter.getI64IntegerAttr(0), + rewriter.getStringAttr("TPU")); + } + + constexpr int64_t kDefaultCostEstimate = 1000000; + rewriter.replaceOpWithNewOp( + op, op.getResultTypes(), op.getInputs(), + /*ancestors=*/rewriter.getArrayAttr({}), + rewriter.getArrayAttr(shape_attrs), + /*shape_inference_graph=*/ + cloned_func ? SymbolRefAttr::get(cloned_func) : SymbolRefAttr(), + /*key=*/rewriter.getStringAttr(""), op.getSendKeyAttr(), + op.getRecvKeyAttr(), + /*cost_estimate_ns=*/rewriter.getI64IntegerAttr(kDefaultCostEstimate), + /*tpu_core=*/rewriter.getI64IntegerAttr(0)); + return success(); + } +}; + +void UpdateArgAttributes(mlir::func::FuncOp func) { + OpBuilder builder(func.getBody()); + for (int i = 0; i < func.getNumArguments(); ++i) { + constexpr char kShardingAttr[] = "mhlo.sharding"; + if (auto sharding = + func.getArgAttrOfType(i, kShardingAttr)) { + if (!sharding.getValue().empty()) { + BlockArgument arg = func.getArgument(i); + // TODO(hinsu): Instead of setting both 'sharding' and '_XlaSharding' + // attributes, only set the 'sharding' attribute. Both attributes are + // currently required as the XlaSharding xla op kernel doesn't use the + // 'sharding' attribute. + // TODO(b/414807890): Not sure whether we need to pass a V2 sharding to + // the _XlaShardingV2, do this when we actually have a use case. + auto updated_arg = TF::XlaShardingOp::create( + builder, func.getLoc(), arg.getType(), arg, /*sharding=*/sharding, + /*_XlaSharding=*/sharding, /*_XlaShardingV2=*/mlir::StringAttr()); + func.getArgument(i).replaceAllUsesExcept( + updated_arg, llvm::SmallPtrSet({updated_arg})); + } + + func.removeArgAttr(i, builder.getStringAttr(kShardingAttr)); + } + } +} + +LogicalResult RewriteCommunicationOps(ModuleOp module) { + MLIRContext* ctx = module.getContext(); + mlir::RewritePatternSet patterns(ctx); + patterns.add(ctx); + if (failed(mlir::applyPatternsGreedily(module, std::move(patterns)))) { + return module.emitError("failed to apply tf export preparation patterns"); + } + + // TODO(hinsu): Investigate if the semantics of keys for these communication + // ops between the old bridge and new bridge can be reconciled. + module.walk([&](Operation* op) { + if (isa(op)) { + StringRef old_key = op->getAttrOfType("key").getValue(); + auto new_key = StringAttr::get(ctx, old_key.str() + "_dtoh_0"); + op->setAttr("key", new_key); + } else if (isa(op)) { + StringRef old_key = op->getAttrOfType("key").getValue(); + auto new_key = StringAttr::get(ctx, old_key.str() + "_htod_0"); + op->setAttr("key", new_key); + } + }); + return success(); +} + +// Sets token input node names attribute and their corresponding original node +// names for tf/xla communication related ops. These attributes are used to +// order operations on device. First op in the region should have a special +// argument token and then remaining operations should have node name of the +// previous communication ops. +LogicalResult SetTokenInputAttrs(ModuleOp module) { + // Collect all the ops that needs to have token input names attributes. These + // ops are communication ops and all their parent ops via nesting or function + // calls. For example, IfRegion op and PartitionedCall op. + std::vector worklist; + absl::flat_hash_set ops_with_tokens; + module.walk([&](Operation* op) { + if (IsCommunicationOp(op)) { + ops_with_tokens.insert(op); + worklist.push_back(op); + } + }); + + SymbolTableCollection table; + SymbolUserMap symbol_map(table, module); + + // Regions that contains ops requiring token input attributes. + absl::flat_hash_set regions_with_token; + while (!worklist.empty()) { + Operation* op = worklist.back(); + worklist.pop_back(); + + Region* region = op->getParentRegion(); + regions_with_token.insert(region); + + // If the parent is not a FuncOp, then add the parent op containing a region + // to worklist. + Operation* parent = region->getParentOp(); + if (!isa(parent)) { + if (ops_with_tokens.insert(parent).second) { + worklist.push_back(parent); + } + continue; + } + + // For functions, get all the users and add them to the worklist. + for (auto& user : symbol_map.getUsers(parent)) { + if (ops_with_tokens.insert(user).second) { + worklist.push_back(user); + } + } + } + + // Use name mapper to uniquely name all ops in the module as export to + // TensorFlow graph may change node names. These op names here doesn't need to + // match the actual names in the graph as this sets original node name + // attribute for all the relevant nodes. + tensorflow::OpOrArgLocNameMapper name_mapper; + MLIRContext* ctx = module.getContext(); + for (Region* region : regions_with_token) { + // Initialize the token with the special argument token. This gets mapped to + // input token in the parent op or a new token for the entry computation. + auto token = StringAttr::get(ctx, tensorflow::kXlaTokenArgNodeName); + for (Operation& op : region->getOps()) { + // Only communication related ops that needs to have token should have the + // extra attribute. + if (!ops_with_tokens.contains(&op)) continue; + + if (!IsCommunicationOp(&op) && !SupportsCommunicationComputation(&op)) { + return op.emitOpError( + "does not support subcomputations with tf/xla communication ops"); + } + + op.setAttr(tensorflow::kXlaTokenInputNodesAttrName, + ArrayAttr::get(ctx, {token})); + + auto node_name = StringAttr::get(ctx, name_mapper.GetUniqueName(&op)); + op.setAttr(tensorflow::kXlaOriginalOutsideCompilationNodeName, node_name); + token = node_name; + } + } + return success(); +} + +void PrepareTpuComputationForTfExportPass::runOnOperation() { + ModuleOp module = getOperation(); + + for (func::FuncOp func : module.getOps()) { + UpdateArgAttributes(func); + } + + // First rewrite communication ops used in the new bridge to match old bridge + // semantics and then set token input node names attributes on the supported + // ops. + if (failed(RewriteCommunicationOps(module)) || + failed(SetTokenInputAttrs(module))) { + signalPassFailure(); + return; + } +} + +} // namespace + +std::unique_ptr> +CreatePrepareTpuComputationForTfExportPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/print.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/print.cc new file mode 100644 index 0000000000000000000000000000000000000000..0a8b8dbdbb83d9df41a5c90847850f66013ce229 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/print.cc @@ -0,0 +1,69 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/Support/Mutex.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_PRINTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class PrintPass : public impl::PrintPassBase { + public: + explicit PrintPass(raw_ostream* os = nullptr); + PrintPass(const PrintPass& other); + void runOnOperation() override; + + private: + llvm::sys::SmartMutex mutex_; + raw_ostream* os_; +}; + +PrintPass::PrintPass(raw_ostream* os) { + if (os) { + os_ = os; + } else { + os_ = &llvm::errs(); + } +} + +PrintPass::PrintPass(const PrintPass& other) : PrintPass(other.os_) {} + +void PrintPass::runOnOperation() { + llvm::sys::SmartScopedLock instrumentationLock(mutex_); + OpPrintingFlags flags = + OpPrintingFlags().elideLargeElementsAttrs().enableDebugInfo(false); + getOperation()->print(*os_, flags); +} +} // namespace + +std::unique_ptr> CreatePrintPass(raw_ostream* os) { + return std::make_unique(os); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/promote_resources_to_args.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/promote_resources_to_args.cc new file mode 100644 index 0000000000000000000000000000000000000000..ba04dca8be45199520484a587d0b3d1bdb016284 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/promote_resources_to_args.cc @@ -0,0 +1,435 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +constexpr char kResourceFunctionMsg[] = + "expects function level resource argument"; +constexpr char kInvalidResourceMsg[] = + "expects resource to be a VarHandleOp or function argument"; +constexpr char kResourceNameArgAttr[] = "tf.resource_name"; + +// Checks if a function has only one block. +mlir::LogicalResult CheckSingleBlockFunction(func::FuncOp function) { + if (!llvm::hasSingleElement(function)) { + return function.emitError() + << "expects function '" << function.getName() + << "' to have 1 block, got " << function.getBlocks().size(); + } + return success(); +} + +// Collects names of users of a resource that are not `tf.ReadVariableOp` and +// not `tf.AssignVariableOp`. +llvm::SmallSet GetCompositeResourceUserNames( + Value resource) { + // SmallSet will use a vector when there is only one element and use std::set + // when there are more than one elements. This ensures that the operations in + // the error message are ordered. + llvm::SmallSet composite_users; + for (Operation* user : resource.getUsers()) + if (!llvm::isa(user)) + composite_users.insert(user->getName().getStringRef()); + + return composite_users; +} + +// Checks that the only users of `tf.VarHandleOp` are +// `tf.ReadVariableOp` and `tf.AssignVariableOp`. +mlir::LogicalResult ValidateVarHandle(TF::VarHandleOp var_handle_op) { + auto composite_ops = GetCompositeResourceUserNames(var_handle_op); + if (!composite_ops.empty()) + return var_handle_op.emitOpError() + << "expects users to be 'tf.ReadVariableOp' or " + "'tf.AssignVariableOp', got [" + << llvm::join(composite_ops.begin(), composite_ops.end(), ", ") + << "]"; + + return success(); +} + +// Checks if resource argument has a valid resource subtype and its users are of +// `tf.ReadVariableOp` and `tf.AssignVariableOp` only. +mlir::LogicalResult ValidateResourceArgument(func::FuncOp function, + BlockArgument resource_arg, + TF::ResourceType resource_type) { + if (resource_type.getSubtypes().size() != 1) + return function.emitError() + << "expects resource type of argument " + << resource_arg.getArgNumber() << " to have one subtype, got " + << resource_type; + + auto composite_ops = GetCompositeResourceUserNames(resource_arg); + if (!composite_ops.empty()) + return function.emitError() + << "expects users of resource argument " + << resource_arg.getArgNumber() + << " to be 'tf.ReadVariableOp' or 'tf.AssignVariableOp', got [" + << llvm::join(composite_ops.begin(), composite_ops.end(), ", ") + << "]"; + + return success(); +} + +bool VariableIsInitialized(TF::VarHandleOp var_handle_op) { + auto is_variable_initialized = + var_handle_op->getAttrOfType("_is_initialized"); + // Assume variable is initialized if attribute is not set. + // There are paths that doesn't mark the variables. All variables + // that doesn't have the attribute will be promoted. + return !is_variable_initialized || is_variable_initialized.getValue(); +} + +// Adds resource arguments for every unique (name) variable handle. Associated +// `tf.VarHandleOp` are removed from the function. Variable shared names are +// returned in `var_handle_shared_names` based on the ordering of added resource +// arguments. +mlir::LogicalResult PromoteVarHandlesToArguments( + func::FuncOp function, bool add_validation, + llvm::SmallVectorImpl* var_handle_shared_names) { + Block& block = function.front(); + auto func_type = function.getFunctionType(); + + auto func_arg_types = llvm::to_vector<4>(func_type.getInputs()); + llvm::SmallDenseMap var_arg_index_by_name; + for (auto var_handle_op : + llvm::make_early_inc_range(block.getOps())) { + if (add_validation && failed(ValidateVarHandle(var_handle_op))) + return failure(); + // In the case of variables that are not initialized at graph creation + // then we keep them as VarHandleOps. + if (!VariableIsInitialized(var_handle_op)) continue; + + llvm::StringRef name = var_handle_op.getSharedNameAttr().getValue(); + auto it = var_arg_index_by_name.insert({name, func_arg_types.size()}); + if (it.second) { + var_handle_shared_names->emplace_back(name); + auto resource_type = var_handle_op.getResource().getType(); + func_arg_types.push_back(resource_type); + var_handle_op.getResource().replaceAllUsesWith( + block.addArgument(resource_type, var_handle_op.getLoc())); + } else { + var_handle_op.getResource().replaceAllUsesWith( + block.getArgument(it.first->getSecond())); + } + var_handle_op.erase(); + } + + if (!var_handle_shared_names->empty()) + function.setType(FunctionType::get(function.getContext(), func_arg_types, + func_type.getResults())); + + return success(); +} + +// Records the current live value for a resource variable and whether a read or +// write on the variable occurred. +struct ResourceInfo { + Value live_value = nullptr; + bool read = false; + bool write = false; +}; + +LogicalResult PromoteResourcesToArguments( + func::FuncOp function, + llvm::ArrayRef var_handle_shared_names) { + Block& block = function.front(); + + auto return_op = + llvm::dyn_cast_or_null(block.getTerminator()); + if (!return_op) + return function.emitError() << "expects function '" << function.getName() + << "' to have a MLIR ReturnOp"; + + llvm::SmallVector resources(function.getNumArguments()); + auto argument_types = + llvm::to_vector<4>(function.getFunctionType().getInputs()); + bool has_resources = false; + auto add_resource_argument = [&](BlockArgument arg, + TF::ResourceType resource_type) { + Type arg_type = resource_type.getSubtypes().front(); + arg.setType(arg_type); + resources[arg.getArgNumber()].live_value = arg; + argument_types[arg.getArgNumber()] = arg_type; + has_resources = true; + }; + + // Loop through the non `tf.VarHandleOp` resource arguments in the function, + // validate its uses and subtype, and store a mapping from that argument to + // itself as the current live value. + auto func_args = function.getArguments().take_front( + function.getNumArguments() - var_handle_shared_names.size()); + for (BlockArgument& func_arg : func_args) { + auto resource_type = mlir::dyn_cast( + getElementTypeOrSelf(func_arg.getType())); + if (!resource_type) continue; + if (failed(ValidateResourceArgument(function, func_arg, resource_type))) + return failure(); + + add_resource_argument(func_arg, resource_type); + } + + // Loop through `tf.VarHandleOp` resource arguments in the function and store + // a mapping from that argument to itself as the current live value. No + // validations are necessary here as these arguments were validated prior to + // being added. + auto var_handle_args = + function.getArguments().take_back(var_handle_shared_names.size()); + for (BlockArgument& var_handle_arg : var_handle_args) { + auto resource_type = mlir::cast( + getElementTypeOrSelf(var_handle_arg.getType())); + add_resource_argument(var_handle_arg, resource_type); + } + + if (!has_resources) return success(); + + // We initially assign the argument for a resource as the live value for the + // resource. We then walk through the operations in the function in their + // lexical order, to update the live value for the resource when we see a + // store to the resource and replace reads of the resource with uses of its + // live value. + for (Operation& op : llvm::make_early_inc_range(block)) { + if (auto read_op = llvm::dyn_cast(&op)) { + if (auto func_arg = + mlir::dyn_cast(read_op.getResource())) { + if (func_arg.getOwner() != &block) + return read_op.emitOpError(kResourceFunctionMsg); + + ResourceInfo& resource_info = resources[func_arg.getArgNumber()]; + resource_info.read = true; + read_op.getValue().replaceAllUsesWith(resource_info.live_value); + } else { + return read_op.emitOpError(kInvalidResourceMsg); + } + + read_op.erase(); + } else if (auto write_op = llvm::dyn_cast(&op)) { + if (auto func_arg = + mlir::dyn_cast(write_op.getResource())) { + if (func_arg.getOwner() != &block) + return write_op.emitOpError(kResourceFunctionMsg); + + ResourceInfo& resource_info = resources[func_arg.getArgNumber()]; + resource_info.write = true; + resource_info.live_value = write_op.getValue(); + } else { + return write_op.emitOpError(kInvalidResourceMsg); + } + + write_op.erase(); + } + } + + const int64_t num_results_before = function.getNumResults(); + auto return_operands = llvm::to_vector<4>(return_op.getOperands()); + auto result_types = llvm::to_vector<4>(return_op.getOperandTypes()); + llvm::SmallVector, 4> + output_only_resources; + llvm::SmallVector, 4> input_output_alias; + + // Collect new return values for variable writes and either (a) output-only + // resource attributes (if the resource is not promoted to an argument) or (b) + // mapping from resource input index to output alias (if the resource has been + // promoted to an argument). Resource arguments that were originally + // `tf.VarHandleOp` but not read are collected and then removed. + OpBuilder builder(return_op); + const int var_handles_start_idx = + function.getNumArguments() - var_handle_shared_names.size(); + int new_argument_index = 0; + llvm::SmallVector argument_indices_to_remove; + for (auto resource_and_index : llvm::enumerate(resources)) { + const auto& resource = resource_and_index.value(); + if (!resource.live_value) { + // Ignore non resource arguments. + ++new_argument_index; + continue; + } + + const int64_t index = resource_and_index.index(); + const bool is_var_handle = index >= var_handles_start_idx; + if (resource.write) { + if (!is_var_handle || resource.read) { + input_output_alias.push_back( + {new_argument_index, return_operands.size()}); + } else if (is_var_handle) { + output_only_resources.push_back( + {return_operands.size(), + var_handle_shared_names[index - var_handles_start_idx]}); + } + return_operands.push_back(resource.live_value); + result_types.push_back(resource.live_value.getType()); + } + + if (is_var_handle && !resource.read) { + assert(block.getArgument(index).getUses().empty()); + argument_indices_to_remove.push_back(index); + } else { + if (is_var_handle) { + // Add resource_name attribute to VarHandleOp read. + function.setArgAttr( + new_argument_index, kResourceNameArgAttr, + builder.getStringAttr( + var_handle_shared_names[index - var_handles_start_idx])); + } + ++new_argument_index; + } + } + + // Remove unread var handle arguments. + for (int argument_index_to_remove : + llvm::reverse(argument_indices_to_remove)) { + block.eraseArgument(argument_index_to_remove); + argument_types.erase(argument_types.begin() + argument_index_to_remove); + } + + // Rewrite return if there are variable writes. + const int return_operands_size = return_operands.size(); + if (return_operands_size > num_results_before) { + func::ReturnOp::create(builder, return_op.getLoc(), return_operands); + return_op.erase(); + } + + // Update function argument and result types with new resource subtypes. + function.setType(builder.getFunctionType(argument_types, result_types)); + + // Add resource_name attribute to the output for the resources. + for (auto& resource : output_only_resources) + function.setResultAttr(resource.first, kResourceNameArgAttr, + builder.getStringAttr(resource.second)); + + // Add aliasing_output attribute to the input argument for the resources that + // are updated by the function. + for (auto& input_output : input_output_alias) + function.setArgAttr(input_output.first, "tf.aliasing_output", + builder.getI64IntegerAttr(input_output.second)); + + return success(); +} + +#define GEN_PASS_DEF_PROMOTERESOURCESTOARGSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class PromoteResourcesToArgsPass + : public impl::PromoteResourcesToArgsPassBase { + public: + PromoteResourcesToArgsPass() = default; + explicit PromoteResourcesToArgsPass(llvm::ArrayRef functions); + void runOnOperation() override; +}; + +PromoteResourcesToArgsPass::PromoteResourcesToArgsPass( + llvm::ArrayRef functions) { + functions_ = functions; +} + +void PromoteResourcesToArgsPass::runOnOperation() { + ModuleOp module = getOperation(); + if (llvm::size(functions_) == 0) { + functions_ = {"main"}; + } + SymbolTable symbolTable(module); + for (const std::string& f : functions_) { + func::FuncOp func = symbolTable.lookup(f); + if (!func) continue; + + // This routine should only be called when control flow operations are still + // represented with TF IfOp and WhileOp operations. In this case, there + // should be only one basic blocks in the MLIR representation. + if (failed(CheckSingleBlockFunction(func))) return signalPassFailure(); + + llvm::SmallVector var_handle_shared_names; + if (failed(ResourceLiftingForFunctionalControlFlow(func)) || + failed(PromoteVarHandlesToArguments(func, /*add_validation=*/true, + &var_handle_shared_names)) || + failed(PromoteResourcesToArguments(func, var_handle_shared_names))) + return signalPassFailure(); + } +} + +#define GEN_PASS_DEF_PROMOTEVARHANDLESTOARGSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class PromoteVarHandlesToArgsPass + : public impl::PromoteVarHandlesToArgsPassBase< + PromoteVarHandlesToArgsPass> { + public: + void runOnOperation() override; +}; + +void PromoteVarHandlesToArgsPass::runOnOperation() { + ModuleOp module = getOperation(); + MLIRContext* context = module.getContext(); + for (auto function : module.getOps()) { + if (failed(CheckSingleBlockFunction(function))) return signalPassFailure(); + + llvm::SmallVector var_handle_shared_names; + (void)PromoteVarHandlesToArguments(function, /*add_validation=*/false, + &var_handle_shared_names); + + // Add resource names for each `tf.VarHandleOp` that were promoted to + // resource arguments. + const int var_handle_args_offset = + function.getNumArguments() - var_handle_shared_names.size(); + for (auto var_name_and_index : llvm::enumerate(var_handle_shared_names)) + function.setArgAttr(var_name_and_index.index() + var_handle_args_offset, + kResourceNameArgAttr, + StringAttr::get(context, var_name_and_index.value())); + } +} + +} // namespace + +std::unique_ptr> CreatePromoteResourcesToArgsPass( + llvm::ArrayRef functions) { + return std::make_unique(functions); +} + +std::unique_ptr> CreatePromoteVarHandlesToArgsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/readonly_references_to_resources.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/readonly_references_to_resources.cc new file mode 100644 index 0000000000000000000000000000000000000000..c095b6c60ccfbfe3a546999a39fafd8f6641eede --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/readonly_references_to_resources.cc @@ -0,0 +1,201 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +// Location attribute. +constexpr StringRef kClassAttr = "_class"; +constexpr StringRef kSharedNameAttr = "shared_name"; +constexpr StringRef kLocationPrefix = "loc:@"; + +#define GEN_PASS_DEF_CONVERTREADONLYREFERENCEVARIABLESTORESOURCEVARIABLESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass that converts readonly reference variables to the corresponding +// resource variables. +// +// It converts (VariableV2 -> Identity) to (VarHandle -> ReadVariable). +// +// For the background, this pass is a part of hoisting VariableV2 ops by +// re-using the pipeline for hoisting (VarHandle -> ReadVariable) cases, which +// can be done by the following passes: +// - Capturing resource values into global tensors (importing saved model). +// - Promoting VarHandle ops to function input/outputs. +// - Freezing global tensor pass. +// +// This path assumes that all the VariableV2 ops is read-only via verifying the +// heuristic method that assumes that all the users of them is Identity op, +// fed directly. +class ConvertReadonlyReferenceVariablesToResourceVariablesPass + : public impl::ConvertReadonlyReferenceVariablesToResourceVariablesPassBase< + ConvertReadonlyReferenceVariablesToResourceVariablesPass> { + void runOnOperation() override; +}; + +// Parse node name from "_class" or "shared_name" attributes. +StringRef GetNodeNameFromClassAttrOrSharedNameAttr(Operation *op) { + // Parse node name from the `shared_name` attribute first. The variable v2 op + // relies on the share name to look up from the TensorFlow's resource manager. + StringAttr shared_name_attr = op->getAttrOfType(kSharedNameAttr); + if (shared_name_attr) { + auto shared_name = StringRef(shared_name_attr.getValue()); + if (!shared_name.empty()) { + return shared_name; + } + } + // Attempt to parse "_class" attribute if there is no "shared_name" + // attribute. + ArrayAttr classes_attr = op->getAttrOfType(kClassAttr); + if (!classes_attr) { + // Attempt to parse "_class" from the IdentityOp that follows VariableV2. + // For read-only reference variables, IdentityOp should be the only user of + // VariableV2. + auto identity_op = op->getUsers().begin(); + classes_attr = identity_op->getAttrOfType(kClassAttr); + if (!classes_attr) { + op->emitOpError() << "has no '_class' and 'shared_name' attributes"; + return StringRef(); + } + } + + StringRef result; + for (Attribute class_attr : classes_attr) { + StringRef node_name = mlir::cast(class_attr).getValue(); + if (!node_name.starts_with(kLocationPrefix)) { + continue; + } + if (!result.empty()) { + // Invalid case since there are multiple loc:@ attributes. + op->emitOpError() + << "expects only one named location in '_class' attribute, but got " + << classes_attr; + return StringRef(); + } + result = node_name.drop_front(kLocationPrefix.size()); + } + if (result.empty()) { + op->emitOpError() << "expects variable name in '_class' attribute, but got " + << classes_attr; + } + return result; +} + +void ConvertReadonlyReferenceVariablesToResourceVariablesPass:: + runOnOperation() { + func::FuncOp func = getOperation(); + + OpBuilder builder(func.getContext()); + SmallVector variable_v2s_to_replace; + + // Checks all the VariableV2 ops is read-only via verifying the heuristic + // method that assumes that all the users of them is Identity op, feeded + // directly. + auto read_only_vars_fn = [&variable_v2s_to_replace]( + VariableV2Op variable_v2_op) { + if (variable_v2_op.getResult().use_empty()) { + // Erase the op when there is no user. + variable_v2_op.erase(); + return mlir::WalkResult::advance(); + } + if (!all_of(variable_v2_op.getResult().getUsers(), [&variable_v2_op]( + Operation *user) { + if (!isa(user)) { + variable_v2_op.emitOpError() + << "expects all users to be 'tf.Identity', but got user " + << user->getName(); + return false; + } + return true; + })) { + return mlir::WalkResult::interrupt(); + } + variable_v2s_to_replace.push_back(variable_v2_op); + return mlir::WalkResult::advance(); + }; + + WalkResult walk_res = func.walk(read_only_vars_fn); + if (walk_res.wasInterrupted()) return signalPassFailure(); + + for (VariableV2Op variable_v2_op : variable_v2s_to_replace) { + builder.setInsertionPoint(variable_v2_op); + ShapedType shaped_type = + mlir::cast(variable_v2_op.getResult().getType()); + TensorType tensor_type = mlir::cast(DropRefType(shaped_type)); + StringAttr device_attr = + variable_v2_op->getAttrOfType("device"); + if (!device_attr) device_attr = builder.getStringAttr(""); + StringRef variable_name = + GetNodeNameFromClassAttrOrSharedNameAttr(variable_v2_op); + if (variable_name.empty()) { + return signalPassFailure(); + } + VarHandleOp var_handle_op = VarHandleOp::create( + builder, variable_v2_op.getLoc(), + ArrayRef{RankedTensorType::get( + {}, TF::ResourceType::get(ArrayRef{tensor_type}, + builder.getContext()))}, + ArrayRef{}, + ArrayRef{ + builder.getNamedAttr("device", device_attr), + builder.getNamedAttr("container", + variable_v2_op.getContainerAttr()), + builder.getNamedAttr("shared_name", + builder.getStringAttr(variable_name))}); + for (Operation *user : + make_early_inc_range(variable_v2_op.getResult().getUsers())) { + builder.setInsertionPoint(user); + ReadVariableOp read_variable_op = ReadVariableOp::create( + builder, user->getLoc(), ArrayRef{tensor_type}, + ArrayRef{var_handle_op}); + user->getResult(0).replaceAllUsesWith(read_variable_op.getResult()); + user->erase(); + } + variable_v2_op.erase(); + } +} + +} // namespace + +std::unique_ptr> +CreateConvertReadonlyReferenceVariablesToResourceVariablesPass() { + return std::make_unique< + ConvertReadonlyReferenceVariablesToResourceVariablesPass>(); +} + +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/region_control_flow_to_functional.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/region_control_flow_to_functional.cc new file mode 100644 index 0000000000000000000000000000000000000000..fdd0bddd0feaaa32adad0082808aa64254c8c732 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/region_control_flow_to_functional.cc @@ -0,0 +1,681 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass transforms region bases control flow operations in +// the TensorFlow dialect to their functional counterparts, i.e., +// tf.IfRegion -> tf.If and tf.WhileRegion -> tf.While + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +#define DEBUG_TYPE "tf-region-cf-to-functional" + +namespace mlir { +namespace TF { + +namespace { + +constexpr char kElseFuncNameAttr[] = "_else_func_name"; +constexpr char kThenFuncNameAttr[] = "_then_func_name"; +constexpr char kXlaPropagateCompileTimeConsts[] = + "_xla_propagate_compile_time_consts"; + +#define GEN_PASS_DEF_REGIONCONTROLFLOWTOFUNCTIONALPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct RegionControlFlowToFunctional + : public impl::RegionControlFlowToFunctionalPassBase< + RegionControlFlowToFunctional> { + void runOnOperation() override; + + private: + LogicalResult ConvertIfOp(SymbolTableCollection& symbol_table, + IfRegionOp if_region); + LogicalResult ConvertCaseOp(SymbolTableCollection& symbol_table, + CaseRegionOp case_region); + LogicalResult ConvertWhileOp(SymbolTableCollection& symbol_table, + WhileRegionOp while_region); + LogicalResult ConvertGeneratorDatasetOp(SymbolTableCollection& symbol_table, + GeneratorDatasetRegionOp regional); + + // Get unique name by using the loc to name mapping. + std::string GetName(Operation* op, StringRef suffix); + + tensorflow::OpOrArgLocNameMapper mapper; + llvm::SmallVector worklist; +}; + +std::string RegionControlFlowToFunctional::GetName(Operation* op, + StringRef suffix) { + return (mapper.GetUniqueName(op) + suffix).str(); +} + +// Returns all the external values referenced from the given regions. If the +// external value is a constant, sink it into the region instead (and do not +// add it to the returned vector). +llvm::SmallVector CollectExternValues(mlir::RegionRange regions) { + llvm::SetVector extern_values; + + for (Region* region : regions) { + llvm::SetVector region_extern_values; + getUsedValuesDefinedAbove(*region, region_extern_values); + + // Sink down constants into the functions. + for (auto extern_value : region_extern_values) { + if (!matchPattern(extern_value, m_Constant())) { + extern_values.insert(extern_value); + continue; + } + // Add constant at start of region. + auto const_builder = OpBuilder::atBlockBegin(®ion->front()); + auto const_value = const_builder.clone(*extern_value.getDefiningOp()); + replaceAllUsesInRegionWith(extern_value, const_value->getResult(0), + *region); + } + } + + return llvm::to_vector<4>(extern_values); +} + +// Copies over optional attributes from source region op `src` to the given +// functional op `dst` and appropriately overrides any necessary attributes. +void CopyAndOverrideAttributes(Operation* src, Operation* dst, + OpBuilder* builder) { + CopyDeviceAndUnderscoredAttributes(src, dst); + + // Explicitly override attribute to propagate constants to the functions + // before compiling to XLA. This is necessary along with conversion to + // functional format because inlined regions may have moved loop invariant ops + // outside of the region which may cause some new legalization failures. + // TODO(b/126739593): Enable this attribute in TensorFlow by default. Also, + // see b/185542519 for the context. + dst->setAttr(kXlaPropagateCompileTimeConsts, builder->getBoolAttr(true)); +} + +// If the region only does a single function call whose operands / returns match +// exactly the block args and results, return the name of the called function. +std::optional UnwrapSingleFunctionCall(Region& region) { + // The pattern we're matching is + // ^block(arg0, arg1, ..., argN): + // r0, r1, ..., rN = func.call @foo(arg0, arg1, ..., argN) + // "tf.yield"(r0, r1, ..., rN) + if (!region.hasOneBlock()) return std::nullopt; + Block& block = region.front(); + if (std::distance(block.begin(), block.end()) != 2) return std::nullopt; + TF::YieldOp yield = + llvm::dyn_cast_or_null(block.getTerminator()); + if (!yield) return std::nullopt; + func::CallOp call = llvm::dyn_cast_or_null(*block.begin()); + if (!call) return std::nullopt; + if (block.getNumArguments() != call.getNumOperands() || + call.getNumResults() != yield.getNumOperands()) + return std::nullopt; + for (auto [arg, operand] : + llvm::zip(block.getArguments(), call.getOperands())) { + if (arg != operand) return std::nullopt; + } + for (auto [ret, operand] : + llvm::zip(call.getResults(), yield.getOperands())) { + if (ret != operand) return std::nullopt; + } + SymbolRefAttr symbol = call.getCallableForCallee().get(); + if (!symbol) return std::nullopt; + return symbol.getLeafReference(); +} + +// Extracts the contents of a region with a single block into a new function. +// `extern_values` is the set of external values that the region refers to. +// Returns the name of the newly created function. +// +// Inputs to the terminator of the region are converted to return values of +// the function. If `extern_values_passthrough` is true, all the extern values +// are also added as return values from the function +StringRef ExtractSingleBlockRegion( + SymbolTableCollection& symbol_table, Region& region, StringRef name, + llvm::SmallVectorImpl& extern_values, + llvm::SmallVectorImpl& worklist, + bool extern_values_passthrough, bool only_one_return_value, + bool allow_return_of_existing = false) { + if (allow_return_of_existing && extern_values.empty()) { + auto existing = UnwrapSingleFunctionCall(region); + if (existing) return *existing; + } + + ModuleOp module = region.getParentOfType(); + OpBuilder builder(module.getContext()); + auto loc = region.getParentOp()->getLoc(); + Block& entry = region.front(); + int num_region_arguments = entry.getNumArguments(); + Operation* terminator = entry.getTerminator(); + + // Build the function type. Region arguments and extern values together + // become the function arguments, with region arguments going first. + auto input_types = llvm::to_vector<4>(entry.getArgumentTypes()); + for (auto input : extern_values) input_types.push_back(input.getType()); + + // Terminator operands and pass through extern values (if enabled) together + // become the function return values. + auto return_types = llvm::to_vector<4>(terminator->getOperandTypes()); + if (extern_values_passthrough) + for (auto input : extern_values) return_types.push_back(input.getType()); + if (only_one_return_value) { + return_types.resize(1); + } + + auto type = FunctionType::get(region.getContext(), input_types, return_types); + + // Create new function and extract region body into the function. + auto outlined_func = func::FuncOp::create(builder, loc, name, type); + Region& func_region = outlined_func.getBody(); + func_region.takeBody(region); + Block& first_block = func_region.front(); + + // Replace all external uses with function arguments. + for (auto it : llvm::enumerate(extern_values)) { + Value arg = first_block.addArgument(it.value().getType(), loc); + replaceAllUsesInRegionWith(it.value(), arg, func_region); + } + + // Function return values are all the terminator operands + pass through + // extern values (if enabled). + auto return_values = llvm::to_vector<4>(terminator->getOperands()); + if (only_one_return_value) { + return_values.resize(1); + } + if (extern_values_passthrough) + return_values.insert(return_values.end(), + first_block.args_begin() + num_region_arguments, + first_block.args_end()); + + // Replace the existing terminator with a return. + terminator = first_block.getTerminator(); + builder.setInsertionPoint(terminator); + func::ReturnOp::create(builder, terminator->getLoc(), return_values); + terminator->erase(); + + outlined_func.setPrivate(); + + // Uniquify the function name, and insert into module. + symbol_table.getSymbolTable(module).insert(outlined_func, + module.getBody()->begin()); + + // Add the outlined function to the worklist in case its body has + // IfRegion or WhileRegion ops that need to converted. + worklist.push_back(outlined_func); + return outlined_func.getName(); +} + +// Returns call for region with single call whose result feeds into the +// terminator of the region. If `allow_to_bool` is true, it allows patterns used +// in the condition of While ops, i.e. it allows a single bool (possibly passed +// through a ToBoolOp) between the region yield and the call. Returns none if +// the region does not conform to this pattern. +std::optional IsSingleCallRegion(Region& region, + bool allow_to_bool = false) { + if (!llvm::hasSingleElement(region)) return std::nullopt; + + Block& block = region.front(); + auto it = block.rbegin(); + YieldOp yield = dyn_cast(*it++); + + if (it == block.rend()) return std::nullopt; + + // Operation which is expected to consume all the call results. + Operation* call_consumer = yield; + + // Allow a single ToBoolOp between the call and the yield (valid only + // when the yield has a single operand) + if (allow_to_bool && yield.getNumOperands() == 1 && isa(*it)) { + if (it->getResult(0) != yield.getOperand(0)) return std::nullopt; + call_consumer = cast(*it); + it++; + if (it == block.rend()) return std::nullopt; + } + + // Check if there is a Call before the Yield. + func::CallOp call = dyn_cast(*it++); + if (!call) return std::nullopt; + + if (allow_to_bool && call.getNumResults() == 1 && + yield->getNumOperands() != 1) { + // Allow patterns of the form + // %cond = call(...) + // yield %cond, [...passthrough args...] + if (yield->getNumOperands() != block.getNumArguments() + 1) + return std::nullopt; + for (auto [yield_operand, block_arg] : + llvm::zip(yield->getOperands().drop_front(1), block.getArguments())) { + if (yield_operand != block_arg) return std::nullopt; + } + } else { + // All call results should feed into expected consumer + // All results of the call should feed into the yield. + if (call.getNumResults() != call_consumer->getNumOperands()) + return std::nullopt; + } + + for (auto res_it : llvm::zip(call.getResults(), call_consumer->getOperands())) + if (std::get<0>(res_it) != std::get<1>(res_it)) return std::nullopt; + + // There can only be non-truncating cast op's prior to the call. + for (; it != block.rend(); ++it) { + CastOp cast = dyn_cast(*it); + if (!cast || cast.getTruncate()) return std::nullopt; + } + + return call; +} + +using ArgMatcherFn = function_ref; + +// Returns whether the arguments of the given 2 calls are match (after looking +// through cast ops). `matcher` is the predicate used to check if two arguments +// match. +bool MatchCallArgs(func::CallOp first, func::CallOp second, + ArgMatcherFn matcher) { + if (first.getNumOperands() != second.getNumOperands()) return false; + + Region& first_region = *first->getParentRegion(); + Region& second_region = *second->getParentRegion(); + + for (auto it : llvm::zip(first.getArgOperands(), second.getArgOperands())) { + // Get the defining Op, skipping over casts. + auto get_defining_op = [](Value value) { + while (auto cast_op = + llvm::dyn_cast_or_null(value.getDefiningOp())) { + // Consider cast compatibility in case + // %cast = "tf.Cast"(%0) : (tensor<2xi64>) -> tensor<2xf32> + // is skipped. + if (cast_op.getSrcT() != cast_op.getDstT()) { + break; + } + value = cast_op.getOperand(); + } + return value; + }; + Value first_arg = get_defining_op(std::get<0>(it)); + Value second_arg = get_defining_op(std::get<1>(it)); + + if (!matcher(first_arg, first_region, second_arg, second_region)) + return false; + } + return true; +} + +// Summary information for trivially transforming region based op's to +// functional ops. A trivial transformation can be done when the regions are +// just calls to functions, in which case no outlining is needed. +struct TrivialTransformInfo { + // Can the op be transformed trivially? + bool can_transform = false; + + // List of callee names (one for each region). + llvm::SmallVector callee_names; + + // Analyzes the given calls (from regions attached to the same parent op) to + // check if the parent op be transformed to functional form trivially (i.e., + // reusing existing functions and without outlining). This is possible when + // all the regions are single call regions (checked using matchers outside + // this class) and the all the calls match using the given argument matcher. + // + // If such a trivial transformation is possible, stash the relevant + // information needed for the transformation, else indicate that a trivial + // transformation is not possible by setting `can_transform` to false. + TrivialTransformInfo(std::optional first_call, + std::optional second_call, + ArgMatcherFn arg_matcher) { + if (!first_call || !second_call) return; + + if (!MatchCallArgs(first_call.value(), second_call.value(), arg_matcher)) + return; + + can_transform = true; + callee_names = {first_call.value().getCallee(), + second_call.value().getCallee()}; + } +}; + +// Transform IfRegionOp to IfOp. +LogicalResult RegionControlFlowToFunctional::ConvertIfOp( + SymbolTableCollection& symbol_table, IfRegionOp if_region) { + llvm::SmallVector extern_values; + + // For IfOp, arguments of calls in the then and else regions match if they + // are the same value. + auto if_arg_matcher = [&](Value first, Region&, Value second, Region&) { + if (first != second) return false; + + // collect the call arguments post lookup through cast Op's + extern_values.push_back(first); + return true; + }; + + const TrivialTransformInfo tti(IsSingleCallRegion(if_region.getThenBranch()), + IsSingleCallRegion(if_region.getElseBranch()), + if_arg_matcher); + + std::string then_name, else_name; + + if (tti.can_transform) { + // We can transform to functional form trivially without outlining. + then_name = tti.callee_names[0].str(); + else_name = tti.callee_names[1].str(); + } else { + // Collect external values that are used within the else and then bodies. + extern_values = CollectExternValues(if_region.getRegions()); + + // These external values need to be added as inputs to the generated If. The + // order is determined by the order of these values the `extern_vales`. + + // Create 2 new functions with the input signature matching this order, + // and outline the `then` and `else` regions by moving the bodies of these + // regions into these functions. Replace tf.yield with a regular return. + if (if_region->hasAttrOfType(kThenFuncNameAttr) && + !if_region.get_thenFuncNameAttr().getValue().empty()) { + then_name = + mapper.GetUniqueName(if_region.get_thenFuncNameAttr().getValue()) + .str(); + } else { + then_name = GetName(if_region, "_then"); + } + then_name = ExtractSingleBlockRegion( + symbol_table, if_region.getThenBranch(), then_name, extern_values, + worklist, /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false); + + if (if_region->hasAttrOfType(kElseFuncNameAttr) && + !if_region.get_elseFuncNameAttr().getValue().empty()) { + else_name = + mapper.GetUniqueName(if_region.get_elseFuncNameAttr().getValue()) + .str(); + } else { + else_name = GetName(if_region, "_else"); + } + else_name = ExtractSingleBlockRegion( + symbol_table, if_region.getElseBranch(), else_name, extern_values, + worklist, /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false); + } + + // Look through ToBool operations for the condition. + Value cond = if_region.getCond(); + auto to_bool = dyn_cast_or_null(cond.getDefiningOp()); + if (to_bool) cond = to_bool.getOperand(); + + // Once we have the `then` and `else` functions ready (either outlined or + // existing ones), replace the region based op with a functional control flow + // op. + OpBuilder builder(if_region); + auto if_op = IfOp::create(builder, if_region.getLoc(), + if_region.getResultTypes(), cond, extern_values, + then_name, else_name, if_region.getIsStateless()); + CopyAndOverrideAttributes(if_region, if_op, &builder); + + if_region.replaceAllUsesWith(if_op.getResults()); + if_region.erase(); + + if (to_bool && to_bool.use_empty()) to_bool.erase(); + return success(); +} + +// Transform CaseRegion to Case. +LogicalResult RegionControlFlowToFunctional::ConvertCaseOp( + SymbolTableCollection& symbol_table, CaseRegionOp case_region) { + llvm::SmallVector branch_symbols; + + // Collect external values that are used within all branches. + llvm::SmallVector extern_values = + CollectExternValues(case_region.getRegions()); + + // These external values need to be added as inputs to the generated Case. The + // order is determined by the order of these values the `extern_vales`. + + for (const auto& item : llvm::enumerate(case_region.getRegions())) { + std::string branch_name = + GetName(case_region, llvm::formatv("_branch{0}", item.index()).str()); + branch_name = ExtractSingleBlockRegion(symbol_table, *item.value(), + branch_name, extern_values, worklist, + /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false); + branch_symbols.push_back( + SymbolRefAttr::get(case_region.getContext(), branch_name)); + } + + // Once we have the branch functions ready, replace the region based op with a + // functional op. + OpBuilder builder(case_region); + auto case_op = CaseOp::create( + builder, case_region.getLoc(), case_region.getResultTypes(), + case_region.getBranchIndex(), extern_values, + builder.getArrayAttr(branch_symbols), case_region.getIsStateless()); + CopyAndOverrideAttributes(case_region, case_op, &builder); + + // Redirect old results to new results. + case_region.replaceAllUsesWith(case_op.getResults()); + case_region.erase(); + return success(); +} + +// Transform WhileRegion to WhileOp. +LogicalResult RegionControlFlowToFunctional::ConvertWhileOp( + SymbolTableCollection& symbol_table, WhileRegionOp while_region) { + // For While, the arguments of the calls in the body and cond regions match + // if they are region arguments with the same region argument numbers. If the + // 2 calls have the same value (an extern value) used as an argument, we + // cannot do a trivial transformation because post transform, we will need to + // pass this extern value as an argument to the function, so we cannot use the + // existing function as is. + auto while_arg_matcher = [](Value first, Region& first_region, Value second, + Region& second_region) { + if (!mlir::isa(first) || !mlir::isa(second)) + return false; + BlockArgument first_block_arg = mlir::cast(first); + BlockArgument second_block_arg = mlir::cast(second); + + // 2 block arguments will match if they are the same argument number, and + // are block arguments of the corresponding containing regions. + return first_block_arg.getArgNumber() == second_block_arg.getArgNumber() && + first_block_arg.getParentBlock() == &first_region.front() && + second_block_arg.getParentBlock() == &second_region.front(); + }; + + const TrivialTransformInfo tti( + IsSingleCallRegion(while_region.getCond(), /*allow_to_bool=*/true), + IsSingleCallRegion(while_region.getBody()), while_arg_matcher); + + // All existing inputs to while region are inputs to the functional while. + auto new_inputs = llvm::to_vector<4>(while_region.getOperands()); + + // All existing results will also be generated by the functional while. + auto new_result_types = llvm::to_vector<4>(while_region.getResultTypes()); + + std::string cond_name, body_name; + if (tti.can_transform) { + // We can transform to functional form trivially without outlining. + cond_name = tti.callee_names[0].str(); + body_name = tti.callee_names[1].str(); + } else { + // The WhileRegion regions can refer to either arguments of the region, or + // external values implicitly captured by the region. When converting to + // functional form, all such external values need to become function + // arguments of the outlined functions, and become pass through values in + // the outlined body function. So when outlining the while body, in addition + // to the region arguments, all these external references need to be added + // as function arguments. + llvm::SmallVector extern_values = + CollectExternValues(while_region.getRegions()); + + // Outline the `cond` and `body` regions by moving the bodies of these + // regions into new functions. Replace tf.yield with a regular return. + cond_name = GetName(while_region, "_cond"); + cond_name = ExtractSingleBlockRegion(symbol_table, while_region.getCond(), + cond_name, extern_values, worklist, + /*extern_values_passthrough=*/false, + /*only_one_return_value=*/true); + + body_name = GetName(while_region, "_body"); + body_name = ExtractSingleBlockRegion(symbol_table, while_region.getBody(), + body_name, extern_values, worklist, + /*extern_values_passthrough=*/true, + /*only_one_return_value=*/false); + + // All extern values become additional inputs and additional output types + // for the functional while. + new_inputs.append(extern_values.begin(), extern_values.end()); + for (auto ext : extern_values) new_result_types.push_back(ext.getType()); + } + + // Once we have the `cond` and `body` functions ready (either outlined or + // existing ones), replace the region based op with a functional op. + OpBuilder builder(while_region); + auto while_op = WhileOp::create( + builder, while_region.getLoc(), new_result_types, new_inputs, cond_name, + body_name, while_region.getParallelIterations(), + while_region.getIsStateless(), while_region.getShapeInvariant()); + CopyAndOverrideAttributes(while_region, while_op, &builder); + + // Redirect old results to new results. + for (auto it : llvm::zip( + while_region.getResults(), + while_op.getResults().take_front(while_region.getNumResults()))) + std::get<0>(it).replaceAllUsesWith(std::get<1>(it)); + + while_region.erase(); + return success(); +} + +// Transform GeneratorDatasetRegion to GeneratorDatasetOp. +LogicalResult RegionControlFlowToFunctional::ConvertGeneratorDatasetOp( + SymbolTableCollection& symbol_table, GeneratorDatasetRegionOp regional) { + mlir::MLIRContext* ctx = regional.getContext(); + std::string init_name, next_name, finalize_name; + + llvm::SmallVector extern_values = + CollectExternValues(regional.getRegions()); + + if (!extern_values.empty()) return failure(); + + init_name = GetName(regional, "_init"); + init_name = ExtractSingleBlockRegion(symbol_table, regional.getInit(), + init_name, extern_values, worklist, + /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false, + /*allow_return_of_existing=*/true); + + next_name = GetName(regional, "_next"); + next_name = ExtractSingleBlockRegion(symbol_table, regional.getNext(), + next_name, extern_values, worklist, + /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false, + /*allow_return_of_existing=*/true); + + finalize_name = GetName(regional, "_finalize"); + finalize_name = + ExtractSingleBlockRegion(symbol_table, regional.getFinalize(), + finalize_name, extern_values, worklist, + /*extern_values_passthrough=*/false, + /*only_one_return_value=*/false, + /*allow_return_of_existing=*/true); + + OpBuilder builder(regional); + auto new_op = TF::GeneratorDatasetOp::create( + builder, regional.getLoc(), regional->getResultTypes(), + regional.getInitFuncOtherArgs(), regional.getNextFuncOtherArgs(), + regional.getFinalizeFuncOtherArgs(), SymbolRefAttr::get(ctx, init_name), + SymbolRefAttr::get(ctx, next_name), + SymbolRefAttr::get(ctx, finalize_name), regional.getOutputTypes(), + regional.getOutputShapes(), regional.getMetadata()); + + regional->replaceAllUsesWith(new_op->getResults()); + regional->erase(); + return success(); +} + +void RegionControlFlowToFunctional::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTableCollection symbol_table; + + // Seed worklist with all functions in the module. + worklist = llvm::to_vector<4>(module.getOps()); + while (!worklist.empty()) { + func::FuncOp function = worklist.pop_back_val(); + + auto result = function.walk([&](Operation* op) { + if (auto if_region = llvm::dyn_cast(op)) { + if (failed(ConvertIfOp(symbol_table, if_region))) { + op->emitOpError() << "failed to convert to functional form"; + return WalkResult::interrupt(); + } + } else if (auto case_region = llvm::dyn_cast(op)) { + if (failed(ConvertCaseOp(symbol_table, case_region))) { + op->emitOpError() << "failed to convert to functional form"; + return WalkResult::interrupt(); + } + } else if (auto while_region = llvm::dyn_cast(op)) { + if (failed(ConvertWhileOp(symbol_table, while_region))) { + op->emitOpError() << "failed to convert to functional form"; + return WalkResult::interrupt(); + } + } else if (auto gen = llvm::dyn_cast(op)) { + if (failed(ConvertGeneratorDatasetOp(symbol_table, gen))) { + op->emitOpError() << "failed to convert to functional form"; + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) return signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> +CreateTFRegionControlFlowToFunctional() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_arguments.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_arguments.cc new file mode 100644 index 0000000000000000000000000000000000000000..1d1038b430e4cff7db0c7f26f4003595270598c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_arguments.cc @@ -0,0 +1,247 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/ControlFlowInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/FunctionInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_REMOVEUNUSEDARGUMENTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Removes unused arguments from functions and their callers. +struct RemoveUnusedArgumentsPass + : public impl::RemoveUnusedArgumentsPassBase { + void runOnOperation() override; +}; + +// Return a bitvector that marks all return values that always come from +// the same value. +llvm::BitVector GetInvariantReturns(Region* region, int number_of_results) { + llvm::BitVector invariant(number_of_results, true); + std::vector used_argument(number_of_results, nullptr); + for (Block& block : region->getBlocks()) { + for (Operation& op : block.getOperations()) { + if (!op.hasTrait()) continue; + for (OpOperand& operand : op.getOpOperands()) { + int i = operand.getOperandNumber(); + if (used_argument[i] && operand.get() != used_argument[i]) { + assert(i < number_of_results); + invariant.reset(i); + } + used_argument[i] = operand.get(); + } + } + } + return invariant; +} + +Operation* GetAnyReturn(Region* region) { + Operation* result; + // We only go one level deep, since "returns" in nested functions + // and return-as-yield CF don't belong to us. + for (Block& block : region->getBlocks()) { + for (Operation& op : block.getOperations()) { + if (op.hasTrait()) result = &op; + } + } + return result; +} + +void EraseReturnOperands(Region* region, llvm::BitVector& erase) { + for (Block& block : region->getBlocks()) { + for (Operation& op : block.getOperations()) { + if (!op.hasTrait()) continue; + op.eraseOperands(erase); + } + } +} + +// Erases the given results from an operation, similar to what +// Operation::eraseArguments does (but for results). +// This is a lengthy bit of code, since it has to recreate the operation. +// TODO(kramm): Move this under utils/ somewhere. +void EraseResults(Operation* op, llvm::BitVector erase) { + assert(!op->getNumRegions()); + std::vector result_types; + for (auto result : op->getResults()) { + if (!erase[result.getResultNumber()]) { + result_types.push_back(result.getType()); + } + } + OpBuilder builder(op); + builder.setInsertionPointAfter(op); + OperationState state(op->getLoc(), op->getName().getStringRef(), + op->getOperands(), result_types, op->getAttrs()); + for (int i = 0; i < op->getNumRegions(); ++i) { + state.addRegion(); + } + Operation* new_op = builder.create(state); + for (const auto& indexed_regions : llvm::enumerate(op->getRegions())) { + Region& region = op->getRegion(indexed_regions.index()); + IRMapping mapping; + indexed_regions.value().cloneInto(®ion, mapping); + } + int new_position = 0; + for (auto result : op->getResults()) { + if (!erase[result.getResultNumber()]) { + result.replaceAllUsesWith(new_op->getResult(new_position++)); + } + } + op->erase(); +} + +void RemoveUnusedArgumentsPass::runOnOperation() { + Operation* module = getOperation(); + + llvm::DenseMap args_to_erase; + llvm::DenseMap results_to_erase; + llvm::DenseMap> args_to_remap; + llvm::DenseSet do_not_touch; // Funcs referenced by non-call ops + + // Find all users of functions that are not through a CallOp. Those + // are functions we need to leave alone. + module->walk([&](SymbolUserOpInterface op) { + if (llvm::isa(op.getOperation())) return; + // SymbolUserOpInterface doesn't tell us which attributes contain + // the symbols, so we have to scan through all of them. + for (auto attr : op->getAttrs()) { + if (auto sym = mlir::dyn_cast(attr.getValue())) { + Operation* func = mlir::SymbolTable::lookupNearestSymbolFrom(op, sym); + if (func) { + do_not_touch.insert(func); + } + } + } + }); + + // Find all functions + module->walk([&](SymbolOpInterface op) { + if (!op.isPrivate()) return; + + auto call = llvm::dyn_cast(op.getOperation()); + if (!call) return; + Region* region = call.getCallableRegion(); + if (!region) return; // happens e.g. for external functions + + auto func = llvm::dyn_cast(op.getOperation()); + if (!func || do_not_touch.count(func)) return; + llvm::BitVector unused_args(func.getNumArguments()); + llvm::BitVector unused_results(func.getNumResults()); + llvm::DenseMap return_to_operand; + llvm::BitVector invariant_returns = + GetInvariantReturns(region, func.getNumResults()); + llvm::DenseMap argument_to_index; + std::vector use_count(func.getNumArguments(), 0); + + // Set up a use count for all function arguments. We'll use this to + // determine whether we have taken care of all uses and can remove + // the arg. + for (BlockArgument arg : func.getArguments()) { + auto uses = arg.getUses(); + use_count[arg.getArgNumber()] = std::distance(uses.begin(), uses.end()); + argument_to_index.insert({arg, arg.getArgNumber()}); + } + + // We're only considering return values that are the same (at this + // position) across all returns, so we only need any single return + // as reference. + Operation* ret = GetAnyReturn(region); + + // Go through all return values and find the ones we can remove. + for (mlir::OpOperand& r : ret->getOpOperands()) { + int i = r.getOperandNumber(); + if (!invariant_returns[i]) continue; + if (!argument_to_index.count(r.get())) continue; + + int arg = argument_to_index.lookup(r.get()); + + // If we see an arg as a result, and we're going + // to remove that result, we can assume one fewer use. + use_count[arg]--; + + return_to_operand.insert({i, arg}); + unused_results.set(i); + } + + // Now mark all args for deletion that don't have uses anymore. + for (BlockArgument arg : func.getArguments()) { + // TODO(b/246310765): This doesn't fully handle recursion. + if (!use_count[arg.getArgNumber()]) unused_args.set(arg.getArgNumber()); + } + + EraseReturnOperands(region, unused_results); + if (failed(func.eraseResults(unused_results))) return; + if (failed(func.eraseArguments(unused_args))) return; + + args_to_erase.insert(std::make_pair(op, unused_args)); + results_to_erase.insert(std::make_pair(op, unused_results)); + args_to_remap.insert(std::make_pair(op, return_to_operand)); + }); + + // Find all callers + module->walk([&](CallOpInterface op) { + auto callable = op.getCallableForCallee(); + mlir::SymbolRefAttr sym = callable.dyn_cast(); + if (!sym) return; + Operation* func = mlir::SymbolTable::lookupNearestSymbolFrom(op, sym); + if (!args_to_erase.count(func)) return; + + auto map = args_to_remap.lookup(func); + for (auto [from, to] : map) { + op.getOperation()->getResult(from).replaceAllUsesWith( + op.getOperation()->getOperand(to)); + } + BitVector operands_to_erase(op->getNumOperands()); + int args_start = op->getNumOperands() + ? op.getArgOperands().getBase()->getOperandNumber() + : 0; + operands_to_erase |= args_to_erase.lookup(func); + operands_to_erase <<= args_start; + op->eraseOperands(operands_to_erase); + + EraseResults(op, results_to_erase.lookup(func)); + }); +} + +} // namespace + +std::unique_ptr> CreateRemoveUnusedArgumentsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_while_results.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_while_results.cc new file mode 100644 index 0000000000000000000000000000000000000000..e62c7e203db7ad0cc2fa945c379105391e252fc4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_unused_while_results.cc @@ -0,0 +1,131 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/log/log.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_REMOVEUNUSEDWHILERESULTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Removes unused results and related ops from while loops. +struct RemoveUnusedWhileResultsPass + : public impl::RemoveUnusedWhileResultsPassBase< + RemoveUnusedWhileResultsPass> { + void runOnOperation() override; +}; + +// Prunes result defining op if possible, returns true if pruning was done. +bool TryPruneResultDefiningOp(TF::WhileRegionOp while_op, OpResult result) { + // Don't prune if result is used. + if (!result.use_empty()) return false; + + Block& body_block = while_op.getBody().front(); + Block& cond_block = while_op.getCond().front(); + Operation* body_yield_op = body_block.getTerminator(); + + // The body yield operand, body block argument, condition block argument, and + // result all correspond to each other (see definition of `WhileRegionOp`). + int idx = result.getResultNumber(); + Value body_yield_operand = body_yield_op->getOperand(idx); + Value body_block_argument = body_block.getArgument(idx); + Value cond_block_argument = cond_block.getArgument(idx); + // Consider the op that defines the unused result as a candidate for pruning. + Operation* candidate_op = body_yield_operand.getDefiningOp(); + if (candidate_op == nullptr) return false; + + // Don't prune if candidate op might have side effects. + if (isa_and_nonnull(candidate_op->getDialect())) { + if (TF::TensorFlowDialect::CanHaveSideEffects(candidate_op)) { + return false; + } + } else if (!isMemoryEffectFree(candidate_op)) { + return false; + } + + // Don't prune if the body block argument has any other user. + for (Operation* op : body_block_argument.getUsers()) { + if (op != candidate_op) return false; + } + + // Don't prune if the condition block argument has any user. + if (!cond_block_argument.use_empty()) return false; + + // Don't prune if `body_yield_operand` has more than one use (that would mean + // it feeds into another op apart from `Yield`). + if (!body_yield_operand.hasOneUse()) return false; + + // Don't prune if any other result of the candidate op is used. + for (Value candidate_result : candidate_op->getResults()) { + if (candidate_result == body_yield_operand) continue; + if (!candidate_result.use_empty()) return false; + } + + // Now we know that it is safe to erase the candidate op along with `result` + // and the corresponding operand. Here we only erase the op and replace its + // result usage with the corresponding block argument, the result and operand + // will be removed later in a canonicalization pattern. + VLOG(4) << "Pruning following op:\n" << debugString(*candidate_op); + body_yield_operand.replaceAllUsesWith(body_block_argument); + candidate_op->erase(); + return true; +} + +void RemoveUnusedWhileResultsPass::runOnOperation() { + func::FuncOp func = getOperation(); + // Try to prune defining ops of unused results. + func.walk([&](TF::WhileRegionOp while_op) { + for (OpResult result : while_op.getResults()) { + TryPruneResultDefiningOp(while_op, result); + } + }); + // Now eliminate passthrough operands/results with existing canonicalization + // pattern. + MLIRContext* context = &getContext(); + RewritePatternSet patterns(context); + TF::WhileRegionOp::getCanonicalizationPatterns(patterns, context); + if (failed(applyPatternsGreedily(func, std::move(patterns)))) { + signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> +CreateRemoveUnusedWhileResultsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_vars_in_session_initializer.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_vars_in_session_initializer.cc new file mode 100644 index 0000000000000000000000000000000000000000..8d20d7d5889853315ddb4522d890e92ce21116c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/remove_vars_in_session_initializer.cc @@ -0,0 +1,106 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +using mlir::TF::VarHandleOp; + +#define GEN_PASS_DEF_REMOVEVARIABLESINSESSIONINITIALIZERPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +class RemoveVariablesInSessionInitializerPass + : public impl::RemoveVariablesInSessionInitializerPassBase< + RemoveVariablesInSessionInitializerPass> { + public: + void runOnOperation() override; +}; + +void RecursiveRemove(Operation* op, + llvm::SmallVectorImpl& erase_list, + llvm::SmallPtrSetImpl& dead_ops) { + for (mlir::Value res : op->getResults()) { + for (Operation* user : res.getUsers()) { + if (!dead_ops.insert(user).second) continue; + RecursiveRemove(user, erase_list, dead_ops); + } + } + + erase_list.push_back(op); + + for (auto& use : op->getOpOperands()) { + if (auto op_result = mlir::dyn_cast(use.get())) { + Operation* def = op_result.getDefiningOp(); + if (!dead_ops.insert(def).second) continue; + RecursiveRemove(def, erase_list, dead_ops); + } + } +} + +void RemoveVariables(llvm::ArrayRef vars) { + // TODO(b/160906885): Repalce the following code with an non-recursive one. + llvm::SmallVector erase_list; + llvm::SmallPtrSet dead_ops; + + // Marks all the variables dead. + dead_ops.insert(vars.begin(), vars.end()); + + // Removes relevant ops in topological order. + for (auto& op : vars) RecursiveRemove(op, erase_list, dead_ops); + + // Erases the ops. + for (auto op : erase_list) op->erase(); +} + +void RemoveVariablesInSessionInitializerPass::runOnOperation() { + ModuleOp module_op = getOperation(); + + for (auto init_func_op : GetInitializerFunctions(module_op)) { + if (!init_func_op) return; + + if (init_func_op.getBlocks().size() != 1) { + init_func_op.emitError("expects exactly one block in the MLIR function"); + return signalPassFailure(); + } + + auto var_handle_ops = + init_func_op.getBlocks().front().getOps(); + llvm::SmallVector init_vars(var_handle_ops.begin(), + var_handle_ops.end()); + RemoveVariables(init_vars); + } +} + +} // namespace + +std::unique_ptr> +CreateRemoveVariablesInSessionInitializerPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replica_id_to_device_ordinal.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replica_id_to_device_ordinal.cc new file mode 100644 index 0000000000000000000000000000000000000000..403431e26b23e460a3f41675ed61fdee6160a2d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replica_id_to_device_ordinal.cc @@ -0,0 +1,109 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass sets the device ordinal attribute of the required op using +// the replica id attribute. + +#include + +#include "llvm/Support/Casting.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" + +namespace mlir { +namespace TFDevice { +namespace { +constexpr char kReplicaIdAttr[] = "_xla_replica_id"; +constexpr char kDeviceOrdinalAttr[] = "device_ordinal"; + +#define GEN_PASS_DEF_REPLICAIDTODEVICEORDINALPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ReplicaIDToDeviceOrdinalPass + : public impl::ReplicaIDToDeviceOrdinalPassBase< + ReplicaIDToDeviceOrdinalPass> { + void runOnOperation() override; +}; + +// Returns whether op requires `device_ordinal` attribute. +bool RequiresDeviceOrdinalAttribute(Operation* op) { + return (llvm::isa(op) && + op->hasAttr(kDeviceOrdinalAttr) && op->hasAttr(kReplicaIdAttr)); +} + +void ReplicaIDToDeviceOrdinalPass::runOnOperation() { + const Dialect* tf_dialect = getContext().getLoadedDialect("tf"); + if (!tf_dialect) { + getOperation().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); + } + + // Get the number of devices per host. + int device_num = 0; + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp( + getOperation()->getParentOfType(), &devices))) + return signalPassFailure(); + for (const auto& device_name : devices.device_names()) { + if (device_name.has_type && device_name.type == "TPU") ++device_num; + } + + if (device_num == 0) return; + + llvm::SmallVector require_device_ordinal_ops; + getOperation().walk([&](Operation* op) { + if (RequiresDeviceOrdinalAttribute(op)) { + require_device_ordinal_ops.push_back(op); + } + }); + + if (require_device_ordinal_ops.size() == 1) { + // If there is only one op which requires the device ordinal being set, + // set the device ordinal to 0. Note: This is for single device use case + // (eg. pf megacore) for which `_xla_replica_id` isn't set via the + // replicate_to_islands pass. + Operation* op = require_device_ordinal_ops.front(); + if (op->getAttrOfType(kDeviceOrdinalAttr).getInt() == -1) { + OpBuilder builder(op); + op->setAttr(kDeviceOrdinalAttr, builder.getI64IntegerAttr(0)); + } + } else { + // If the device ordinal attribute is -1, set it with the replica id + // attribute modulo the number of TPU cores in the system. + for (auto op : require_device_ordinal_ops) { + if (op->getAttrOfType(kDeviceOrdinalAttr).getInt() == -1) { + OpBuilder builder(op); + int device_ordinal = + op->getAttrOfType(kReplicaIdAttr).getInt() % + device_num; + op->setAttr(kDeviceOrdinalAttr, + builder.getI64IntegerAttr(device_ordinal)); + } + } + } +} +} // namespace + +std::unique_ptr> +CreateReplicaIDToDeviceOrdinalPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_invariant_op_hoisting.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_invariant_op_hoisting.cc new file mode 100644 index 0000000000000000000000000000000000000000..5345afeb0c23eaaad51d8cad0f213030423cb1ea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_invariant_op_hoisting.cc @@ -0,0 +1,193 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass hoists replicate invariant ops, or ops that yield the same +// result(s) regardless of replication, out of their respective replicate. + +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_REPLICATEINVARIANTOPHOISTINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ReplicateInvariantOpHoistingPass + : public impl::ReplicateInvariantOpHoistingPassBase< + ReplicateInvariantOpHoistingPass> { + void runOnOperation() override; +}; + +// Check if op directly uses a key in `virtual_devices`. +bool DirectUseOfVirtualDevice(const DictionaryAttr& virtual_devices, + Operation* op) { + StringAttr op_device = op->getAttrOfType(kDeviceAttr); + if (!op_device) return false; + if (virtual_devices.get(op_device.getValue())) return true; + return false; +} + +// Check if op or its ancestor uses a key in `virtual_devices`. +bool AncestorUsesVirtualDevice( + const std::optional& virtual_devices, Operation* op) { + if (!virtual_devices.has_value()) return false; + if (!op) return false; + if (llvm::isa(op)) return false; + if (DirectUseOfVirtualDevice(*virtual_devices, op)) return true; + return AncestorUsesVirtualDevice(virtual_devices, op->getParentOp()); +} + +// Check if op or its descendant uses a key in `virtual_devices`. +bool DescendantUsesVirtualDevice( + const std::optional& virtual_devices, + Operation* operation) { + if (!virtual_devices.has_value()) return false; + + auto result = operation->walk([&](Operation* op) { + if (DirectUseOfVirtualDevice(*virtual_devices, op)) + return WalkResult::interrupt(); + return WalkResult::advance(); + }); + return result.wasInterrupted(); +} + +// Make invariant the `ShapeOp`s or a `ReadVariableOp` that's the `ShapeOp`'s +// predecessor. +void MakeShapeOpInvariant(tf_device::ReplicateOp replicate_op, int num_replicas, + Block* replicate_block, TF::ShapeOp shape_op) { + // Ignore ShapeOps that have virtual devices. + if (AncestorUsesVirtualDevice(replicate_op.getDevices(), shape_op)) return; + + Value input = shape_op.getInput(); + // If ShapeOp operand is replicate tensor block argument, replace with the + // associated first replica operand. + if (auto block_arg = mlir::dyn_cast(input)) { + if (block_arg.getOwner() != replicate_block) return; + + shape_op.setOperand(replicate_op.GetReplicaOperandForBlockArgument( + block_arg, /*replica=*/0)); + + return; + } + + Operation* input_def = input.getDefiningOp(); + + // If ShapeOp operand is a ReadVariableOp result where the ReadVariableOp + // operand is a replicate resource block argument, replace ShapeOp with + // VariableShapeOp and use the associated first replica operand as its + // operand. + auto read_var_op = llvm::dyn_cast(input_def); + if (!read_var_op) return; + + // TODO(lyandy): Check if resource (first replica or replicate block arg) + // shape has not changed in replicate prior to read. Currently after both + // ResourceOpLiftingPass and TPURewritePass, there should not be any updates + // to resources prior to their respective ReadVariableOp. + if (auto block_arg = + mlir::dyn_cast(read_var_op.getResource())) { + if (block_arg.getOwner() != replicate_block) return; + + OpBuilder builder(shape_op); + auto new_shape_op = TF::VariableShapeOp::create( + builder, shape_op.getLoc(), shape_op.getType(), + replicate_op.GetReplicaOperandForBlockArgument(block_arg, + /*replica=*/0)); + shape_op.replaceAllUsesWith(new_shape_op.getOperation()); + shape_op.erase(); + } +} + +// Checks if op and inner op operands are all replicate invariant. +bool IsOpReplicateInvariant(Region* replicate_region, Operation* op) { + auto ancestor_of_replicate = [&](Region* region) { + return region && region->isProperAncestor(replicate_region); + }; + + for (Value operand : op->getOperands()) + if (!ancestor_of_replicate(operand.getParentRegion())) return false; + + // _TPUDeviceOrdinalPlaceholder implicitly depends on the replica. + if (llvm::isa(op)) return false; + + bool has_replicate_operands = false; + visitUsedValuesDefinedAbove(op->getRegions(), [&](OpOperand* operand) { + if (!ancestor_of_replicate(operand->get().getParentRegion())) + has_replicate_operands = true; + }); + + return !has_replicate_operands; +} + +// Hoists replicate invariant ops out of associated `tf_device.replicate` op. +// Ops to be hoisted are determined by if all of their operands are replicate +// invariant. Shape ops are rewritten to be invariant when possible, prior to +// hoisting ops. +void HoistReplicateInvariantOps(tf_device::ReplicateOp replicate_op) { + const int num_replicas = replicate_op.getN(); + Block* replicate_block = &replicate_op.GetBody(); + + // A `ShapeOp` that directly depends on a `tf_device.replicate` param and does + // not have a virtual device is assumed to return the same shape across all + // replicas. Thus it is invariant across replicas. + // TODO(b/277936694): Remove this assumption and special case. + replicate_op.walk([&](TF::ShapeOp shape_op) { + MakeShapeOpInvariant(replicate_op, num_replicas, replicate_block, shape_op); + }); + + Region* replicate_region = &replicate_op.getBody(); + std::optional virtual_device_list = replicate_op.getDevices(); + for (Operation& inner_op : + llvm::make_early_inc_range(replicate_op.GetBody())) { + if (llvm::isa(inner_op)) continue; + // Skip hoisting if the inner op device attribute is a virtual device + // defined by tf_device.replicate. + if (DescendantUsesVirtualDevice(virtual_device_list, &inner_op)) continue; + + if (IsOpReplicateInvariant(replicate_region, &inner_op)) + inner_op.moveBefore(replicate_op); + } +} + +void ReplicateInvariantOpHoistingPass::runOnOperation() { + getOperation().walk( + [](tf_device::ReplicateOp op) { HoistReplicateInvariantOps(op); }); +} +} // anonymous namespace + +std::unique_ptr> +CreateReplicateInvariantOpHoistingPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_tensor_list_init_ops_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_tensor_list_init_ops_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..f4dc012057a57fd9428ab0a2fc875c9a95d86a86 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_tensor_list_init_ops_pass.cc @@ -0,0 +1,79 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_REPLICATETENSORLISTINITOPSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Replicates the TensorList initialization ops for all the uses. +// No need to delete the original TensorList as it might be used elsewhere. +template +void ReplicateTensorListForUses(T tensor_list_op) { + Value tensor_list = tensor_list_op.getResult(); + std::vector uses; + for (auto& use : tensor_list.getUses()) { + uses.emplace_back(&use); + } + OpBuilder builder(tensor_list_op.getOperation()); + for (OpOperand* operand : uses) { + auto new_op = builder.clone(*tensor_list_op.getOperation()); + operand->set(new_op->getResult(0)); + } +} + +// This transformation pass replicates TensorList initialization ops. +class ReplicateTensorListInitOps + : public impl::ReplicateTensorListInitOpsPassBase< + ReplicateTensorListInitOps> { + public: + void runOnOperation() override { + getOperation().walk([](Operation* op) { + if (auto tl_reserve = dyn_cast(op)) { + ReplicateTensorListForUses(tl_reserve); + } + if (auto tl_empty = dyn_cast(op)) { + ReplicateTensorListForUses(tl_empty); + } + }); + } +}; +} // namespace + +std::unique_ptr> +CreateReplicateTensorListInitOpsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc new file mode 100644 index 0000000000000000000000000000000000000000..25bd75368bf8590927e8935e14d4a3424ce003e8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc @@ -0,0 +1,403 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass forms `tf_executor.island` per replica from a single +// `tf_device.replicate` island. + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h" + +namespace mlir { +namespace TFDevice { +namespace { +constexpr char kDeviceAttr[] = "device"; +constexpr char kReplicaIdAttr[] = "_xla_replica_id"; +constexpr char kDeviceOrdinalAttr[] = "device_ordinal"; + +#define GEN_PASS_DEF_REPLICATETOISLANDPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ReplicateToIslandPass + : public impl::ReplicateToIslandPassBase { + explicit ReplicateToIslandPass(bool legacy_graph_export) { + legacy_graph_export_ = legacy_graph_export; + } + void runOnOperation() override; +}; + +// Returns whether op requires `_xla_replica_id` attribute. +bool RequiresReplicaIDAttribute(Operation* op) { + return llvm::isa(op); +} + +// Returns the device ordinal (`device_ordinal`) for a replica (`replica_id`) +// and logical core (`logical_core`). +// `replica_id` is the index of the ancestor ReplicateOp in [0, num_replicas). +// `logical_core` is the index of the TPU core in [0, num_cores_per_replica). +// `device_ordinal` is the index of the TPU core relative to its host. +LogicalResult GetDeviceOrdinal(const std::optional& devices, + const unsigned replica_id, + const uint64_t logical_core, + int64_t& device_ordinal, Operation* op) { + if (!devices.has_value()) { + return op->emitOpError() + << "devices attribute is not present in 'tf.device.replicate' op"; + } + auto logical_core_name = + tensorflow::GetDeviceAliasForLogicalCore(logical_core); + auto tpu_replica = devices.value().get(logical_core_name); + if (!tpu_replica) { + return op->emitOpError() + << "requires device ordinal from device " << logical_core_name + << " to be present in 'tf.device.replicate' op"; + } + llvm::StringRef tpu_device = + llvm::cast(llvm::cast(tpu_replica)[replica_id]) + .getValue(); + return tensorflow::GetDeviceOrdinalFromDeviceString(op->getLoc(), tpu_device, + &device_ordinal); +} + +// Updates replica variant ops in a region based on replica `replica_id`. +// TODO(b/157624749): Replace this with better abstraction to differentiate ops +// for different replicas. Some ops, such as XlaHostCompute op or TPU Embedding +// ops, require replica id to be added as an op attribute to be used during +// execution. Handle such ops separately and add an integer attribute that +// represents replica id. +LogicalResult UpdateRegionReplicateVariantOps( + OpBuilder& builder, Location loc, Region& region, int replica_id, + const std::optional& devices) { + auto result = region.walk([&](Operation* op) -> WalkResult { + if (RequiresReplicaIDAttribute(op)) { + op->setAttr(kReplicaIdAttr, builder.getI64IntegerAttr(replica_id)); + return WalkResult::advance(); + } + + if (auto placeholder = dyn_cast(op)) { + int64_t device_ordinal; + if (failed(GetDeviceOrdinal(devices, replica_id, + placeholder.getLogicalCore(), device_ordinal, + op))) + return failure(); + + OpBuilder builder(op); + auto const_op = TF::ConstOp::create( + builder, op->getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get({}, builder.getI64Type()), + {device_ordinal})); + op->replaceAllUsesWith(const_op); + op->erase(); + return WalkResult::advance(); + } + + if (!devices.has_value()) return WalkResult::advance(); + + // Map aliased devices to explicit devices based on replica. + if (auto launch = dyn_cast(op)) + if (auto device_by_replica = devices.value().get(launch.getDevice())) + launch->setAttr(kDeviceAttr, + llvm::cast(llvm::cast( + device_by_replica)[replica_id])); + + return WalkResult::advance(); + }); + + return failure(result.wasInterrupted()); +} + +// Creates islands per replica from `tf_device.replicate` region. If for a +// `tf_device.launch` op the device is an aliased device of the +// `tf_device.replicate`, the device will be remapped to an explicit device +// for the associated replica island. +LogicalResult ExpandReplicateIntoReplicas( + const Dialect* tf_dialect, OpBuilder& builder, + tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op, + int num_replicas, llvm::SmallVectorImpl& replicas, + bool legacy_graph_export, int replica_group_idx) { + replicas.reserve(num_replicas); + auto devices = replicate_op.getDevices(); + + // Collect result types and operands. + Operation& terminator = replicate_op.GetBody().back(); + llvm::SmallVector output_types(terminator.getOperandTypes()); + auto control_type = tf_executor::ControlType::get(island_op.getContext()); + llvm::SmallVector replica_inputs(island_op.getControlInputs()); + + // Replace replicate terminator with YieldOp. + builder.setInsertionPoint(&terminator); + tf_executor::YieldOp::create(builder, terminator.getLoc(), + terminator.getOperands()); + terminator.erase(); + + builder.setInsertionPoint(island_op); + IRMapping mapping; + for (int i : llvm::seq(0, num_replicas)) { + // Create new island for replica. + auto replica = + tf_executor::IslandOp::create(builder, island_op.getLoc(), output_types, + control_type, replica_inputs); + + // Map block arg to replica arg. + mapping.clear(); + for (auto& block_arg : replicate_op.GetBody().getArguments()) + mapping.map(block_arg, + replicate_op.GetReplicaOperandForBlockArgument(block_arg, i)); + + // Copy over replicate region into replica island. + replicate_op.getBody().cloneInto(&replica.getBody(), mapping); + + if (failed(UpdateRegionReplicateVariantOps(builder, replicate_op.getLoc(), + replica.getBody(), + /*replica_id=*/i, devices))) + return failure(); + + // In new graph export pipeline, we will update control dependencies in the + // end of the pipeline. Mostly, it will rely on side effect analysis by + // considering accessing resource only. However, for branches under parallel + // group, there should not be any control deps between them even side effect + // analysis indicate some control deps. Therefore, we will mark parallel + // group and branch information here so that `UpdateControlDependenciesPass` + // can fetch the related information later. + if (!legacy_graph_export) { + std::string group_annotation = absl::StrCat( + "r", std::to_string(replica_group_idx), ":", std::to_string(i)); + if (auto parallel_group_attr = replicate_op->getAttrOfType( + TF::kParallelExecAnnotation)) { + // Extend the existing attribute so that nested parallel execution + // structure is supported. + group_annotation = absl::StrCat(parallel_group_attr.getValue().str(), + ",", group_annotation); + } + for (auto& op : replica.GetBody()) { + op.setAttr(TF::kParallelExecAnnotation, + builder.getStringAttr(group_annotation)); + } + } + replicas.push_back(replica); + } + + return success(); +} + +// Creates islands per replica from `tf_device.replicate` region and remap +// replicate results with new island outputs. A single island is created to +// forward control dependencies if there is a control dependency output from the +// replicate island. Devices are remapped from aliased devices to explicit +// devices, for `tf_device.launch` ops. +// +// For example, the following: +// +// %0:2 = tf_executor.island(%control) { +// %1:4 = tf_device.replicate([%arg0, %arg1] as %ri: tensor) +// {n = 2 : i32, +// devices = {DEVICE_ALIAS_0 = ["/DEVICE:0", "/DEVICE:1"], +// DEVICE_ALIAS_1 = ["/DEVICE:2", "/DEVICE:3"]}} { +// %a = "tf_device.launch"() ({ +// %2 = "tf.opA"(%ri) : (tensor) -> tensor +// tf_device.return %2 : tensor +// }) {device = "DEVICE_ALIAS_0"} : () -> tensor +// %b = "tf_device.launch"() ({ +// %3 = "tf.opB"(%a) : (tensor) -> tensor +// tf_device.return %3 : tensor +// }) {device = "DEVICE_ALIAS_1"} : () -> tensor +// tf_device.return %a, %b : tensor, tensor +// } +// tf_executor.yield %1#0 : tensor +// } +// +// gets lowered to: +// +// %0:3 = tf_executor.island(%control) { +// %a0 = "tf_device.launch"() ({ +// %1 = "tf.opA"(%arg0) : (tensor) -> tensor +// tf_device.return %1 : tensor +// }) {device = "/DEVICE:0"} : () -> tensor +// %b0 = "tf_device.launch"() ({ +// %2 = "tf.opB"(%a0) : (tensor) -> tensor +// tf_device.return %2 : tensor +// }) {device = "/DEVICE:2"} : () -> tensor +// tf_executor.yield %a0, %b0 : tensor, tensor +// } +// %3:3 = tf_executor.island(%control) { +// %a1 = "tf_device.launch"() ({ +// %4 = "tf.opA"(%arg1) : (tensor) -> tensor +// tf_device.return %4 : tensor +// }) {device = "/DEVICE:1"} : () -> tensor +// %b1 = "tf_device.launch"() ({ +// %5 = "tf.opB"(%a1) : (tensor) -> tensor +// tf_device.return %5 : tensor +// }) {device = "/DEVICE:3"} : () -> tensor +// tf_executor.yield %a1, %b1 : tensor, tensor +// } +LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect, + tf_executor::GraphOp graph_op, + tf_executor::IslandOp island_op, + tf_device::ReplicateOp replicate_op, + bool legacy_graph_export, + int replica_group_idx) { + OpBuilder builder(island_op); + const int num_replicas = replicate_op.getN(); + + // Create islands per replica. + llvm::SmallVector replicas; + if (failed(ExpandReplicateIntoReplicas( + tf_dialect, builder, island_op, replicate_op, num_replicas, replicas, + legacy_graph_export, replica_group_idx))) + return failure(); + + // Collect all replica results. + llvm::SmallVector replicas_outputs(replicate_op.getNumResults(), + nullptr); + for (auto replica_and_idx : llvm::enumerate(replicas)) + for (auto replica_result_and_idx : + llvm::enumerate(replica_and_idx.value().getOutputs())) + replicas_outputs[num_replicas * replica_result_and_idx.index() + + replica_and_idx.index()] = + replica_result_and_idx.value(); + + // Remap replicate results to per replica result. + for (auto result : llvm::zip(island_op.getOutputs(), replicas_outputs)) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + // Add sink island to pin all replicas as a control dependency if there is a + // control dependency leading from the replicate originally. + if (!island_op.getControl().use_empty()) { + llvm::SmallVector island_operands; + for (auto& replica : replicas) + island_operands.push_back(replica.getControl()); + + builder.setInsertionPoint(island_op); + auto island_sink = tf_executor::IslandOp::create( + builder, island_op.getLoc(), llvm::ArrayRef{}, + tf_executor::ControlType::get(island_op.getContext()), island_operands); + island_sink.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&island_sink.GetBody()); + tf_executor::YieldOp::create(builder, island_op.getLoc(), + llvm::ArrayRef{}); + island_op.getControl().replaceAllUsesWith(island_sink.getControl()); + } + + if (legacy_graph_export) { + // Replicas with no uses should be pinned to a graph fetch so they still + // execute. + llvm::SmallVector unused_replica_controls; + for (auto& replica : replicas) + if (replica.use_empty()) + unused_replica_controls.push_back(replica.getControl()); + + if (!unused_replica_controls.empty()) { + tf_executor::FetchOp fetch = graph_op.GetFetch(); + auto fetches = llvm::to_vector<8>(fetch.getOperands()); + fetches.append(unused_replica_controls.begin(), + unused_replica_controls.end()); + builder.setInsertionPoint(fetch); + tf_executor::FetchOp::create(builder, fetch.getLoc(), fetches); + fetch.erase(); + } + } else { + // Now, finally, we need to maintain the invariant expected to be maintained + // throughout the graph export pipeline that all islands always perfectly + // wrap a single op. So we'll split all replica islands. + auto control_type = tf_executor::ControlType::get(island_op.getContext()); + for (auto& replica : replicas) { + if (replica.GetBody().getOperations().size() > 1) { + mlir::TF::SplitIsland(replica, control_type); + } + } + } + + island_op.erase(); + return success(); +} + +void ReplicateToIslandPass::runOnOperation() { + const Dialect* tf_dialect = getContext().getLoadedDialect("tf"); + if (!tf_dialect) { + getOperation().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); + } + + // Find islands with a single `tf_device.replicate` and create individual + // islands per replica of the replicate. + llvm::SmallVector replicate_op_islands; + getOperation().walk([&](tf_executor::GraphOp graph_op) { + for (auto island_op : graph_op.getOps()) { + if (!island_op.WrapsSingleOp()) continue; + + if (isa(&island_op.GetBody().front())) + replicate_op_islands.push_back(island_op); + } + }); + + // This number is unique within each function which is sufficient for + // `UpdateControlDependenciesPass` which consumes the related attributes. + // However, this assumes that we don't inline functions between this pass + // and `UpdateControlDependenciesPass`. + // If we need globally unique replica group IDs in the future, + // we can either make this pass a module pass (using a global counter) + // or use an atomic counter. + int replica_group_idx = 0; + for (tf_executor::IslandOp island_op : replicate_op_islands) { + auto graph_op = island_op->getParentOfType(); + auto replicate_op = + cast(island_op.GetBody().front()); + if (failed(CreateIslandsFromReplicate(tf_dialect, graph_op, island_op, + replicate_op, legacy_graph_export_, + replica_group_idx))) { + return signalPassFailure(); + } + replica_group_idx++; + } +} +} // anonymous namespace + +std::unique_ptr> CreateReplicateToIslandPass( + bool legacy_graph_export) { + return std::make_unique(legacy_graph_export); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_analyzer_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_analyzer_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..d1ac9c4918879d296f6a7f7a5bafe833e0b5c108 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_analyzer_test_pass.cc @@ -0,0 +1,88 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "llvm/Support/FormatVariadic.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_value_typed_analyzer.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" + +namespace mlir { +namespace tf_test { +namespace { + +#define GEN_PASS_DEF_RESOURCEANALYZERTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +class ResourceAnalyzerTestPass + : public impl::ResourceAnalyzerTestPassBase { + public: + void runOnOperation() override; +}; + +// A set of values that identifies a resource. +struct ResourceKey { + StringRef device; + StringRef container; + StringRef shared_name; +}; + +ResourceKey GetResourceKey(TF::VarHandleOp var_handle_op) { + ResourceKey resource_key; + + if (auto attr = var_handle_op->getAttrOfType("device")) { + resource_key.device = attr.getValue(); + } + + resource_key.container = var_handle_op.getContainer(); + resource_key.shared_name = var_handle_op.getSharedName(); + + return resource_key; +} + +// Prints the analysis result for each resource ops found in `module_op` for +// testing purposes. +void PrintAnalysisResults(const TF::ResourceAnalyzer& analyzer, + ModuleOp module_op) { + module_op.getRegion().walk([&analyzer](Operation* op) { + if (auto var_handle_op = dyn_cast(op)) { + const ResourceKey resource_key = GetResourceKey(var_handle_op); + op->emitRemark(llvm::formatv( + "device: \"{0}\", container: \"{1}\", shared_name: \"{2}\", " + "is_potentially_written: {3}", + resource_key.device, resource_key.container, resource_key.shared_name, + analyzer.IsPotentiallyWritten(var_handle_op.getResource()))); + } + }); +} + +void ResourceAnalyzerTestPass::runOnOperation() { + ModuleOp module_op = getOperation(); + TF::ResourceAnalyzer resource_analyzer(module_op); + + PrintAnalysisResults(resource_analyzer, module_op); +} + +} // namespace + +std::unique_ptr> CreateResourceAnalyzerTestPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc new file mode 100644 index 0000000000000000000000000000000000000000..3b3c1bc3f624c8161136f6aa31f8bddd804764c4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc @@ -0,0 +1,320 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +#define DEBUG_TYPE "tf-resource-device-inference" + +namespace mlir { +namespace TF { + +namespace { +constexpr char kDeviceAttr[] = "device"; +constexpr char kFuncDeviceAttr[] = "tf.device"; + +#define GEN_PASS_DEF_RESOURCEDEVICEINFERENCEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass that propagates device assignment of resources on a module. It +// performs in-function propagation, as well as cross-function propagation from +// callers to callees. +// +// This pass changes the module by adding "tf.device" attribute to function +// arguments and adding "device" attribute to TF ops. +struct ResourceDeviceInference + : public impl::ResourceDeviceInferencePassBase { + void runOnOperation() override; +}; + +// A class that records each resource's device assignment in a function. +class PerFunctionResult { + public: + explicit PerFunctionResult( + func::FuncOp func_op, + const TF::ResourceAliasAnalysis::Info& alias_analysis) + : alias_analysis_(alias_analysis) {} + + // Returns the recorded device assignment for a resource, if any. + std::optional DeviceForResource(Value resource) const { + std::optional result; + if (alias_analysis_.IsUnknownResource(resource)) return std::nullopt; + for (int64_t id : alias_analysis_.GetResourceUniqueIds(resource)) { + auto it = resource_id_to_device_.find(id); + if (it == resource_id_to_device_.end()) continue; + if (!result || result == it->second) { + result = it->getSecond(); + continue; + } + // Got conflicting assignments + return std::nullopt; + } + return result; + } + + // Records the device assignment for a resource. If the new assignment + // conflicts with an existing one, returns an error. + // + // If `changed` is provided, assign *changed to true if anything is modified. + LogicalResult AddResourceDevice(Value resource, StringRef device, + bool* changed = nullptr) { + if (alias_analysis_.IsUnknownResource(resource)) return success(); + for (int64_t id : alias_analysis_.GetResourceUniqueIds(resource)) { + auto emplace_res = resource_id_to_device_.try_emplace(id, device); + if (emplace_res.second) { + if (changed) *changed = true; + } else if (emplace_res.first->getSecond() != device) { + // Existing assignment does not equal the new assignment. + return failure(); + } + } + return success(); + } + + private: + llvm::SmallDenseMap resource_id_to_device_; + const TF::ResourceAliasAnalysis::Info& alias_analysis_; +}; + +// Tries to record device assignment for a resource. +LogicalResult AddResourceDeviceAndEmitError(Value resource, StringRef device, + Operation* error_reporting_op, + PerFunctionResult* result, + bool* changed = nullptr) { + auto res = result->AddResourceDevice(resource, device, changed); + if (failed(res)) { + error_reporting_op->emitError() + << "Conflicting device assignment for resource"; + } + return res; +} + +// Extracts and canonicalizes the device attribute. +inline StringRef GetDeviceAttr(func::FuncOp func, int arg_no) { + auto device_attr = + func.getArgAttrOfType(arg_no, kFuncDeviceAttr); + return device_attr ? device_attr.getValue() : ""; +} + +// Extracts and canonicalizes the device attribute. +inline StringRef GetDeviceAttr(Operation* op) { + auto device_attr = op->getAttrOfType(kDeviceAttr); + return device_attr ? device_attr.getValue() : ""; +} + +// Print operation with debug info (to get line number info for debugging) +void dump(StringRef message, Operation* op) { + llvm::dbgs() << message; + op->print(llvm::dbgs(), OpPrintingFlags().enableDebugInfo( + /*enable=*/true, /*prettyForm=*/true)); + llvm::dbgs() << "\n"; +} + +// Propagates device assignment inside a function. +LogicalResult ComputeResourceDevicesInComputation(func::FuncOp func_op, + PerFunctionResult* result) { + OpBuilder builder(func_op); + // Function arguments. + for (auto arg : filter_resources(func_op.getArguments())) { + StringRef device_attr = GetDeviceAttr(func_op, arg.getArgNumber()); + if (device_attr.empty()) { + // If device_attr does not exist, try to construct it from any recorded + // assignment. + if (auto device = result->DeviceForResource(arg)) { + func_op.setArgAttr(arg.getArgNumber(), kFuncDeviceAttr, + builder.getStringAttr(*device)); + } + continue; + } + // Record the attribute. + auto res = AddResourceDeviceAndEmitError(arg, device_attr, func_op, result); + if (failed(res)) return res; + } + + // To support WhileRegion, we need to propagate device attributes from + // WhileRegion operands to body/cond region arguments *prior* to visiting + // these regions, so use a pre-order walk. + WalkResult walk_res = func_op.walk([&](Operation* op) { + if (auto var_handle = dyn_cast(op)) { + // Record VarHandleOp's device attribute. + StringRef device_attr = GetDeviceAttr(op); + if (device_attr.empty()) return WalkResult::advance(); + auto res = AddResourceDeviceAndEmitError(var_handle.getResource(), + device_attr, op, result); + if (failed(res)) return WalkResult::interrupt(); + } else if (auto identity = dyn_cast(op)) { + LLVM_DEBUG(dump("Visiting ", identity)); + // Try to construct IdentityOp's attribute from recorded assignment. + if (!GetDeviceAttr(op).empty()) return WalkResult::advance(); + for (auto output : filter_resources(op->getResults())) { + LLVM_DEBUG(llvm::dbgs() << " Processing output #" + << output.getResultNumber() << "\n"); + if (auto device = result->DeviceForResource(output)) { + LLVM_DEBUG(llvm::dbgs() << " Setting device = " << *device << "\n"); + identity->setAttr(kDeviceAttr, builder.getStringAttr(*device)); + } + } + } else if (auto while_region = dyn_cast(op)) { + // For WhileRegion, do local analysis prior to visiting the attached + // regions and propagate device annotations to the cond and body + // region arguments. The annotations are the union of annotations + // on the input and result. Resource alias analysis already propagates + // resource ID from the inputs to the results for a while, so just + // need to consider the results. + LLVM_DEBUG(llvm::dbgs() << "Visiting WhileRegion\n"); + + for (auto output : filter_resources(while_region.getResults())) { + auto device = result->DeviceForResource(output); + int output_index = output.getResultNumber(); + if (!device) { + LLVM_DEBUG(llvm::dbgs() + << " No device for output #" << output_index << "\n"); + continue; + } + // Transfer the annotation to both region arguments + for (Region* region : while_region.getRegions()) { + BlockArgument arg = region->getArgument(output_index); + LLVM_DEBUG(llvm::dbgs() + << " Propagating device = '" << *device << "' to arg #" + << output_index << " of region #" + << region->getRegionNumber() << "\n"); + if (failed(AddResourceDeviceAndEmitError(arg, *device, while_region, + result))) + return WalkResult::interrupt(); + } + } + } + return WalkResult::advance(); + }); + return failure(walk_res.wasInterrupted()); +} + +void ResourceDeviceInference::runOnOperation() { + auto module = getOperation(); + const auto& resource_alias_analysis = + getAnalysis(); + + llvm::SmallDenseMap per_function_results; + llvm::SetVector worklist; + for (auto func_op : module.getOps()) { + worklist.insert(func_op); + per_function_results.try_emplace( + func_op, func_op, resource_alias_analysis.GetAnalysisForFunc(func_op)); + } + // Helper that propagates an op's recorded operand device assignments to its + // called function's arguments. + auto propagate_operands_to_callee_arguments = + [&](Operation* caller, Operation::operand_range caller_operands, + ArrayRef callees, const PerFunctionResult& caller_res) { + for (func::FuncOp callee : callees) { + assert(callee); + auto& callee_res = per_function_results.find(callee)->getSecond(); + bool callee_needs_recompute = false; + for (BlockArgument arg : filter_resources(callee.getArguments())) { + Value arg_operand = caller_operands[arg.getArgNumber()]; + auto device = caller_res.DeviceForResource(arg_operand); + if (!device) continue; + LLVM_DEBUG(llvm::dbgs() + << "Propagating '" << *device << "' to arg #" + << arg.getArgNumber() << " of function @" + << callee.getName() << "\n"); + if (failed(AddResourceDeviceAndEmitError(arg, *device, caller, + &callee_res, + &callee_needs_recompute))) + return failure(); + } + // If the callee recording is modified, make sure that it will be + // reprocessed. + if (callee_needs_recompute) worklist.insert(callee); + } + return success(); + }; + + while (!worklist.empty()) { + auto func_op = worklist.pop_back_val(); + auto& func_res = per_function_results.find(func_op)->getSecond(); + // In-function propagation. + if (failed(ComputeResourceDevicesInComputation(func_op, &func_res))) + return signalPassFailure(); + + // Propagation to callees. + auto walk_res = func_op.walk([&](Operation* op) { + if (auto while_op = dyn_cast(op)) { + if (failed(propagate_operands_to_callee_arguments( + while_op, while_op.getOperands(), + {while_op.body_function(), while_op.cond_function()}, + func_res))) + return WalkResult::interrupt(); + } else if (auto if_op = dyn_cast(op)) { + if (failed(propagate_operands_to_callee_arguments( + if_op, if_op.getInput(), + {if_op.then_function(), if_op.else_function()}, func_res))) + return WalkResult::interrupt(); + } else if (auto call = dyn_cast(op)) { + auto func = dyn_cast(call.resolveCallable()); + if (!func) { + op->emitError( + "Cannot propagate device attribute to callee: Unable to resolve " + "call"); + return WalkResult::interrupt(); + } + LLVM_DEBUG(llvm::dbgs() + << "Visiting call to function @" << func.getName() << "\n"); + if (failed(propagate_operands_to_callee_arguments( + call, call.getArgOperands(), {func}, func_res))) + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (walk_res.wasInterrupted()) return signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> CreateResourceDeviceInferencePass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting.cc new file mode 100644 index 0000000000000000000000000000000000000000..2257b7772529ecf251da7d77ac199a6317fb7fb4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting.cc @@ -0,0 +1,1353 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass lifts resource variable operations outside of device computation. + +#include +#include +#include +#include + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" + +namespace mlir { + +namespace { + +constexpr char kDeviceAttr[] = "device"; + +#define GEN_PASS_DEF_RESOURCEOPLIFTINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +// Lift resource operations out of device computation. +struct ResourceOpLiftingPass + : public impl::ResourceOpLiftingPassBase { + void runOnOperation() override; +}; + +bool IsResource(Value value) { + return mlir::isa(getElementTypeOrSelf(value.getType())); +} + +// Get the type of the data contained in a resource. Returns null if there is +// no single type in the resource. +Type GetResourceSubtype(Value value) { + auto resource_type = + mlir::dyn_cast(getElementTypeOrSelf(value.getType())); + auto subtypes = resource_type.getSubtypes(); + if (subtypes.size() == 1) return subtypes[0]; + return nullptr; +} + +// Replaces all `tf.VarIsInitializedOp` in a block with a constant true. +// TODO(b/171039585): Replace this with proper analysis of +// `tf.VarIsInitializedOp` in regards to resource writes and control flow. +void SetAllVarIsInitializedToTrue(Block* block) { + auto builder = OpBuilder::atBlockBegin(block); + TF::ConstOp const_true = nullptr; + for (auto op : + llvm::make_early_inc_range(block->getOps())) { + builder.setInsertionPoint(op); + if (!const_true) + const_true = TF::ConstOp::create( + builder, op.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get(/*shape=*/{}, builder.getI1Type()), true)); + + op.getIsInitialized().replaceAllUsesWith(const_true); + op.erase(); + } +} + +// Performs store-load forwarding. This effectively removes +// 1) Any resource loads after a store to that same resource is done +// 2) Any resource stores except the last one. +// TODO(ycao): Store-load forwarding implemented here is only correct when +// computation is purely sequential (no concurrency). Need to support concurrent +// computation as well. +void ForwardStoreToLoad(Block* block) { + // resource_handle_to_last_store_op keeps track of the most recent (last) + // store to each resource. Non-existent entry indicates that a resource has + // not been stored to yet. + llvm::SmallDenseMap + resource_handle_to_last_store_op; + + // Only iterate through ops directly in the block as we can't handle ops + // nested deeper in regions. + for (Operation& op : llvm::make_early_inc_range(*block)) { + if (auto read_variable_op = dyn_cast(&op)) { + Value resource = read_variable_op.getResource(); + auto last_store = resource_handle_to_last_store_op[resource]; + if (!last_store) continue; + + // Use stored value in last_store to replace all uses of current resource + // load's result, then erase this resource load. Add an intermediate + // CastOp if the shape of types doesn't exactly match. + Type read_type = read_variable_op.getValue().getType(); + if (read_type != last_store.getValue().getType()) { + OpBuilder builder(last_store); + builder.setInsertionPointAfter(last_store); + auto cast = TF::CastOp::create(builder, last_store.getLoc(), read_type, + last_store.getValue(), + /*Truncate=*/builder.getBoolAttr(false)); + read_variable_op.getValue().replaceAllUsesWith(cast); + } else { + read_variable_op.getValue().replaceAllUsesWith(last_store.getValue()); + } + + read_variable_op.erase(); + continue; + } + + if (auto assign_variable_op = dyn_cast(&op)) { + Value resource = assign_variable_op.getResource(); + auto last_store = resource_handle_to_last_store_op[resource]; + // Previous store ops to same resource can be erased. + if (last_store) last_store.erase(); + + resource_handle_to_last_store_op[resource] = assign_variable_op; + } + } +} + +//===----------------------------------------------------------------------===// +// RegionResourceHoister +//===----------------------------------------------------------------------===// + +// Helper class to hoist resource ops out of regions attached to an op. +class RegionResourceHoister { + public: + explicit RegionResourceHoister(Operation* op) : op_(op) {} + + // Analyzes attached regions to record resources read and written. + LogicalResult Analyze(); + + // Returns all resources accessed by the regions attached the op. + auto& GetResources() { return resources_; } + + // Returns if the given value is a resource that needs lifting. + bool Contains(Value resource) const { + return resources_.find(resource) != resources_.end(); + } + + // Drops the given resource from lifting. + void DropResource(Value resource) { + resources_.erase(resource); + written_resources_.remove(resource); + } + + // Replaces all resource loads in all regions attached to the op. + void ReplaceResourceLoads(bool read_only) { + llvm::for_each(op_->getRegions(), [&](Region& region) { + ReplaceResourceLoads(region, read_only); + }); + } + + static LogicalResult ReplaceOpWithNewOp(Operation* op); + + private: + // Returns if any resources need lifting. + bool NeedsLifting() const { return !resources_.empty(); } + + // Returns the number of results generated by the lifted op. + int GetLiftedNumResults() const { return num_new_results_; } + + // Generates hoisted reads for resources that need them before the op. + void GenerateHoistedReads(); + + // Replaces all resource loads in the given region with hoisted loads. If + // `read_only` is true, limit this replacement to read only resources. + void ReplaceResourceLoads(Region& region, bool read_only); + + // Appends final values writte to resources to the region returns for the + // given set of regions. + void AppendResourceStoreValueToReturn(RegionRange regions); + + // Performs the final replacement of the op. + void ReplaceOpWithNewOp(); + + // Returns is this resource was written to in any of the regions. + bool IsWritten(Value resource) const { + return written_resources_.contains(resource); + } + + static LogicalResult HoistResourcesOutOfIfCaseCluster(Operation* op); + static LogicalResult HoistResourcesOutOfWhileRegion(TF::WhileRegionOp op); + + Operation* op_; + + // Per resource information about accesses to that resource. + struct ResourceInfo { + // Is this resource read in any of the regions? + bool is_read; + // Is this resource written in any of the regions? + bool is_written; + // Is this resource written in all of the regions? + bool is_written_all; + // The hoisted read used to replace region reads. + Value hoisted_read; + // the type of the data held by the resource. + Type data_type; + // For written resources, the result # of the lifted op which will hold the + // value of the resource. This result will be used to generates writes to + // the resource after the lifted op. + int result_index; + // Attributes on the read operation. + DictionaryAttr read_attrs; + // Attributes on the write operation. + DictionaryAttr write_attrs; + + ResourceInfo() + : is_read(false), + is_written(false), + is_written_all(false), + hoisted_read(nullptr), + data_type(nullptr), + result_index(-1) {} + + bool IsResultIndexAssigned() { return result_index != -1; } + + // Refine the resource type using the given type `type`. + void RefineType(Type type) { + if (!data_type) { + data_type = type; + } else { + data_type = TF::GetCastCompatibleType(data_type, type, + /*may_ignore_ref_type_a=*/false); + assert(data_type != nullptr && "Resource used with incompatible types"); + } + } + }; + llvm::MapVector resources_; + llvm::SetVector written_resources_; + // number of new results after lifting. + int num_new_results_; +}; + +// Analyzes resources that are read or written within attached regions. +LogicalResult RegionResourceHoister::Analyze() { + // Hoisting of child regions might have created opportunity for store-load + // forwarding. + for (Region& region : op_->getRegions()) { + ForwardStoreToLoad(®ion.front()); + } + + llvm::SetVector all_resources; + bool is_func = false; + // For functions, the resources to analyze are the function arguments. + // Otherwise, its the region captures. + if (func::FuncOp func = dyn_cast(op_)) { + is_func = true; + Region& body = func.getBody(); + for (BlockArgument arg : body.getArguments()) { + if (IsResource(arg)) all_resources.insert(arg); + } + } else { + getUsedValuesDefinedAbove(op_->getRegions(), all_resources); + all_resources.remove_if([](Value value) { return !IsResource(value); }); + } + + num_new_results_ = op_->getNumResults(); + + for (auto resource : all_resources) { + ResourceInfo info; + info.data_type = GetResourceSubtype(resource); + llvm::BitVector written_regions(op_->getNumRegions()); + bool unsupported_use = false; + for (OpOperand& use : resource.getUses()) { + Operation* user = use.getOwner(); + // If the user is not in one of the regions, we are not interested in it. + // Since all the sub-regions within this region (i.e., regions attached to + // op's in this region) have themselves gone through lifting, all resource + // users are expected to be operations in this region and not embedded + // within other sub-regions attached to ops in this region. So the check + // for whether a user is in one of the regions attached to this op is + // straightforward. + if (user->getParentRegion()->getParentOp() != op_) continue; + + // For functions, if the resource is used as a return operand, use that + // as its result index. + if (is_func && isa(user)) { + assert(!info.IsResultIndexAssigned() && + "Expect resource argument to returned no more than once"); + info.result_index = use.getOperandNumber(); + continue; + } + + auto read = dyn_cast(user); + auto write = dyn_cast(user); + if (!read && !write) { + unsupported_use = true; + break; + } + + if (read && !info.is_read) { + info.is_read = true; + info.RefineType(read.getValue().getType()); + info.read_attrs = user->getAttrDictionary(); + } + + if (write) { + info.is_written = true; + info.RefineType(write.getValue().getType()); + info.write_attrs = user->getAttrDictionary(); + written_regions.set(user->getParentRegion()->getRegionNumber()); + } + } + + // If the resource is used in an op that we do not understand, skip + // lifting for that resource. + if (unsupported_use) continue; + + info.is_written_all = written_regions.count() == op_->getNumRegions(); + + // If the resource is written in some but not all regions, we would need + // a read for the value before these regions. Note that this is applicable + // only to multi-region ops: + // If/Case: If not all regions write to the resource, post hoisting the read + // value need to be routed through all paths that don't write. + // While: since while condition cannot write, any resource written in the + // while body will need to be read as well in case the while body is never + // executed. + // Both cases are handled by the condition below. + if (info.is_written && !info.is_written_all) info.is_read = true; + + // Allocate a result index for written resources that don't have one. + if (info.is_written) { + written_resources_.insert(resource); + if (!info.IsResultIndexAssigned()) info.result_index = num_new_results_++; + } + + resources_.insert({resource, info}); + } + return success(); +} + +// Generates hoisted reads for all resources that need them just before the op. +void RegionResourceHoister::GenerateHoistedReads() { + OpBuilder builder(op_); + DictionaryAttr empty_attrs = builder.getDictionaryAttr({}); + for (auto& resource_it : GetResources()) { + Value resource = resource_it.first; + auto& info = resource_it.second; + + if (info.is_read) { + Operation* read = TF::ReadVariableOp::create(builder, op_->getLoc(), + info.data_type, resource); + read->setAttrs(info.read_attrs ? info.read_attrs : empty_attrs); + read->removeAttr(kDeviceAttr); + info.hoisted_read = read->getResult(0); + } + } +} + +// Replaces all resource reads with the hoisted read. +void RegionResourceHoister::ReplaceResourceLoads(Region& region, + bool read_only) { + assert(llvm::hasSingleElement(region) && "Expected single block region"); + // Only iterate through ops directly in the body as we can't handle + // ops nested deeper in regions. + auto all_reads = region.front().getOps(); + for (auto read_op : llvm::make_early_inc_range(all_reads)) { + Value resource = read_op.getResource(); + if (!Contains(resource)) continue; + + ResourceInfo& info = resources_[resource]; + // If replacing loads for read only resources, skip if the resource + // was written to. + if (read_only && info.is_written) continue; + + read_op.replaceAllUsesWith(info.hoisted_read); + read_op.erase(); + } +} + +// For written resources, add its value at the end of each region to that +// regions return value. For a region, its value at the end may be a value +// written to that resource in that region, or its hoisted read value if the +// resource is not written in that region. The return value can be vended out +// either as an existing return value, or a newly allocated return value. +void RegionResourceHoister::AppendResourceStoreValueToReturn( + RegionRange regions) { + for (Region* region : regions) { + assert(llvm::hasSingleElement(*region) && "Expected single block region"); + Block& front = region->front(); + auto old_return = front.getTerminator(); + assert(old_return->getNumOperands() == op_->getNumResults()); + auto new_return_operands = llvm::to_vector<4>(old_return->getOperands()); + new_return_operands.resize(num_new_results_); + + // initialize return values for written resources to be the hoisted reads. + for (Value resource : written_resources_) { + const ResourceInfo& info = resources_[resource]; + new_return_operands[info.result_index] = info.hoisted_read; + } + + // Only iterate through ops directly in the body as op's embedded in child + // regions should have been lifted out. + auto assign_ops = front.getOps(); + for (auto assign_variable_op : llvm::make_early_inc_range(assign_ops)) { + Value resource = assign_variable_op.getResource(); + if (!IsWritten(resource)) continue; + + // TODO(ycao): Prevent same value from being returned multiple times. + // TODO(ycao): Do not return resource store value if it is defined outside + // of cluster. Both of these can be post-resource-op-lifting cleanup + // passes. + int result_index = resources_[resource].result_index; + new_return_operands[result_index] = assign_variable_op.getValue(); + assign_variable_op.erase(); + } + old_return->setOperands(new_return_operands); + } +} + +// Replace the old op with a new op (with potentially additional results), and +// add stores to written resources after the new op. +void RegionResourceHoister::ReplaceOpWithNewOp() { + auto new_result_types = llvm::to_vector<4>(op_->getResultTypes()); + int result_region = isa(op_) ? 1 : 0; + Operation* terminator = op_->getRegion(result_region).front().getTerminator(); + auto extra_result_types = + terminator->getOperands().drop_front(op_->getNumResults()).getTypes(); + new_result_types.insert(new_result_types.end(), extra_result_types.begin(), + extra_result_types.end()); + OpBuilder builder(op_); + // Clone this old operation but with new result types. + Operation* new_op = Operation::create( + op_->getLoc(), op_->getName(), new_result_types, op_->getOperands(), + op_->getAttrs(), op_->getPropertiesStorage(), op_->getSuccessors(), + op_->getNumRegions()); + builder.insert(new_op); + + // Move regions to the new op. + for (auto it : llvm::zip(op_->getRegions(), new_op->getRegions())) { + Region& old_region = std::get<0>(it); + Region& new_region = std::get<1>(it); + new_region.takeBody(old_region); + } + + // Insert stores to all written resources. + for (Value resource : written_resources_) { + ResourceInfo& info = resources_[resource]; + Value value_to_write = new_op->getResult(info.result_index); + Operation* write = TF::AssignVariableOp::create(builder, op_->getLoc(), + resource, value_to_write); + write->setAttrs(info.write_attrs); + write->removeAttr(kDeviceAttr); + } + + // As a part of lifting, we either reuse an existing slot for resource type + // results or add a new slot. Resource type results should not have any uses + // to begin with. So we can safely replace each old op result with the + // corresponding new op result. + int old_num_results = op_->getNumResults(); + op_->replaceAllUsesWith(new_op->getResults().take_front(old_num_results)); + op_->erase(); + op_ = nullptr; +} + +// Lift resource load and stores out of regions attached to `op`, where op is +// an If/case/cluster op. +LogicalResult RegionResourceHoister::HoistResourcesOutOfIfCaseCluster( + Operation* op) { + RegionResourceHoister hoister(op); + if (failed(hoister.Analyze())) return failure(); + + // If there are no resource region captures, then nothing to do. + if (!hoister.NeedsLifting()) return success(); + + // Start the transformation. For each region, replace the resource read with + // the value read before the op. + hoister.GenerateHoistedReads(); + hoister.ReplaceResourceLoads(/*read_only=*/false); + hoister.AppendResourceStoreValueToReturn(op->getRegions()); + hoister.ReplaceOpWithNewOp(); + return success(); +} + +// Lift resource loads and stores out of WhileRegion +LogicalResult RegionResourceHoister::HoistResourcesOutOfWhileRegion( + TF::WhileRegionOp op) { + // For WhileRegion, post canonicalization all resource used within the + // body and condition regions are replaced with captured values, so we do not + // need to take into account the body and condition region arguments. + RegionResourceHoister hoister(op); + + if (failed(hoister.Analyze())) return failure(); + + // If there are no resource region captures, then nothing to do. + if (!hoister.NeedsLifting()) return success(); + + // The resources captured for While loop fall into two categories: + // (a) read-only. These reads can be replaced by a hoisted read created + // before the WhileOp (similar to if and case). + // (b) written: since the value is written in the loop (which can only in + // loop body, all these will become loop variables. Since all resource + // variables are removed from the loop variabled during + // canonicalizationW, we need to create new operand/result slots. The + // input operands for these slots are the read values + // prior to the op, and all references to these are replaced by the + // corresponding slot argument. We need to generate writes following + // the while for these resources. + // + // Note that for WhileRegion ops, if a resource is written, it will be written + // only in the body and not the condition, so the hoister analysis will infer + // it as needing a read as well. + + // Generate hoisted reads before the while. + hoister.GenerateHoistedReads(); + + // Replace just the read-only resources with the hoisted reads. + hoister.ReplaceResourceLoads(/*read_only=*/true); + + // For written resources, add additional operands to the while op. + int num_old_results = op.getNumResults(); + int num_new_results = hoister.GetLiftedNumResults(); + int num_extra_results = num_new_results - num_old_results; + + SmallVector new_result_types; + SmallVector new_while_operands; + new_result_types.resize(num_extra_results); + new_while_operands.resize(num_extra_results); + + for (auto& it : hoister.GetResources()) { + if (!it.second.is_written) continue; + int index = it.second.result_index - num_old_results; + new_result_types[index] = it.second.data_type; + new_while_operands[index] = it.second.hoisted_read; + } + op.getOperation()->insertOperands(op.getNumOperands(), new_while_operands); + + // Patch the cond and body regions to have additional arguments, and replace + // the remaining resource reads (which will be resource reads for written + // resources) with these arguments. + Location loc = op.getLoc(); + for (Region* region : op.getRegions()) { + region->addArguments(new_result_types, + SmallVector(new_result_types.size(), loc)); + // Point hoisted read for written resources to the region's arguments. + for (auto& it : hoister.GetResources()) { + if (!it.second.is_written) continue; + it.second.hoisted_read = region->getArgument(it.second.result_index); + } + hoister.ReplaceResourceLoads(*region, /*read_only=*/false); + } + + // Add additional return values to body return. These correspond to values + // written to resources in the body region. + hoister.AppendResourceStoreValueToReturn(op.getRegions().drop_front()); + + // Finally, create a new while with additional return values. + hoister.ReplaceOpWithNewOp(); + return success(); +} + +// Lift resources out of the regions attached to `op` +LogicalResult RegionResourceHoister::ReplaceOpWithNewOp(Operation* op) { + if (auto while_op = dyn_cast(op)) + return HoistResourcesOutOfWhileRegion(while_op); + return HoistResourcesOutOfIfCaseCluster(op); +} + +// Holds information about a function's use of a resource argument. +struct ResourceArgUseInfo { + // Data type of the data contained in the resource. + Type data_type; + // Is the resource argument used in an assign op? + bool updated; + // Is the resource argument used in a read or assign op? + bool used; +}; + +// Finds the ResourceArgUseInfo for each resource argument. Forwarding to the +// output (i.e., the argument is an operand of the return op) is not considered +// as a use. This doesn't support nesting of ops, so before calling this, nested +// ops/functions need to be already resource-lifted. +LogicalResult FindResourceArgUseInfo( + func::FuncOp func_op, + llvm::SmallDenseMap* result) { + auto return_op = func_op.front().getTerminator(); + for (auto arg : TF::filter_resources(func_op.getArguments())) { + ResourceArgUseInfo info; + info.used = false; + info.updated = false; + bool read_or_assigned = false; + bool used_in_unsupported_op = false; + for (auto user : arg.getUsers()) { + if (user == return_op) continue; + info.used = true; + if (auto read = llvm::dyn_cast(user)) { + read_or_assigned = true; + info.data_type = read.getType(); + continue; + } + + if (auto assign = llvm::dyn_cast(user)) { + read_or_assigned = true; + info.updated = true; + info.data_type = assign.getValue().getType(); + continue; + } + + used_in_unsupported_op = true; + break; + } + + // If the arg is used in an unsupported op, skip lifting it. + if (used_in_unsupported_op) continue; + (*result)[arg.getArgNumber()] = info; + } + return success(); +} + +// Merges two sets of resource arg use infos. An argument is considered used in +// the merged result as long as either set marks it as used. This is used to +// merge results from functions that have aliasing inputs, e.g., a while loop's +// body and condition. The sets of keys of the two maps must be the same. +llvm::SmallDenseMap MergeArgResourceUseInfo( + const llvm::SmallDenseMap& infos0, + const llvm::SmallDenseMap& infos1) { + llvm::SmallDenseMap result; + for (const auto& entry : infos0) { + auto info1_it = infos1.find(entry.getFirst()); + // If the entry is missing in any input, we should not touch this entry. + if (info1_it == infos1.end()) continue; + auto& info = result[entry.getFirst()]; + info = entry.getSecond(); + if (info.updated) continue; + if (info1_it->getSecond().used) { + info.used = true; + info.updated = info1_it->getSecond().updated; + info.data_type = info1_it->getSecond().data_type; + } + } + return result; +} + +// Removes the unused resource arguments, and the return values that forward the +// removed arguments. If old_to_new_arg_indices is provided, it will store the +// new argument index that corresponds to each original index (-1 means it is +// removed). If remaining_resource_data_types is provided, it will store the +// data types of the remaining resource arguments, where the indices are after +// removing unused ones. +LogicalResult RemoveUnusedResourceArgumentsAndForwardedRetvals( + const llvm::SmallDenseMap& infos, + func::FuncOp func_op, + llvm::SmallVector* old_to_new_arg_indices = nullptr, + llvm::SmallDenseMap* remaining_resource_data_types = + nullptr) { + // Remove return values forwarded from unused arguments. + auto return_op = func_op.front().getTerminator(); + auto old_return_vals = llvm::to_vector<8>(return_op->getOperands()); + int64_t skipped_retvals = 0; + for (auto entry : llvm::enumerate(old_return_vals)) { + auto return_val = entry.value(); + if (auto arg = mlir::dyn_cast(return_val)) { + auto it = infos.find(arg.getArgNumber()); + if (it != infos.end() && !it->getSecond().used) { + return_op->eraseOperand(entry.index() - skipped_retvals++); + } + } + } + llvm::BitVector indices_to_erase(func_op.getNumArguments()); + llvm::SmallVector new_types; + int64_t skipped_args = 0; + for (auto arg : func_op.getArguments()) { + auto it = infos.find(arg.getArgNumber()); + if (it != infos.end() && !it->getSecond().used) { + indices_to_erase.set(arg.getArgNumber()); + skipped_args++; + if (old_to_new_arg_indices != nullptr) { + old_to_new_arg_indices->push_back(-1); + } + } else { + new_types.push_back(arg.getType()); + if (old_to_new_arg_indices != nullptr) { + old_to_new_arg_indices->push_back(arg.getArgNumber() - skipped_args); + } + if (it != infos.end() && remaining_resource_data_types != nullptr) { + (*remaining_resource_data_types)[arg.getArgNumber() - skipped_args] = + it->second.data_type; + } + } + } + if (failed(func_op.eraseArguments(indices_to_erase))) { + return failure(); + } + func_op.setType( + FunctionType::get(func_op.getContext(), new_types, + llvm::to_vector<4>(return_op->getOperandTypes()))); + return success(); +} + +// Lifts reads/writes of resource arguments from func_op and changes its +// signature. resource_data_types is the (index, data type) pair for each +// resource argument. handle_updated_arg_value is a caller-provided function +// that handles the updated value for an resource argument. +LogicalResult LiftArgRetResourcesForFunction( + func::FuncOp func_op, + const llvm::SmallDenseMap& resource_data_types, + llvm::function_ref handle_updated_arg_value) { + RegionResourceHoister hoister(func_op); + if (failed(hoister.Analyze())) return failure(); + + // Each of these resources could be read or written in the function. If its + // read, we need to replace the resource arg with a value arg to get the + // read value. If its written, we need to replace the write with an additional + // value to be written. + + // Now create read values that will be used to replace each resource that + // is read in the function body. These read values are just the same argument + // with type replaced. + llvm::SmallVector skipped_args; + for (auto& it : hoister.GetResources()) { + BlockArgument arg = mlir::dyn_cast(it.first); + assert(arg && "Expect resources for FuncOp to be its arguments"); + auto type_iter = resource_data_types.find(arg.getArgNumber()); + if (type_iter == resource_data_types.end()) { + // Skip lifting the resource if it's not present in the data type map. + // This indicates that the resource is not to be lifted because it is used + // in an unsupported op in some other function. + skipped_args.push_back(arg); + } else { + arg.setType(type_iter->second); + it.second.hoisted_read = arg; + } + } + + // Drop all the args that have to be skipped. + for (Value arg : skipped_args) hoister.DropResource(arg); + + hoister.ReplaceResourceLoads(/*read_only=*/false); + + // For writes, invoke the callback and then erase the write. + auto assign_ops = func_op.front().getOps(); + for (auto assign_variable_op : llvm::make_early_inc_range(assign_ops)) { + Value resource = assign_variable_op.getResource(); + if (!hoister.Contains(resource)) continue; + + auto arg = mlir::dyn_cast(resource); + handle_updated_arg_value(arg.getArgNumber(), assign_variable_op.getValue()); + assign_variable_op.erase(); + } + + func_op.setType(FunctionType::get( + func_op.getContext(), func_op.front().getArgumentTypes(), + func_op.front().getTerminator()->getOperandTypes())); + + return success(); +} + +// Returns a vector filtered from range where the unused elements (specified by +// resource_arg_uses) are removed. +template +llvm::SmallVector FilterRange( + Range range, + const llvm::SmallDenseMap& resource_arg_uses) { + llvm::SmallVector filtered; + for (auto entry : llvm::enumerate(range)) { + auto it = resource_arg_uses.find(entry.index()); + if (it == resource_arg_uses.end() || it->getSecond().used) + filtered.push_back(entry.value()); + } + return filtered; +} + +// Changes the types of the control flow op (e.g., while, if) and adds loads and +// stores around it. arg_data_type_and_updated_output_index maps an operand (to +// be changed) index to its data type and the updated value index in the output +// (-1 means not updated.) +void AddLoadsStoresOutsideControlFlowOp( + Operation* caller, + const llvm::SmallDenseMap>& + arg_data_type_and_updated_output_index) { + OpBuilder builder(caller); + auto new_operands = llvm::to_vector<8>(caller->getOperands()); + llvm::SmallVector changed_indices; + // Find the operands to change, and create the loads. + for (auto& entry : arg_data_type_and_updated_output_index) { + int64_t index = entry.getFirst(); + Type new_type = entry.getSecond().first; + int64_t updated_index = entry.getSecond().second; + auto operand = caller->getOperand(index); + builder.setInsertionPoint(caller); + new_operands[index] = TF::ReadVariableOp::create(builder, caller->getLoc(), + ArrayRef{new_type}, + ArrayRef{operand}); + caller->setOperand(index, new_operands[index]); + if (updated_index < 0) continue; + builder.setInsertionPointAfter(caller); + TF::AssignVariableOp::create( + builder, caller->getLoc(), ArrayRef{}, + ArrayRef{operand, caller->getResult(updated_index)}); + } +} + +// Lifts loads/stores from while loop's body and cond functions. +LogicalResult HandleWhileLoop(TF::WhileOp while_op, func::FuncOp body, + func::FuncOp cond) { + auto return_op = body.front().getTerminator(); + llvm::SmallDenseMap body_use_info; + llvm::SmallDenseMap cond_use_info; + if (failed(FindResourceArgUseInfo(body, &body_use_info)) || + failed(FindResourceArgUseInfo(cond, &cond_use_info))) { + return failure(); + } + // A resource is considered used as long as it is used in either body or cond. + auto resource_arg_uses = + MergeArgResourceUseInfo(body_use_info, cond_use_info); + if (resource_arg_uses.empty()) return success(); + + // Remove unused resources in functions. + llvm::SmallVector old_to_new_indices; + llvm::SmallDenseMap remaining_resource_data_types; + if (failed(RemoveUnusedResourceArgumentsAndForwardedRetvals( + resource_arg_uses, body, &old_to_new_indices, + &remaining_resource_data_types))) { + return failure(); + } + if (failed(RemoveUnusedResourceArgumentsAndForwardedRetvals(resource_arg_uses, + cond))) { + return failure(); + } + (void)LiftArgRetResourcesForFunction( + body, remaining_resource_data_types, + [&](int64_t index, Value value) { return_op->setOperand(index, value); }); + (void)LiftArgRetResourcesForFunction(cond, remaining_resource_data_types, + [&](int64_t index, Value value) { + // We already checked that cond should + // not have variable writes. + assert(false && "Should not happen"); + }); + // Recreate the while op. + OpBuilder builder(while_op); + // Now use the filtered original operands, which will be replaced by + // AddLoadsStoresOutsideControlFlowOp(). + auto new_while = TF::WhileOp::create( + builder, while_op.getLoc(), body.getFunctionType().getResults(), + FilterRange(while_op.getOperands(), + resource_arg_uses), + while_op->getAttrs()); + // Prepare for AddLoadsStoresOutsideControlFlowOp(). + llvm::SmallDenseMap> + arg_data_type_and_updated_output_index; + for (const auto& entry : remaining_resource_data_types) { + int64_t update_index = return_op->getOperand(entry.getFirst()) == + body.getArgument(entry.getFirst()) + ? -1 + : entry.getFirst(); + arg_data_type_and_updated_output_index[entry.getFirst()] = { + entry.getSecond(), update_index}; + } + AddLoadsStoresOutsideControlFlowOp(new_while, + arg_data_type_and_updated_output_index); + // Replace uses. + for (int64_t i = 0, end = old_to_new_indices.size(); i < end; ++i) { + if (old_to_new_indices[i] >= 0) { + while_op.getResult(i).replaceAllUsesWith( + new_while.getResult(old_to_new_indices[i])); + } + } + while_op.erase(); + return success(); +} + +// Lifts loads/stores from an IfOp or CaseOp's branches. +template +LogicalResult HandleCaseOrIfOp(CaseOrIfOp op, ArrayRef branches) { + // For canonicalized If/Case, there should not be any resource outputs + int64_t non_resource_results = op.getNumResults(); + + llvm::SmallDenseMap resource_arg_uses; + if (failed(FindResourceArgUseInfo(branches.front(), &resource_arg_uses))) + return failure(); + + for (auto func : branches.drop_front()) { + llvm::SmallDenseMap branch_use_info; + if (failed(FindResourceArgUseInfo(func, &branch_use_info))) + return failure(); + // A resource is considered used as long as it is used in either branch. + resource_arg_uses = + MergeArgResourceUseInfo(resource_arg_uses, branch_use_info); + } + + if (resource_arg_uses.empty()) return success(); + // Remove unused resources in functions. + llvm::SmallDenseMap remaining_resource_data_types; + if (failed(RemoveUnusedResourceArgumentsAndForwardedRetvals( + resource_arg_uses, branches.front(), + /*old_to_new_arg_indices=*/nullptr, + &remaining_resource_data_types))) { + return failure(); + } + for (auto func : branches.drop_front()) { + if (failed(RemoveUnusedResourceArgumentsAndForwardedRetvals( + resource_arg_uses, func))) { + return failure(); + } + } + + // Forward resource inputs updated in any branch to the outputs of both + // branches. First prepare the mapping from arg to new update output. + llvm::SmallDenseMap resource_arg_to_new_output; + { + int64_t removed_args = 0; + for (const auto& entry : resource_arg_uses) { + if (!entry.getSecond().used) { + removed_args++; + continue; + } + if (!entry.getSecond().updated) continue; + int64_t new_output_index = + non_resource_results + resource_arg_to_new_output.size(); + resource_arg_to_new_output[entry.getFirst() - removed_args] = + new_output_index; + } + } + + // Append resource updates to the return ops: now they are just forwarded + // input resources, but will be replaced by the data value in + // LiftArgRetResourcesForFunction(). + for (auto branch : branches) { + auto new_retvals = + llvm::to_vector<4>(branch.front().getTerminator()->getOperands()); + new_retvals.resize(new_retvals.size() + resource_arg_to_new_output.size()); + for (const auto& entry : resource_arg_to_new_output) { + int64_t resource_arg_index = entry.getFirst(); + int64_t output_index = entry.getSecond(); + new_retvals[output_index] = branch.getArgument(resource_arg_index); + } + auto old_return = branch.front().getTerminator(); + OpBuilder builder(old_return); + auto new_return = + func::ReturnOp::create(builder, old_return->getLoc(), new_retvals); + old_return->erase(); + (void)LiftArgRetResourcesForFunction( + branch, remaining_resource_data_types, [&](int64_t index, Value value) { + new_return.setOperand(resource_arg_to_new_output[index], value); + }); + } + + // Recreate the op without resource operands. + OpBuilder builder(op); + // Now use the filtered original operands, which will be replaced by + // AddLoadsStoresOutsideControlFlowOp(). + auto new_operands = + FilterRange(op.getInput(), resource_arg_uses); + new_operands.insert(new_operands.begin(), op.getOperand(0)); + func::FuncOp first_func = branches.front(); + auto new_op = CaseOrIfOp::create(builder, op.getLoc(), + first_func.getFunctionType().getResults(), + new_operands, op->getAttrs()); + // Prepare for AddLoadsStoresOutsideControlFlowOp() + llvm::SmallDenseMap> + arg_data_type_and_updated_output_index; + for (const auto& entry : remaining_resource_data_types) { + auto new_output_it = resource_arg_to_new_output.find(entry.getFirst()); + int64_t update_index = new_output_it == resource_arg_to_new_output.end() + ? -1 + : new_output_it->getSecond(); + arg_data_type_and_updated_output_index[entry.getFirst() + 1] = { + entry.getSecond(), update_index}; + } + AddLoadsStoresOutsideControlFlowOp(new_op, + arg_data_type_and_updated_output_index); + // Replace uses. + op.replaceAllUsesWith(new_op.getResults().take_front(op.getNumResults())); + op.erase(); + return success(); +} + +// A resource-lifted function for (potentially multiple) PartitionedCallOps and +// information about the lifting changes. +struct PartitionedCallLiftingInfo { + // Function with resources lifted. Can be nullptr if nothing needs to change. + func::FuncOp lifted_callee; + // Mapping from old resource outputs to their aliasing output inputs. + llvm::SmallDenseMap old_outputs_aliasing_old_inputs; + // Mapping from old to new output indices in case any output is removed. + llvm::SmallVector old_to_new_output_indices; + // ResourceArgUseInfo for each old resource argument. + llvm::SmallDenseMap use_info; + // Input for AddLoadsStoresOutsideControlFlowOp(), see its comment. + llvm::SmallDenseMap> + arg_data_type_and_updated_output_index; +}; + +// Lifts loads/stores from a PartitionedCallOp's callee function. If anything +// needs to be changed, the original function will be preserved, and the lifting +// happens on a clone, which will be stored in `result`. +LogicalResult HandlePartitionedCallOpCallee( + func::FuncOp callee, PartitionedCallLiftingInfo* result) { + // Sanity check: return of resources should be aliases of inputs. Such outputs + // will be removed later. + int64_t non_resource_results = 0; + for (auto entry : + llvm::enumerate(callee.front().getTerminator()->getOperands())) { + auto retval = entry.value(); + if (!mlir::isa(getElementTypeOrSelf(retval.getType()))) { + result->old_to_new_output_indices.push_back(non_resource_results++); + continue; + } + auto aliasing_arg = mlir::dyn_cast(retval); + if (!aliasing_arg) { + return callee.emitOpError("unsupported function call: ") + << "resource return value does not alias an input."; + } + result->old_outputs_aliasing_old_inputs[entry.index()] = + aliasing_arg.getArgNumber(); + result->old_to_new_output_indices.push_back(-1); + } + + if (failed(FindResourceArgUseInfo(callee, &result->use_info))) { + return failure(); + } + if (result->use_info.empty()) { + result->lifted_callee = nullptr; + return success(); + } + + // Clone the callee before making changes. + SmallString<64> name_base = callee.getName(); + auto module = callee->getParentOfType(); + name_base += "_resource_lifted"; + auto name = name_base; + callee = callee.clone(); + callee.setPrivate(); + callee.setName(mlir::StringAttr::get(callee->getContext(), name)); + SymbolTable(module).insert(callee); + result->lifted_callee = callee; + + // Remove unused resources in functions. + llvm::SmallDenseMap remaining_resource_data_types; + if (failed(RemoveUnusedResourceArgumentsAndForwardedRetvals( + result->use_info, callee, /*old_to_new_arg_indices=*/nullptr, + &remaining_resource_data_types))) { + return failure(); + } + for (const auto& entry : remaining_resource_data_types) { + result->arg_data_type_and_updated_output_index[entry.getFirst()] = { + entry.getSecond(), -1}; + } + llvm::SmallVector retval_indices_to_preserve; + for (auto& val : callee.front().getTerminator()->getOpOperands()) { + // Store indices of results that are not resources. + if (!mlir::isa(getElementTypeOrSelf(val.get().getType()))) + retval_indices_to_preserve.push_back(val.getOperandNumber()); + } + int64_t num_retvals = retval_indices_to_preserve.size(); + llvm::SmallVector new_retvals; + // Lift resources. + (void)LiftArgRetResourcesForFunction( + callee, remaining_resource_data_types, [&](int64_t index, Value value) { + result->arg_data_type_and_updated_output_index[index].second = + num_retvals++; + new_retvals.push_back(value); + }); + + auto old_return = callee.front().getTerminator(); + llvm::SmallVector old_and_new_retvals; + old_and_new_retvals.reserve(retval_indices_to_preserve.size() + + new_retvals.size()); + for (int64_t retval_index : retval_indices_to_preserve) + old_and_new_retvals.push_back(old_return->getOperand(retval_index)); + + old_and_new_retvals.append(new_retvals.begin(), new_retvals.end()); + // Replace old return with the new ones with update values. + OpBuilder builder(old_return); + auto new_return = func::ReturnOp::create(builder, old_return->getLoc(), + old_and_new_retvals); + old_return->erase(); + callee.setType(FunctionType::get( + callee.getContext(), callee.getFunctionType().getInputs(), + llvm::to_vector<4>(new_return.getOperandTypes()))); + return success(); +} + +// Updates a PartitionedCallOp/StatefulPartitionedCallOp according to the +// resource-lifted new callee function in lifting_info. +template +void UpdatePartitionedCallOpWithNewCallee( + CallOpType call_op, PartitionedCallLiftingInfo& lifting_info) { + if (!lifting_info.lifted_callee) return; + // Replace output resource uses with the aliasing input, so that we can remove + // this output. + for (const auto& entry : lifting_info.old_outputs_aliasing_old_inputs) { + call_op.getResult(entry.getFirst()) + .replaceAllUsesWith(call_op.getOperand(entry.getSecond())); + } + // Recreate the call op. + OpBuilder builder(call_op); + // Now use the filtered original operands, which will be replaced by + // AddLoadsStoresOutsideControlFlowOp(). + auto new_operands = FilterRange(call_op.getArgs(), + lifting_info.use_info); + auto new_call = CallOpType::create( + builder, call_op.getLoc(), + lifting_info.lifted_callee.getFunctionType().getResults(), new_operands, + call_op->getAttrs()); + new_call->setAttr("f", + SymbolRefAttr::get(builder.getContext(), + lifting_info.lifted_callee.getName())); + AddLoadsStoresOutsideControlFlowOp( + new_call, lifting_info.arg_data_type_and_updated_output_index); + // Replace uses. + for (int64_t i = 0, end = lifting_info.old_to_new_output_indices.size(); + i < end; ++i) { + if (lifting_info.old_to_new_output_indices[i] >= 0) { + call_op.getResult(i).replaceAllUsesWith( + new_call.getResult(lifting_info.old_to_new_output_indices[i])); + } + } + call_op.erase(); +} + +LogicalResult HoistForControlFlow( + Block*, ModuleOp, bool, + llvm::SmallDenseMap*); + +// A templated routine for handling both PartitionedCallOp and +// StatefulPartitionedCallOp. If the callee is already lifted, it just updates +// the caller op itself; otherwise, it first recursively handles nested control +// flow, then performs lifting on the callee. +template +LogicalResult HandlePartitionedCallOp( + CallOpType call_op, func::FuncOp callee, ModuleOp module, + bool vars_initialized, + llvm::SmallDenseMap* + lifted_callees) { + auto emplace_res = lifted_callees->try_emplace(callee.getName(), + PartitionedCallLiftingInfo()); + if (emplace_res.second) { + // Unseen callee. Perform resource lifting on it. + if (failed(HoistForControlFlow(&callee.front(), module, vars_initialized, + lifted_callees))) + return failure(); + + if (failed(HandlePartitionedCallOpCallee( + callee, &emplace_res.first->getSecond()))) { + return failure(); + } + } + UpdatePartitionedCallOpWithNewCallee(call_op, emplace_res.first->getSecond()); + return success(); +} + +// Hoists resource loads/stores from control flow ops in `block` outside the +// body/cond/branch/callee functions. +LogicalResult HoistForControlFlow( + Block* block, ModuleOp module, bool vars_initialized, + llvm::SmallDenseMap* + lifted_partitioned_call_callees) { + if (vars_initialized) SetAllVarIsInitializedToTrue(block); + + for (Operation& op : llvm::make_early_inc_range(*block)) { + if (auto while_op = llvm::dyn_cast(&op)) { + auto body = while_op.body_function(); + auto cond = while_op.cond_function(); + // Recursively handle the nested control flow. + (void)HoistForControlFlow(&body.front(), module, vars_initialized, + lifted_partitioned_call_callees); + (void)HoistForControlFlow(&cond.front(), module, vars_initialized, + lifted_partitioned_call_callees); + if (failed(HandleWhileLoop(while_op, body, cond))) return failure(); + } else if (auto if_op = llvm::dyn_cast(&op)) { + auto then_branch = if_op.then_function(); + auto else_branch = if_op.else_function(); + // Recursively handle the nested control flow. + (void)HoistForControlFlow(&then_branch.front(), module, vars_initialized, + lifted_partitioned_call_callees); + (void)HoistForControlFlow(&else_branch.front(), module, vars_initialized, + lifted_partitioned_call_callees); + if (failed(HandleCaseOrIfOp(if_op, {then_branch, else_branch}))) + return failure(); + } else if (auto case_op = llvm::dyn_cast(&op)) { + SmallVector branch_functions; + case_op.get_branch_functions(branch_functions); + for (func::FuncOp func : branch_functions) { + // Recursively handle the nested control flow. + (void)HoistForControlFlow(&func.front(), module, vars_initialized, + lifted_partitioned_call_callees); + } + if (failed(HandleCaseOrIfOp(case_op, branch_functions))) return failure(); + } else if (auto call_op = llvm::dyn_cast(&op)) { + auto callee = call_op.func(); + if (!callee) { + return call_op.emitOpError( + "resource lifting does not support call with nested references."); + } + if (failed(HandlePartitionedCallOp(call_op, callee, module, + vars_initialized, + lifted_partitioned_call_callees))) { + // Nested control flow handling is done in HandlePartitionedCallOp(). + return failure(); + } + } else if (auto call_op = + llvm::dyn_cast(&op)) { + if (failed(HandlePartitionedCallOp(call_op, call_op.func(), module, + vars_initialized, + lifted_partitioned_call_callees))) { + return failure(); + } + } else if (isa(op)) { + for (Region& region : op.getRegions()) + (void)HoistForControlFlow(®ion.front(), module, vars_initialized, + lifted_partitioned_call_callees); + LogicalResult result = RegionResourceHoister::ReplaceOpWithNewOp(&op); + if (failed(result)) return failure(); + } + } + + // After we have hoisted operations in the block, we may have added new read + // and writes of resources to this block. Clean them up by doing store-load + // forwarding. + ForwardStoreToLoad(block); + return success(); +} + +// Lifts resource operation from tf_device.cluster ops nested in `op` outside. +// Returns failure if there are remaining resource-type values that can not be +// lifted. +void ResourceOpLiftingPass::runOnOperation() { + llvm::SmallDenseMap + lifted_partitioned_call_callees; + ModuleOp module = getOperation(); + + if (failed(TF::CleanupAndCanonicalizeForResourceOpLifting(module))) + return signalPassFailure(); + + auto walk_result = module.walk([&](func::FuncOp func_op) { + return func_op.walk([&](tf_device::ClusterOp cluster) { + LogicalResult result = HoistForControlFlow( + &cluster.GetBody(), module, /*vars_initialized=*/true, + &lifted_partitioned_call_callees); + if (failed(result)) return WalkResult::interrupt(); + result = RegionResourceHoister::ReplaceOpWithNewOp(cluster); + if (failed(result)) return WalkResult::interrupt(); + return WalkResult::advance(); + }); + }); + + if (walk_result.wasInterrupted()) return signalPassFailure(); + + // Clean up and canonicalize to remove dead local variables as some local + // variables might be dead after hoisting resource loads/stores. + if (failed(TF::CleanupAndCanonicalizeForResourceOpLifting(module))) + return signalPassFailure(); +} + +#define GEN_PASS_DEF_RESOURCEOPLIFTINGFORMAINFUNCTIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct ResourceOpLiftingForMainFunctionPass + : public impl::ResourceOpLiftingForMainFunctionPassBase< + ResourceOpLiftingForMainFunctionPass> { + void runOnOperation() override; +}; + +void ResourceOpLiftingForMainFunctionPass::runOnOperation() { + ModuleOp module = getOperation(); + func::FuncOp main_func = module.lookupSymbol("main"); + if (!main_func) { + return; + } + + if (failed(TF::ResourceLiftingForFunctionalControlFlow(main_func))) { + return signalPassFailure(); + } +} + +} // namespace + +namespace TFDevice { +std::unique_ptr> CreateResourceOpLiftingPass() { + return std::make_unique(); +} + +std::unique_ptr> +CreateResourceOpLiftingForMainFunctionPass() { + return std::make_unique(); +} + +} // namespace TFDevice + +namespace TF { +LogicalResult ResourceLiftingForFunctionalControlFlow(func::FuncOp function) { + // This routine should only be called when control flow operations are still + // represented with TF IfOp and WhileOp operations. In this case, there should + // be only one basic blocks in the MLIR representation. + if (!llvm::hasSingleElement(function)) { + return function.emitError() + << "expect the function to have 1 block while it has " + << function.getBlocks().size(); + } + + if (failed(TF::CleanupAndCanonicalizeForResourceOpLifting(function))) + return failure(); + + llvm::SmallDenseMap + lifted_partitioned_call_callees; + if (failed(HoistForControlFlow( + &function.front(), cast(function->getParentOp()), + /*vars_initialized=*/false, &lifted_partitioned_call_callees))) + return failure(); + + // Clean up and canonicalize to remove dead local variables as some local + // variables might be dead after hoisting resource loads/stores from control + // flow ops. + return TF::CleanupAndCanonicalizeForResourceOpLifting(function); +} +} // namespace TF + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.cc new file mode 100644 index 0000000000000000000000000000000000000000..469d11d27b774f2c28bc7227ad0de49806a34806 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.cc @@ -0,0 +1,528 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.h" + +#include +#include + +#include "llvm/ADT/BitVector.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace { + +bool IsResource(Value value) { + return mlir::isa(getElementTypeOrSelf(value.getType())); +} + +// Checks if a cast op is casting a resource -> resource. +bool IsCastOfResource(Operation &op) { + auto cast = dyn_cast(op); + if (!cast) return false; + return IsResource(cast.getX()); +} + +// Removes passthrough ops in the block. The device computation does not need +// such nodes to carry information. +void RemovePassthroughOp(Block &block) { + for (auto &op : llvm::make_early_inc_range(block)) { + if (isa(op) || IsCastOfResource(op)) { + op.replaceAllUsesWith(op.getOperands()); + op.erase(); + } + } +} + +using LocalVarOp = std::variant; + +Value LocalVarOp_resource(LocalVarOp &op) { + if (auto var_handle_op = std::get_if(&op)) { + return var_handle_op->getResource(); + } else { + return std::get(op).getResource(); + } +} + +void LocalVarOp_erase(LocalVarOp &op) { + if (auto var_handle_op = std::get_if(&op)) { + var_handle_op->erase(); + } else { + std::get(op).erase(); + } +} + +std::optional IsLocalVarOp(Operation &op) { + if (TF::MlirLocalVarOp mlir_local_var_op = + llvm::dyn_cast(&op)) { + return std::make_optional(LocalVarOp(mlir_local_var_op)); + } + if (TF::VarHandleOp var_handle_op = llvm::dyn_cast(&op)) { + auto ANONYMOUS_NAME = ::tensorflow::ResourceHandle::ANONYMOUS_NAME; + if (var_handle_op.getSharedName() == ANONYMOUS_NAME) { + return std::make_optional(LocalVarOp(var_handle_op)); + } + } + return {}; +} + +// Eliminate local variables that are only assigned to but never read, and thus +// are dead. +void RemoveDeadLocalVariables(Block &block) { + llvm::SmallVector local_vars; + for (Operation &op : block) { + if (auto local_var = IsLocalVarOp(op)) { + local_vars.push_back(local_var.value()); + } + } + for (auto local_var : local_vars) { + auto users = LocalVarOp_resource(local_var).getUsers(); + if (llvm::all_of(users, [](const Operation *user) { + return isa(user) || + isa(user); + })) { + for (auto user : llvm::make_early_inc_range(users)) user->erase(); + LocalVarOp_erase(local_var); + } + } +} + +LogicalResult CleanupAndCanonicalize(Operation *parent_op); + +// Eliminates unusued results from an operation `op` by cloning it with reduced +// result types and doing appropriate use replacements. `results_to_eliminate` +// is a bitvector of result positions to eliminate. If its null, then all unused +// results of the operation will be eliminated. +void EliminateUnusedResults( + Operation *op, const llvm::BitVector *results_to_eliminate = nullptr) { + auto can_eliminate = [&](OpResult &result) -> bool { + if (!result.use_empty()) return false; + if (results_to_eliminate) + return results_to_eliminate->test(result.getResultNumber()); + else + return true; + }; + SmallVector new_result_types; + for (OpResult result : op->getResults()) { + if (can_eliminate(result)) continue; + new_result_types.push_back(result.getType()); + } + + // Rebuild the new operation with lesser number of results. + OpBuilder builder(op); + Operation *new_op = Operation::create( + op->getLoc(), op->getName(), new_result_types, op->getOperands(), + op->getAttrs(), op->getPropertiesStorage(), op->getSuccessors(), + op->getNumRegions()); + builder.insert(new_op); + + // Move region bodies to the new operation. + for (auto it : llvm::zip(op->getRegions(), new_op->getRegions())) { + Region &old_region = std::get<0>(it); + Region &new_region = std::get<1>(it); + new_region.takeBody(old_region); + } + + // Replace used results and erase the old op. + int next_result_idx = 0; + for (OpResult result : op->getResults()) { + if (can_eliminate(result)) continue; + result.replaceAllUsesWith(new_op->getResult(next_result_idx++)); + } + op->erase(); +} + +// Clones a function if it cannot be patched in place. Clone if there are +// multiple uses or unknown uses (for external functions). The cloned function +// will be marked as private. +func::FuncOp CloneFunctionIfNeeded(func::FuncOp func) { + ModuleOp module = func->getParentOfType(); + auto func_uses = SymbolTable::getSymbolUses(func, &module.getBodyRegion()); + if (func_uses.has_value() && llvm::hasSingleElement(func_uses.value())) + return func; + func::FuncOp cloned = func.clone(); + cloned.setPrivate(); + cloned.setName( + StringAttr::get(func.getContext(), func.getName().str() + "_lifted")); + SymbolTable(module).insert(cloned); + return cloned; +} + +// Eliminates unused results for If/Case operations. Also patches up the +// branch functions to (a) drop the ununsed return values, and (b) as a result +// if some argument becomes unused in all branches, drop that argument and the +// corresponding if/case input operand. +LogicalResult EliminateUnusedResultsForIfCase(Operation *op, + ArrayRef branches) { + // Clone branch functions if needed since we will be mutating them. + SmallVector cloned_branches; + cloned_branches.reserve(branches.size()); + for (func::FuncOp func : branches) { + func::FuncOp cloned = CloneFunctionIfNeeded(func); + cloned_branches.push_back(cloned); + if (cloned == func) continue; + // Patch up the op attribute to point to the new function. + for (NamedAttribute attr : op->getAttrs()) { + auto symref = mlir::dyn_cast(attr.getValue()); + if (!symref) continue; + if (symref.getValue() != func.getName()) continue; + op->setAttr(attr.getName(), + FlatSymbolRefAttr::get(op->getContext(), cloned.getName())); + break; + } + } + + // Traverse results backward so that indices to be deleted stay unchanged. + for (OpResult result : llvm::reverse(op->getResults())) { + if (!result.use_empty()) continue; + int result_idx = result.getResultNumber(); + for (func::FuncOp func : cloned_branches) + func.front().getTerminator()->eraseOperand(result_idx); + } + + // Check which function arguments are unused in all branches. We can drop + // those as well. + int num_args = cloned_branches[0].getNumArguments(); + llvm::BitVector used_args(num_args); + for (func::FuncOp func : branches) { + for (BlockArgument arg : func.getArguments()) { + if (!arg.use_empty()) used_args.set(arg.getArgNumber()); + } + } + + // There are some unused args that we can drop. Also drop the corresponding + // input operand. + if (used_args.count() != num_args) { + // Traverse arguments backward so that indices to be deleted stay unchanged. + for (int idx = num_args - 1; idx >= 0; --idx) { + if (used_args.test(idx)) continue; + for (func::FuncOp func : cloned_branches) { + if (failed(func.eraseArgument(idx))) { + return failure(); + } + } + // For if/case, arg #i of attached function corresponds to operand #i+1 + op->eraseOperand(idx + 1); + } + } + + // Patch up function types (with less number of return values and potentially + // less number of arguments) + for (func::FuncOp func : cloned_branches) { + func.setType( + FunctionType::get(func.getContext(), func.front().getArgumentTypes(), + func.front().getTerminator()->getOperandTypes())); + } + + EliminateUnusedResults(op); + return success(); +} + +// Eliminated unused results from a functional while. +LogicalResult EliminateUnusedResultsForWhile(TF::WhileOp op) { + func::FuncOp cond = op.cond_function(); + func::FuncOp body = op.body_function(); + + llvm::BitVector can_eliminate(op.getNumResults()); + for (OpResult result : llvm::reverse(op.getResults())) { + if (!result.use_empty()) continue; + int result_idx = result.getResultNumber(); + BlockArgument cond_arg = cond.getArgument(result_idx); + BlockArgument body_arg = cond.getArgument(result_idx); + Operation *body_ret = body.front().getTerminator(); + // We can eliminate a result if its unused and the corresponding argument + // is unused in cond and the only use in body is use it as a return value. + if (cond_arg.use_empty() && body_arg.hasOneUse() && + body_arg.use_begin()->getOperandNumber() == result_idx && + body_arg.use_begin()->getOwner() == body_ret) { + can_eliminate.set(result_idx); + } + } + + if (can_eliminate.empty()) return success(); + + func::FuncOp cloned_cond = CloneFunctionIfNeeded(cond); + func::FuncOp cloned_body = CloneFunctionIfNeeded(body); + op.setCondAttr( + FlatSymbolRefAttr::get(op.getContext(), cloned_cond.getName())); + op.setBodyAttr( + FlatSymbolRefAttr::get(op.getContext(), cloned_body.getName())); + + // Drop cond/body args and return value. WhileOp result will be dropped later + // in EliminateUnusedResults. Traverse in reverse order so that indices to be + // deleted stay unchanged. + for (int idx = op.getNumResults() - 1; idx >= 0; --idx) { + if (!can_eliminate.test(idx)) continue; + if (failed(cloned_cond.eraseArgument(idx))) { + return failure(); + } + cloned_body.front().getTerminator()->eraseOperand(idx); + if (failed(cloned_body.eraseArgument(idx))) { + return failure(); + } + } + + // Patch up branch function types. + for (func::FuncOp func : {cloned_cond, cloned_body}) { + func.setType( + FunctionType::get(func.getContext(), func.front().getArgumentTypes(), + func.front().getTerminator()->getOperandTypes())); + } + EliminateUnusedResults(op, &can_eliminate); + return success(); +} + +// For resource results, replace all uses with the resource input to which the +// result is tied to. After this, resource outputs of this op are expected to be +// unused. +LogicalResult ForwardCommonArgToOutput(Operation *op, + ArrayRef branches, + ValueRange branch_args, + bool &has_resource_result) { + // For while, the branch inputs and outputs need to match. + bool io_match = isa(op); + + has_resource_result = false; + // Check if the same input argument number is passed through all functions. + for (OpResult result : op->getResults()) { + if (!IsResource(result)) continue; + + has_resource_result = true; + int result_idx = result.getResultNumber(); + std::optional common_arg_index; + for (func::FuncOp func : branches) { + auto ret = func.front().getTerminator(); + auto block_arg = + mlir::dyn_cast(ret->getOperand(result_idx)); + if (!block_arg) { + return op->emitOpError("result #") + << result_idx << " not tied to function argument for branch @" + << func.getName(); + } + if (!common_arg_index.has_value()) { + common_arg_index = block_arg.getArgNumber(); + } else if (common_arg_index.value() != block_arg.getArgNumber()) { + return op->emitError("result #") + << result_idx + << " is not tied to the same argument across all branches"; + } + } + + if (io_match && result_idx != common_arg_index.value()) { + return op->emitOpError("Result #") + << result_idx << " is tied to argument #" + << common_arg_index.value(); + } + + // Forward the corresponding input to the output + result.replaceAllUsesWith(branch_args[common_arg_index.value()]); + } + return success(); +} + +// Canonicalizes a function if. Forwards input argument to resource results and +// then deletes the resource results. +LogicalResult CanonicalizeFunctionalIfCase(Operation *op, + ArrayRef branches, + ValueRange branch_args) { + for (func::FuncOp func : branches) { + if (failed(CleanupAndCanonicalize(func))) return failure(); + } + + bool has_resource_result = false; + if (failed(ForwardCommonArgToOutput(op, branches, branch_args, + has_resource_result))) + return failure(); + + // If no resource type results were found, no further cleanup needed. + if (!has_resource_result) return success(); + + // Drop unused results. + if (failed(EliminateUnusedResultsForIfCase(op, branches))) { + return failure(); + } + return success(); +} + +// Canonicalizes a functional while. Forwards common argument to results and +// drop resource results if posible. +LogicalResult CanonicalizeFunctionalWhile(TF::WhileOp op) { + for (func::FuncOp func : {op.cond_function(), op.body_function()}) { + if (failed(CleanupAndCanonicalize(func))) return failure(); + } + + // For while, just use the body function to forward operand to result. + bool has_resource_result = false; + if (failed(ForwardCommonArgToOutput(op, {op.body_function()}, + op.getOperands(), has_resource_result))) + return failure(); + // If no resource type results were found, no further cleanup needed. + if (!has_resource_result) return success(); + + // Drop unused results. + if (failed(EliminateUnusedResultsForWhile(op))) { + return failure(); + } + return success(); +} + +// Canonicalizes region based if/case and cluster operations. If the same +// captured resource typed value is used for all region results, then that value +// is forwared to the result and the result is dropped. +LogicalResult CanonicalizeRegionIfCaseCluster(Operation *op) { + // Check if the same value is used for all region results for this output. + bool has_resource_result = false; + for (OpResult result : op->getResults()) { + if (!IsResource(result)) continue; + has_resource_result = true; + int result_idx = result.getResultNumber(); + + Value ret0 = + op->getRegion(0).front().getTerminator()->getOperand(result_idx); + for (Region ®ion : op->getRegions().drop_front()) { + Value ret = region.front().getTerminator()->getOperand(result_idx); + if (ret != ret0) { + return op->emitError("Result #") + << result_idx + << " not tied to the same capture across all regions"; + } + } + result.replaceAllUsesWith(ret0); + } + + if (!has_resource_result) return success(); + + // Eliminate unused region results. Traverse in reverse order so that + // indices to be deleted stay unchanged. + for (OpResult result : llvm::reverse(op->getResults())) { + if (!result.use_empty()) continue; + int result_idx = result.getResultNumber(); + for (Region ®ion : op->getRegions()) + region.front().getTerminator()->eraseOperand(result_idx); + } + EliminateUnusedResults(op); + return success(); +} + +// Canonicalizes a region based while. If the same value is passed through +// the body, the result is replaced with the operand and all argument/results +// and retuns values corresponding to that result are dropped. +LogicalResult CanonicalizeWhileRegion(TF::WhileRegionOp op) { + Region &body = op.getBody(); + Region &cond = op.getCond(); + llvm::BitVector can_eliminate(op.getNumResults()); + + // Traverse in reverse order so that indices to be deleted stay unchanged. + for (OpResult result : llvm::reverse(op.getResults())) { + if (!IsResource(result)) continue; + int result_idx = result.getResultNumber(); + Operation *yield_op = body.front().getTerminator(); + Value yield_operand = yield_op->getOperand(result_idx); + Value while_operand = op.getOperand(result_idx); + Value body_arg = body.getArgument(result_idx); + Value cond_arg = cond.getArgument(result_idx); + if (yield_operand != body_arg && yield_operand != while_operand) { + return op.emitOpError("Result #") << result_idx << " is not tied to arg #" + << result_idx << " of the body"; + } + body_arg.replaceAllUsesWith(while_operand); + cond_arg.replaceAllUsesWith(while_operand); + result.replaceAllUsesWith(while_operand); + body.front().getTerminator()->eraseOperand(result_idx); + body.eraseArgument(result_idx); + cond.eraseArgument(result_idx); + op.getOperation()->eraseOperand(result_idx); + can_eliminate.set(result_idx); + } + EliminateUnusedResults(op, &can_eliminate); + return success(); +} + +// Removes identities and canonicalizes all operations within `parent_op`. +LogicalResult CleanupAndCanonicalize(Operation *parent_op) { + auto walk_result = parent_op->walk([](Operation *op) { + // Cleanup code in attached regions. + for (Region ®ion : op->getRegions()) { + if (!llvm::hasSingleElement(region)) return WalkResult::interrupt(); + RemovePassthroughOp(region.front()); + RemoveDeadLocalVariables(region.front()); + } + + LogicalResult result = success(); + + // While condition cannot write to resource variables. + auto check_while_cond = [&](TF::AssignVariableOp assign) { + op->emitOpError("found resource write in loop condition."); + return WalkResult::interrupt(); + }; + + if (auto if_op = dyn_cast(op)) { + result = CanonicalizeFunctionalIfCase( + op, {if_op.then_function(), if_op.else_function()}, if_op.getInput()); + } else if (auto case_op = dyn_cast(op)) { + SmallVector branches; + case_op.get_branch_functions(branches); + result = + CanonicalizeFunctionalIfCase(case_op, branches, case_op.getInput()); + } else if (auto while_op = dyn_cast(op)) { + if (while_op.cond_function().walk(check_while_cond).wasInterrupted()) + return WalkResult::interrupt(); + result = CanonicalizeFunctionalWhile(while_op); + } else if (isa( + op)) { + result = CanonicalizeRegionIfCaseCluster(op); + } else if (auto while_region = dyn_cast(op)) { + if (while_region.getCond().walk(check_while_cond).wasInterrupted()) + return WalkResult::interrupt(); + // For while region, the body input and output arg should match. + result = CanonicalizeWhileRegion(while_region); + } else if (auto call = dyn_cast(op)) { + func::FuncOp func = dyn_cast(call.resolveCallable()); + if (!func) return WalkResult::interrupt(); + result = CleanupAndCanonicalize(func); + } + return failed(result) ? WalkResult::interrupt() : WalkResult::advance(); + }); + + return failure(walk_result.wasInterrupted()); +} + +} // anonymous namespace + +namespace TF { + +LogicalResult CleanupAndCanonicalizeForResourceOpLifting(func::FuncOp func) { + return CleanupAndCanonicalize(func); +} + +LogicalResult CleanupAndCanonicalizeForResourceOpLifting(ModuleOp module) { + auto walk_result = module.walk([](tf_device::ClusterOp cluster) { + if (failed(CleanupAndCanonicalize(cluster))) return WalkResult::interrupt(); + return WalkResult::advance(); + }); + return failure(walk_result.wasInterrupted()); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.h new file mode 100644 index 0000000000000000000000000000000000000000..8f25404fa25f40422c2cac98361d892904e74be2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/resource_op_lifting_cleanup.h @@ -0,0 +1,47 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_RESOURCE_OP_LIFTING_CLEANUP_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_RESOURCE_OP_LIFTING_CLEANUP_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project + +// Performs IR cleanup and canonicalization in preparation for Resource Op +// Lifting pass. It does several things: +// - Eliminate identity nodes to remove (most) of resource aliasing +// - Canonicalize functional control flow. For functional control flow we +// expect that any resource output of these ops matches the corresponding +// input, and then forward that input to the output. Fails if this is not the +// case. If successful, the following invariants will hold true: +// (a) For if/case, any resource type results will be deleted. +// (b) For while, any resource type results will be unused. +// - Canonicalize region based control flow. Again, any resource outputs are +// expected to be resolved to be one of the captured resource inputs. Fails +// if this is not the case. If successful, the following invariants will hold +// true: +// (a) For if/case, any resource type results will be deleted. +// (b) For while, any resource type results will be unused. +namespace mlir { +namespace TF { +LogicalResult CleanupAndCanonicalizeForResourceOpLifting(ModuleOp module); +LogicalResult CleanupAndCanonicalizeForResourceOpLifting(func::FuncOp func); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_RESOURCE_OP_LIFTING_CLEANUP_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_tpu_embedding_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_tpu_embedding_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..5d743172dec65931f243f3f55212fdb42730d6a2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_tpu_embedding_ops.cc @@ -0,0 +1,127 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_REWRITETPUEMBEDDINGOPSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Rewrites RecvTPUEmbeddingActivationsOp and SendTPUEmbeddingGradients ops to +// internal variants by introducing XlaRecvTPUEmbeddingDeduplicationData op. +struct RewriteTPUEmbeddingOps + : public impl::RewriteTPUEmbeddingOpsPassBase { + void runOnOperation() override; +}; + +// Rewrites the given op to `OpT` op after adding the given operand at the end. +template +OpT AddOperandAndRewriteAs(Operation* op, Value operand, NamedAttrList attr, + OpBuilder* builder) { + builder->setInsertionPoint(op); + auto operands = llvm::to_vector<4>(op->getOperands()); + operands.push_back(operand); + auto new_op = OpT::create(*builder, op->getLoc(), op->getResultTypes(), + operands, attr.getAttrs()); + op->replaceAllUsesWith(new_op.getOperation()->getResults()); + op->erase(); + return new_op; +} + +// Returns success if the function has at most one op of the template type and +// assigns it to `result`, if present. If there are multiple such ops, returns +// failure. +template +LogicalResult GetOp(Region* region, OpT* result) { + *result = {}; + for (auto op : region->getOps()) { + if (*result) return op.emitError("should be unique within a function"); + *result = op; + } + return success(); +} + +LogicalResult RunOnRegion(Region* region) { + RecvTPUEmbeddingActivationsOp recv_op; + if (failed(GetOp(region, &recv_op))) return failure(); + + SendTPUEmbeddingGradientsOp send_op; + if (failed(GetOp(region, &send_op))) return failure(); + + // No TPU embedding ops. + if (!recv_op && !send_op) return success(); + + Location loc = recv_op ? recv_op.getLoc() : send_op.getLoc(); + StringRef config = recv_op ? recv_op.getConfig() : send_op.getConfig(); + + // Create XlaRecvTPUEmbeddingDeduplicationData op. + OpBuilder builder(region); + auto output_ty = + RankedTensorType::get({}, VariantType::get(region->getContext())); + auto dedup_op = XlaRecvTPUEmbeddingDeduplicationDataOp::create( + builder, loc, output_ty, config); + + // Rewrite RecvTPUEmbeddingActivations op to the corresponding internal op. + if (recv_op) + AddOperandAndRewriteAs( + recv_op, dedup_op, recv_op->getAttrs(), &builder); + + // Rewrite SendTPUEmbeddingGradients op to the corresponding internal op and + // then update the OperandSegmentSize attribute. + if (send_op) { + int32_t operand_sizes[] = {static_cast(send_op.getN()), + static_cast(send_op.getNN()), 1}; + auto operand_size_attr = builder.getDenseI32ArrayAttr(operand_sizes); + NamedAttrList attrs(send_op->getAttrs()); + attrs.set(send_op.getOperandSegmentSizeAttr(), operand_size_attr); + + AddOperandAndRewriteAs(send_op, dedup_op, + attrs, &builder); + } + return success(); +} + +void RewriteTPUEmbeddingOps::runOnOperation() { + func::FuncOp func = getOperation(); + if (failed(RunOnRegion(&func.getBody()))) return signalPassFailure(); + + func.walk([&](Operation* op) { + for (Region& region : op->getRegions()) { + if (failed(RunOnRegion(®ion))) return signalPassFailure(); + } + }); +} + +} // anonymous namespace + +std::unique_ptr> +CreateRewriteTPUEmbeddingOpsPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..6293956e5dfb9047e5f643d72b6b5499a65dac54 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.cc @@ -0,0 +1,86 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h" + +#include +#include + +#include "mlir/IR/Value.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TF { + +namespace { + +const char kDeviceAttr[] = "device"; +const char kDeviceGpu[] = "GPU"; + +std::optional GetOpDevice(mlir::Operation *op) { + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) { + return std::nullopt; + } + tensorflow::DeviceNameUtils::ParsedName parsed_name; + if (!tensorflow::DeviceNameUtils::ParseFullName(device.str(), &parsed_name)) { + return std::nullopt; + } + if (!parsed_name.has_type) { + return std::nullopt; + } + return parsed_name.type; +} + +} // namespace + +bool IsOnGpuDevice(mlir::Operation *op) { + std::optional device = GetOpDevice(op); + if (!device) return false; + return *device == kDeviceGpu; +} + +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::OpResult src, + mlir::OpResult dest) { + CopyDeviceAndUnderscoredAttributesAdaptor(src.getOwner(), dest.getOwner()); +} + +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::Operation *src, + mlir::OpResult dest) { + CopyDeviceAndUnderscoredAttributesAdaptor(src, dest.getOwner()); +} + +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::Operation *src, + mlir::Operation *dest) { + CopyDeviceAndUnderscoredAttributes(src, dest); +} + +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::OpResult src, + mlir::OpResult dest) { + CopyXlaOutsideCompilationAttributesAdaptor(src.getOwner(), dest.getOwner()); +} + +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::Operation *src, + mlir::OpResult dest) { + CopyXlaOutsideCompilationAttributesAdaptor(src, dest.getOwner()); +} + +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::Operation *src, + mlir::Operation *dest) { + CopyXlaOutsideCompilationAttributes(src, dest); +} +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h new file mode 100644 index 0000000000000000000000000000000000000000..ad4cf0dbe48729f15b1608e0cfa15586a4606e46 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.h @@ -0,0 +1,95 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL_H_ + +#include + +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Returns int, float or complex DenseElementsAttr with scalar shape with the +// given element type and the integer value. +template +DenseElementsAttr GetScalarOfType(Type ty, T raw_value) { + RankedTensorType scalar_ty = RankedTensorType::get({}, ty); + if (auto float_ty = mlir::dyn_cast(ty)) { + FloatAttr attr = FloatAttr::get(float_ty, raw_value); + return DenseElementsAttr::get(scalar_ty, attr); + } else if (auto int_ty = mlir::dyn_cast(ty)) { + IntegerAttr attr = IntegerAttr::get(int_ty, raw_value); + return DenseElementsAttr::get(scalar_ty, attr); + } else if (auto complex_ty = mlir::dyn_cast(ty)) { + Type complex_element_ty = complex_ty.getElementType(); + if (complex_element_ty.isF32()) { + return DenseElementsAttr::get( + scalar_ty, static_cast>(raw_value)); + } else if (complex_element_ty.isF64()) { + return DenseElementsAttr::get( + scalar_ty, static_cast>(raw_value)); + } + } + llvm_unreachable("unsupported type"); +} + +// Returns true if `value` is compile-time constant and its splat value equals +// to `raw_value`. +template +bool IsConstantValueOf(Value value, T raw_value) { + auto element_type = mlir::cast(value.getType()).getElementType(); + if (mlir::isa(element_type)) { + DenseFPElementsAttr float_attr; + if (matchPattern(value, m_Constant(&float_attr)) && float_attr.isSplat() && + float_attr.getSplatValue().isExactlyValue(raw_value)) + return true; + } else if (mlir::isa(element_type)) { + DenseIntElementsAttr int_attr; + if (matchPattern(value, m_Constant(&int_attr)) && int_attr.isSplat() && + int_attr.getSplatValue() == raw_value) + return true; + } + + return false; +} + +// Returns true if `op` is placed on GPU device, and false if it's on other +// devices or the device is not specified. +bool IsOnGpuDevice(mlir::Operation *op); + +// Wrappers for CopyDeviceAndUnderscoredAttributes +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::OpResult src, + mlir::OpResult dest); +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::Operation *src, + mlir::OpResult dest); +void CopyDeviceAndUnderscoredAttributesAdaptor(mlir::Operation *src, + mlir::Operation *dest); + +// Wrappers for CopyXlaOutsideCompilationAttributes +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::OpResult src, + mlir::OpResult dest); +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::Operation *src, + mlir::OpResult dest); +void CopyXlaOutsideCompilationAttributesAdaptor(mlir::Operation *src, + mlir::Operation *dest); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td new file mode 100644 index 0000000000000000000000000000000000000000..b7409fe8a342693aae6f9e63ac0a6bbc605c8f44 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/rewrite_util.td @@ -0,0 +1,34 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL + +include "mlir/IR/OpBase.td" +include "mlir/IR/PatternBase.td" + +// Here, the element type can be any integer or float type. But, note that only +// 32 bit integers are supported for the values. +class GetScalarOfType : NativeCodeCall< + "GetScalarOfType(getElementTypeOrSelf($0)," # value # ")">; + +// Here, the element type can be any integer or float type. But, note that only +// 32 bit integers are supported for the values. +class IsConstantValueOf : Constraint>; + +def IsOnGpuDevicePred : CPred<"IsOnGpuDevice($0.getOwner())">; + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_REWRITE_UTIL diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.cc new file mode 100644 index 0000000000000000000000000000000000000000..9f1cefad7a0182444278993d8d86650832d14a32 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.cc @@ -0,0 +1,157 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h" + +#include +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project // IWYU pragma: keep +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/WalkResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/layout.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/shape.h" +#include "xla/tpu/c_api_conversions.h" +#include "xla/tpu/c_api_decl.h" +#include "xla/tpu/tpu_executor_api.h" +#include "xla/tpu/tpu_executor_c_api.h" + +namespace mlir { + +static FailureOr> GetTPUInfeedLayoutFromAPI( + RankedTensorType t) { + // Call the TPU API to determine the right infeed layout. Note that + // this can fail if we're not running on a TPU-enabled node. + // TODO(kramm): Move this into a separate pass. See b/184944903 + xla::Shape old_shape = xla::TypeToShape(t); + XLA_Shape old_shape_c = {}; // NOLINT + XLA_Shape new_shape_c = {}; + if (!stream_executor::tpu::IsInitialized()) { + return failure(); + } + const TfTpu_ExecutorApiFn* executor = stream_executor::tpu::ExecutorApiFn(); + ApiConverter::ToC(old_shape, &old_shape_c); + executor->TpuTransferManager_GetInfeedLayoutFn(&old_shape_c, &new_shape_c); + xla::Shape new_shape = ApiConverter::FromC(&new_shape_c); + ApiConverter::Destroy(&old_shape_c); + ApiConverter::Destroy(&new_shape_c); + + auto minor_to_major = new_shape.layout().minor_to_major(); + return std::vector(minor_to_major.begin(), minor_to_major.end()); +} + +FailureOr GetTPUInfeedLayout(const ArrayRef types, + OpBuilder& rewriter) { + auto i64_type = rewriter.getIntegerType(64); + if (types.size() > 1) { + llvm::SmallVector v; + v.reserve(types.size()); + for (const mlir::Type& t : types) { + if (mlir::isa(t)) continue; + auto layout = GetTPUInfeedLayout({t}, rewriter); + if (failed(layout)) return failure(); + v.push_back(layout.value()); + } + ArrayRef shape(v); + return rewriter.getArrayAttr(shape); + } else if (mlir::isa(types[0])) { + auto tuple_type = mlir::dyn_cast(types[0]); + const auto& types = tuple_type.getTypes(); + llvm::SmallVector v; + v.reserve(types.size()); + for (const mlir::Type& t : types) { + if (mlir::isa(t)) continue; + auto layout = GetTPUInfeedLayout({t}, rewriter); + if (failed(layout)) return failure(); + v.push_back(layout.value()); + } + ArrayRef shape(v); + return rewriter.getArrayAttr(shape); + } else if (auto t = mlir::dyn_cast(types[0])) { + if (!t.hasStaticShape()) return failure(); + auto layout = GetTPUInfeedLayoutFromAPI(t); + std::vector minor_to_major; + if (succeeded(layout)) { + minor_to_major = layout.value(); + } else { + /* If we're not running on a TPU node, we might not be able to + * actually call the part of the TPU API that gives us layout. + * This happens e.g. for unit tests. Below we just create a reasonable + * layout. We sort by dimension size, which makes the layout agree with + * the "correct" TPU layout in surprisingly many cases. + * Note that the corresponding InfeedEnqueue op will be generated + * through another path, and might still generate an (incompatible) + * layout using the TPU API. Running legalize_tf.cc on non-TPU nodes + * thus is a potential source of bugs. + */ + minor_to_major.resize(t.getRank()); + std::iota(minor_to_major.begin(), minor_to_major.end(), 0); + std::sort(minor_to_major.begin(), minor_to_major.end(), + [=](int64_t a, int64_t b) { + int64_t da = t.getDimSize(a); + int64_t db = t.getDimSize(b); + return da > db || (da == db && a > b); + }); + } + std::vector elements; + elements.reserve(minor_to_major.size()); + for (auto e : minor_to_major) { + elements.push_back(rewriter.getIntegerAttr(i64_type, e)); + } + return rewriter.getArrayAttr(elements); + } else { + // types.size() == 1 and types[0] == TokenType + // For this case, we return an empty array attribute. + return rewriter.getArrayAttr({}); + } +} + +bool SetTPUInfeedLayout(mlir::ModuleOp mlir_module) { + auto res = mlir_module->walk([&](mlir::TF::InfeedDequeueTupleOp op) { + mlir::OpBuilder builder(op.getContext()); + std::vector result_types; + result_types.reserve(op.getResultTypes().size()); + + for (mlir::Type t : op.getResultTypes()) { + auto ty = mlir::cast(t); // NOLINT + if (!ty.hasStaticShape()) return mlir::WalkResult::interrupt(); // NOLINT + result_types.push_back(t); + } + + auto layout = mlir::GetTPUInfeedLayout( + mlir::TupleType::get(builder.getContext(), result_types), builder); + if (failed(layout)) return mlir::WalkResult::interrupt(); + // Do not append a UnitAttr for the "token" operand here to avoid + // compilation failure when exporting the "layouts" attribute to a graph + // node. Instead, add the UnitAttr during LegalizeTF pass. + op->setAttr("layouts", layout.value()); + + return mlir::WalkResult::advance(); + }); + return !res.wasInterrupted(); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h new file mode 100644 index 0000000000000000000000000000000000000000..05f5aa3e485c0eed60e127fb292c3160931674cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h @@ -0,0 +1,32 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SET_TPU_INFEED_LAYOUT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SET_TPU_INFEED_LAYOUT_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project + +namespace mlir { + +// Set layouts attribute of tf.InfeedDequeueTuple ops. +bool SetTPUInfeedLayout(ModuleOp mlir_module); + +// Try to determine the right TPU infeed layout. +FailureOr GetTPUInfeedLayout(ArrayRef types, + OpBuilder& rewriter); + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SET_TPU_INFEED_LAYOUT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.cc new file mode 100644 index 0000000000000000000000000000000000000000..ce47fff7e17cc138d894ede36020f9556bddfd44 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.cc @@ -0,0 +1,3452 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinDialect.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/FoldInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/FunctionInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/InferTypeOpInterface.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/FoldUtils.h" // from @llvm-project +#include "stablehlo/dialect/Serialization.h" // from @stablehlo +#include "stablehlo/dialect/Version.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/tf2xla/kernels/xla_call_module_loader.h" +#include "xla/hlo/translate/hlo_to_mhlo/hlo_utils.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/service/shape_inference.h" +#include "xla/shape.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/util/env_var.h" +#include "xla/window_util.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/ir/types/dialect.h" + +#define DEBUG_TYPE "tf-shape-inference" + +#define DCOMMENT(MSG) LLVM_DEBUG(llvm::dbgs() << MSG << "\n") +#define DCOMMENT_OP(OP, MSG) \ + LLVM_DEBUG(OP->print(llvm::dbgs() << MSG << " "); llvm::dbgs() << "\n") + +using ::int64_t; +using mlir::func::FuncOp; +using tensorflow::shape_inference::DimensionHandle; +using tensorflow::shape_inference::InferenceContext; +using tensorflow::shape_inference::ShapeHandle; + +namespace mlir { +namespace TF { +namespace { + +MLIRContext::Threading GetMlirContextThreading() { + bool enable_single_thread_mlir_context = []() { + bool result = false; + if (auto status = tsl::ReadBoolFromEnvVar(kMLIRContextSingleThreadVar, + /*default_val=*/false, &result); + status.ok()) { + return result; + } + return false; + }(); + return enable_single_thread_mlir_context ? MLIRContext::Threading::DISABLED + : MLIRContext::Threading::ENABLED; +} + +// Compute a refined type between two types `lhs` and `rhs`, the result type +// is always at least as refined as (i.e. has more static information) than +// `lhs` This method will actually merge the information contained in the types, +// it is capable of refining: +// tensor>> +// and: +// tensor>> +// into: +// tensor>> +// +// In case of inconsistencies (rank disagreement for example), it returns `lhs`. +Type TypeMeet(Type lhs, Type rhs) { + DCOMMENT("RefineTypeWith : " << lhs << " : " << rhs); + if (lhs == rhs) return lhs; + + auto rhs_shape_type = mlir::dyn_cast(rhs); + if (!rhs_shape_type) return lhs; + auto lhs_shape_type = mlir::cast(lhs); + if (lhs_shape_type.hasRank() && rhs_shape_type.hasRank() && + lhs_shape_type.getRank() != rhs_shape_type.getRank()) { + DCOMMENT("Unexpected rank mismatch: " << lhs << " vs " << rhs); + return lhs; + } + + SmallVector shape; + bool refined_shape = false; + // Build the shape of the refined type, if lhs is unranked it + // will be directly the shape of the refined type, otherwise we merged by + // taking the most specialized. This combines `10x?x?` and `?x?x8` into + // `10x?x8`. + if (!lhs_shape_type.hasRank()) { + if (rhs_shape_type.hasRank()) { + shape.append(rhs_shape_type.getShape().begin(), + rhs_shape_type.getShape().end()); + refined_shape = true; + } + } else if (rhs_shape_type.hasRank()) { + for (auto shape_elts : llvm::enumerate( + llvm::zip(lhs_shape_type.getShape(), rhs_shape_type.getShape()))) { + if (ShapedType::isDynamic(std::get<0>(shape_elts.value())) && + !ShapedType::isDynamic(std::get<1>(shape_elts.value()))) { + shape.push_back(std::get<1>(shape_elts.value())); + refined_shape = true; + DCOMMENT("-> refining shape element #" << shape_elts.index()); + } else { + DCOMMENT("-> not refining shape element #" << shape_elts.index()); + shape.push_back(std::get<0>(shape_elts.value())); + } + } + } + + // Some tensor have an element type wrapping a subtensor, like resource and + // variants. In this case we may recurse on the wrapped subtype. + // `element_type` will contain the refined inferred element type for the + // returned type. + auto lhs_element_type = lhs_shape_type.getElementType(); + auto rhs_element_type_with_subtype = + mlir::dyn_cast( + rhs_shape_type.getElementType()); + // Look for resource or variant element type and ensure we refine the subtype. + // We only support a single subtype at the moment, we won't handle something + // like: + // tensor, tensor<8xf32>> + if (rhs_element_type_with_subtype && + rhs_element_type_with_subtype.GetSubtypes().size() == 1) { + auto lhs_element_type_with_subtype = + mlir::dyn_cast(lhs_element_type); + TensorType subtype; + if (!lhs_element_type_with_subtype) { + DCOMMENT( + "Unexpected inferred `TensorFlowTypeWithSubtype` when original " + "result isn't"); + } else if (lhs_element_type_with_subtype.GetSubtypes().size() > 1) { + DCOMMENT( + "Unexpected `TensorFlowTypeWithSubtype` original type with size>1"); + } else if (lhs_element_type_with_subtype.GetSubtypes().empty()) { + subtype = rhs_element_type_with_subtype.GetSubtypes().front(); + } else { + // Recurse on the subtypes in the variant/resource. Basically if the input + // were: + // tensor>> + // and: + // tensor>> + // we'll try here to refine tensor with tensor<10x8xf32>. + auto refined_subtype = mlir::cast( + TypeMeet(lhs_element_type_with_subtype.GetSubtypes().front(), + rhs_element_type_with_subtype.GetSubtypes().front())); + if (refined_subtype != + lhs_element_type_with_subtype.GetSubtypes().front()) + subtype = refined_subtype; + } + // If we managed to refine the subtype, recreate the element type itself + // (i.e. the tf.variant or tf.resource). + if (subtype) { + lhs_element_type = lhs_element_type_with_subtype.clone({subtype}); + } + } + if (refined_shape || lhs_element_type != lhs_shape_type.getElementType()) { + Type new_type; + if (!lhs_shape_type.hasRank() && !rhs_shape_type.hasRank()) + new_type = UnrankedTensorType::get(lhs_element_type); + else + new_type = lhs_shape_type.clone(shape, lhs_element_type); + DCOMMENT("Refined to: " << new_type); + return new_type; + } + DCOMMENT("No refinement " << lhs); + return lhs; +} + +// Returns whether `original_type` type can be refined with +// `potential_refined_type` type. +bool CanRefineTypeWith(Type original_type, Type potential_refined_type) { + return original_type != TypeMeet(original_type, potential_refined_type); +} + +// Returns if the shape inference pass supports an op outside the TF dialect. +bool IsSupportedNonTFOp(Operation* op) { + return isa(op) || + isa(op); +} + +// Returns whether a cast back would need to be inserted, e.g., whether the +// operation of which use is an operand allows for shape refinement without +// a cast. +bool NeedsCastBack(OpOperand& use, Dialect* tf_dialect) { + return use.getOwner()->getDialect() != tf_dialect && + !IsSupportedNonTFOp(use.getOwner()); +} + +TensorType CreateTensorType(std::optional> shape, + Type element_type) { + if (shape.has_value()) + return tensorflow::GetTypeFromTFTensorShape(shape.value(), element_type); + return UnrankedTensorType::get(element_type); +} + +// Returns true if the op creates a TensorList. +bool IsTensorListInitOp(Operation* op) { + return isa(op) || isa(op) || + isa(op); +} + +// Returns the `element_shape` operand of the ops that create a TensorList. +Value GetElementShapeOperand(Operation* op) { + if (auto empty_tl = dyn_cast(op)) + return empty_tl.getElementShape(); + if (auto tl_reserve = dyn_cast(op)) + return tl_reserve.getElementShape(); + if (auto tl_from_tensor = dyn_cast(op)) + return tl_from_tensor.getElementShape(); + llvm_unreachable("unsupported TensorList op"); +} + +// Utility function to create a ranked tensor type after dropping the first +// dimension from the input type. +RankedTensorType DropFirstDimension(Type type) { + RankedTensorType ranked_type = mlir::dyn_cast(type); + if (!ranked_type) return {}; + llvm::ArrayRef dims_except_first = + ranked_type.getShape().drop_front(); + return tensorflow::GetTypeFromTFTensorShape(dims_except_first, + ranked_type.getElementType()); +} + +Operation* InsertCast(OpBuilder& b, Location loc, Type dst_type, Value input) { + Type element_type = getElementTypeOrSelf(dst_type); + if (mlir::isa(element_type)) + return tensor::CastOp::create(b, loc, dst_type, input); + if (isa(element_type.getDialect())) + return TF::CastOp::create(b, loc, dst_type, input, + /*truncate=*/b.getBoolAttr(false)); + return nullptr; +} + +// Follow the use chain of TensorList and return true iff all elements written +// to TensorList have same static shape. If all elements have same shape, assign +// it to `potential_element_type`. +// +// This can handle multiple mutations of a TensorList object and would return +// true if across all mutations the elements written have the same shape. +bool CanInferTensorListElementType(Value tensorlist, + Value initial_element_shape, + RankedTensorType* potential_element_type) { + DCOMMENT("CanInferTensorListElementType " << tensorlist << " with initial " + << initial_element_shape); + // Verifies if the new element type has static shape and matches the potential + // type passed from caller. Updates the potential_element_type, if not defined + // yet. + auto verify_and_update_potential_element_type = + [&](RankedTensorType new_element_type) -> bool { + DCOMMENT("\t\tConsidering " << new_element_type << " with old " + << *potential_element_type); + if (!new_element_type || !new_element_type.hasStaticShape()) return false; + if (!*potential_element_type) { + DCOMMENT("\t\tUpdating potential_element_type " << new_element_type); + *potential_element_type = new_element_type; + return true; + } + return *potential_element_type == new_element_type; + }; + + std::stack worklist; + worklist.emplace(tensorlist); + + // Track the set of values we've already visited to avoid exponential blowup. + absl::flat_hash_set visited; + auto add_to_worklist = [&worklist, &visited](Value v) { + if (visited.find(v.getAsOpaquePointer()) == visited.end()) { + worklist.emplace(v); + visited.emplace(v.getAsOpaquePointer()); + } + }; + + while (!worklist.empty()) { + tensorlist = worklist.top(); + worklist.pop(); + + // TensorLists are semantically immutable. For example, TensorListSetItem + // takes a TensorList as input and produces a TensorList as output. So to + // traverse modifications to TensorList and verify that all elements written + // to it have the same shape, we need to follow use-def chain of ops that + // (conceptually) modify it i.e., ops that take an input TensorList and + // produce an output TensorList. + for (auto& use : tensorlist.getUses()) { + if (auto push = llvm::dyn_cast(use.getOwner())) { + auto element_type = + mlir::dyn_cast(push.getTensor().getType()); + if (!verify_and_update_potential_element_type(element_type)) + return false; + add_to_worklist(push.getOutputHandle()); + continue; + } + if (auto scatter = llvm::dyn_cast( + use.getOwner())) { + // For scatter op we can get the element shape by dropping the first + // dimension of the input tensor. + RankedTensorType element_type = + DropFirstDimension(scatter.getTensor().getType()); + if (!verify_and_update_potential_element_type(element_type)) + return false; + add_to_worklist(scatter.getOutputHandle()); + continue; + } + if (auto set_item = llvm::dyn_cast(use.getOwner())) { + auto element_type = + mlir::dyn_cast(set_item.getItem().getType()); + DCOMMENT("\tTensorListSetItemOp " << element_type); + if (!verify_and_update_potential_element_type(element_type)) + return false; + add_to_worklist(set_item.getOutputHandle()); + continue; + } + if (auto pop = llvm::dyn_cast(use.getOwner())) { + add_to_worklist(pop.getOutputHandle()); + continue; + } + if (auto resize = llvm::dyn_cast(use.getOwner())) { + add_to_worklist(resize.getOutputHandle()); + continue; + } + // WhileRegionOp can explicitly capture TensorList value to be used inside + // its regions. So we check the uses of corresponding block argument in + // each region and the use of TensorList returned using YieldOp. + if (auto while_region = llvm::dyn_cast(use.getOwner())) { + DCOMMENT("\tTL WhileRegion"); + for (auto branch : while_region.getRegions()) + add_to_worklist(branch->getArgument(use.getOperandNumber())); + continue; + } + if (auto yield = llvm::dyn_cast(use.getOwner())) { + Operation* parent = yield->getParentOp(); + add_to_worklist(parent->getResult(use.getOperandNumber())); + continue; + } + // TODO(jpienaar): This can be generalized. + if (isa(use.getOwner())) { + add_to_worklist(use.getOwner()->getResult(use.getOperandNumber())); + continue; + } + // Refining the tensor list element type might change the output of + // TensorListElementShape which is expected to be the originally assigned + // shape to TensorList init ops. So replace it with the original element + // shape value. + if (auto tl_element_shape = + dyn_cast(use.getOwner())) { + // If element types match, we can do a direct replacement. + if (getElementTypeOrSelf(tl_element_shape.getResult()) == + getElementTypeOrSelf(initial_element_shape.getType())) { + tl_element_shape.replaceAllUsesWith(initial_element_shape); + } else { + OpBuilder b(use.getOwner()); + Operation* cast_op = InsertCast( + b, use.getOwner()->getLoc(), + tl_element_shape.getResult().getType(), initial_element_shape); + if (!cast_op) return false; + tl_element_shape.replaceAllUsesWith(cast_op->getResult(0)); + } + continue; + } + // Ignore ops that just consume a TensorList and do not output another + // TensorList. + if (isa(use.getOwner())) + continue; + + // For any other unknown users of the TensorList, we are conservative and + // stop element shape inference. + DCOMMENT("TensorListType infer, unknown op " << *use.getOwner()); + return false; + } + } + return true; +} + +// Returns the tensor type created from the `shape_attr` and `type_attr` +// attributes. +Type GetType(Attribute shape_attr, Attribute type_attr) { + auto shape = mlir::cast(shape_attr); + auto type = mlir::cast(type_attr); + if (shape.hasRank()) + return tensorflow::GetTypeFromTFTensorShape(shape.getShape(), + type.getValue()); + else + return UnrankedTensorType::get(type.getValue()); +} +} // namespace + +// Create a MLIRContext based on the threading setup in the env var. +std::unique_ptr MakeMLIRContextWithThreading() { + return std::make_unique(GetMlirContextThreading()); +} + +// Returns whether type can be further refined. +bool CanBeRefined(Type type) { + auto shape_type = mlir::dyn_cast(type); + if (!shape_type) return false; + + // Returns whether type with subtypes can be further refined. + auto can_refine_subtypes = [](TF::TensorFlowTypeWithSubtype tws) { + return tws.GetSubtypes().empty() || + llvm::any_of(tws.GetSubtypes(), CanBeRefined); + }; + auto type_with_subtype = mlir::dyn_cast( + shape_type.getElementType()); + if (type_with_subtype && can_refine_subtypes(type_with_subtype)) return true; + + return !shape_type.hasStaticShape(); +} + +// Returns a new arg type based on the shape and element type. If there are +// dynamic bounds attribute to the arg, update the bounds based on the shape +// as well. +Type GetNewArgType(Type old_arg_type, ArrayRef shape, + Type element_type, mlir::MLIRContext* context) { + Type new_arg_type = tensorflow::GetTypeFromTFTensorShape(shape, element_type); + + if (auto input_ty = mlir::dyn_cast(old_arg_type)) { + ArrayRef bounds = hlo::encodingToBounds(input_ty.getEncoding()); + // The input type has bounded dynamic dimension. + if (!bounds.empty()) { + SmallVector new_bounds(bounds.begin(), bounds.end()); + SmallVector new_shape(shape.begin(), shape.end()); + // If dimension of the input type is dynamic. Update the + // bounds of the dim with the new type if needed. + for (int i = 0; i < input_ty.getShape().size(); i++) { + if (hlo::isDynamicDimSize(input_ty.getShape()[i])) { + new_bounds[i] = new_shape[i]; + new_shape[i] = ShapedType::kDynamic; + } + } + new_arg_type = tensorflow::GetTypeFromTFTensorShape( + new_shape, element_type, + mlir::mhlo::TypeExtensionsAttr::get(context, new_bounds)); + } + } + return new_arg_type; +} + +// Combination of value producer and port of value produced (e.g., +// :, +// so for tf.Const -> tensor<10x20xf32>, [0,2,18] would point to a unique output +// scalar value). +struct ValuePort { + PointerUnion producer; + SmallVector port; + + bool operator==(const ValuePort& other) const { + return producer == other.producer && port == other.port; + } + + ValuePort() = default; + + // Convert output value to ValuePort. + explicit ValuePort(Value v) { + OpResult opr = mlir::dyn_cast(v); + if (opr) { + producer = opr.getOwner(); + port = {opr.getResultNumber()}; + } else { + producer = mlir::cast(v); + port = {0}; + } + } + ValuePort(PointerUnion producer, + SmallVector port) + : producer(producer), port(port) {} + + raw_ostream& print(raw_ostream& os) const { + if (auto* op = producer.dyn_cast()) + os << "op " << op->getName(); + if (auto ba = producer.dyn_cast()) + os << "block_arg " << ba.getArgNumber(); + os << formatv(" [{0}]", llvm::make_range(port.begin(), port.end())); + return os; + } + + bool IsValid() const { return !producer.isNull(); } +}; + +struct ValuePortHasher { + std::size_t operator()(const ValuePort& other) const { + return hash_combine(llvm::hash_value(other.producer.getOpaqueValue()), + hash_value(ArrayRef(other.port))); + } +}; + +using ValuePortResultMap = + absl::flat_hash_map; +using ComputedQueryFn = function_ref; +using ValueQueryFn = function_ref; +using ValuePortInputs = SmallVectorImpl; + +// Note: Following implements the rank 1 pack op case so could be +// generalized. +// +// Maps the specified component in the `port` of the given op's result to one of +// the element in the input. +ValuePort ComputeInputComponentFor(PackOp op, ArrayRef port) { + auto type = mlir::cast(op.getType()); + if (!type.hasRank() || type.getRank() != 1) return {}; + if (port.size() != 2) return {}; + assert(port[0] == 0); + return ValuePort(op.getOperand(port[1])); +} + +ValuePort ComputeInputComponentFor(ConcatV2Op op, ArrayRef port) { + if (port.size() != 2) return {}; + assert(port[0] == 0); + + int64_t element_idx = port[1]; + for (Value val : op.getValues()) { + auto val_ty = mlir::cast(val.getType()); + if (!val_ty.hasStaticShape() || val_ty.getRank() != 1) return {}; + + int64_t dim_size = val_ty.getNumElements(); + if (element_idx >= dim_size) { + element_idx -= dim_size; + continue; + } + + ValuePort req(val); + req.port.push_back(element_idx); + return req; + } + return {}; +} + +ValuePort ComputeInputComponentFor(GatherV2Op op, ArrayRef port) { + if (port.size() != 2) return {}; + assert(port[0] == 0); + + auto params = op.getParams(); + auto params_ty = mlir::dyn_cast(params.getType()); + if (!params_ty || !params_ty.hasStaticShape() || params_ty.getRank() != 1 || + op.getBatchDims() != 0) { + return {}; + } + + DenseIntElementsAttr axis; + if (!matchPattern(op.getAxis(), m_Constant(&axis)) || + axis.getNumElements() != 1 || + !axis.getSplatValue().isZero()) { + return {}; + } + + DenseIntElementsAttr indices; + if (!matchPattern(op.getIndices(), m_Constant(&indices)) || + indices.getType().getRank() != 1 || port[1] >= indices.getNumElements()) { + return {}; + } + + int64_t input_idx = indices.getValues()[port[1]].getInt(); + if (input_idx >= params_ty.getDimSize(0)) return {}; + + ValuePort req(params); + req.port.push_back(input_idx); + return req; +} + +ValuePort ComputeInputComponentFor(Operation* op, ArrayRef port) { + if (auto pack_op = llvm::dyn_cast(op)) { + return ComputeInputComponentFor(pack_op, port); + } + if (auto concat_op = llvm::dyn_cast(op)) { + return ComputeInputComponentFor(concat_op, port); + } + if (auto gather_op = llvm::dyn_cast(op)) { + return ComputeInputComponentFor(gather_op, port); + } + return {}; +} + +// TODO(jpienaar): ComputeInputsRequiredForOutput and ComputeOutputComponent are +// intended to be switched to op interfaces once more refined. +LogicalResult ComputeInputsRequiredForOutput(ValuePort value_port, + ComputedQueryFn has_been_computed, + ValuePortInputs* inputs) { + auto op = value_port.producer.dyn_cast(); + auto& port = value_port.port; + if (!op) return failure(); + + // No inputs required for constants and ShapeOp. + if (matchPattern(op, m_Constant()) || isa(op)) return success(); + + ValuePort req = ComputeInputComponentFor(op, port); + if (req.IsValid()) { + if (!has_been_computed(req)) inputs->push_back(req); + return success(); + } + + return failure(); +} + +// Computes the output produced by ValuePort using the query function of +// existing computed values. +Attribute ComputeOutputComponent(const ValuePort& value_port, + ValueQueryFn values) { + LLVM_DEBUG(value_port.print(llvm::dbgs() << "Computing output for ") << "\n"); + if (auto known = values(value_port)) return known; + + auto op = value_port.producer.dyn_cast(); + if (!op) return nullptr; + auto& port = value_port.port; + + if (port.empty()) { + LLVM_DEBUG(llvm::dbgs() << "skipping, port outside spec of " << op << "\n"); + return nullptr; + } + + ElementsAttr attr; + if (matchPattern(op, m_Constant(&attr))) { + if (port.size() == 1 && port[0] == 0) return attr; + if (port.size() == 2) { + assert(port[0] == 0); + DenseIntElementsAttr value; + if (!matchPattern(op, m_Constant(&value)) || + value.getType().getRank() != 1 || port[1] >= value.getNumElements()) { + return nullptr; + } + + auto range = value.getValues(); + auto component_ty = RankedTensorType::get({1}, value.getElementType()); + return DenseElementsAttr::get(component_ty, range[port[1]]); + } + return nullptr; + } + + if (auto id = dyn_cast(op)) { + if (port.size() == 1 && port[0] == 0) + return ComputeOutputComponent(ValuePort(id.getInput()), values); + return nullptr; + } + + if (auto shape_op = dyn_cast(op)) { + // No shape available in an unranked tensor type. + auto operand_ty = + mlir::dyn_cast(shape_op.getOperand().getType()); + if (!operand_ty) return nullptr; + + // Shape op has a single output so the first element should always be zero + // and the second element of port points to a particular element in the + // shape result. + if (port.size() != 2 || port[0] != 0 || port[1] >= operand_ty.getRank()) + return nullptr; + + // If the dim is dynamic, the dimension can't be inferred during + // compilation. + int64_t dim = operand_ty.getDimSize(port[1]); + if (dim == ShapedType::kDynamic) return nullptr; + + // Create an elements attribute for the particular dimension. + Type element_ty = getElementTypeOrSelf(shape_op.getType()); + APInt dim_value(element_ty.getIntOrFloatBitWidth(), dim); + auto component_ty = RankedTensorType::get({1}, element_ty); + return DenseElementsAttr::get(component_ty, {dim_value}); + } + + if (auto graph = dyn_cast(op)) { + if (port.size() == 1) + return ComputeOutputComponent( + ValuePort(graph.GetFetch().getFetches()[port[0]]), values); + return nullptr; + } + + if (auto island = dyn_cast(op)) { + if (port.size() == 1) + return ComputeOutputComponent( + ValuePort(island.GetYield().getFetches()[port[0]]), values); + return nullptr; + } + + ValuePort req = ComputeInputComponentFor(op, port); + if (req.IsValid()) return values(req); + + return nullptr; +} + +// Context used during ShapeInference. This class contains common information +// that is required by the individual shape inference helper functions (e.g., +// TF Graph version, constant values computed, etc.) +class ShapeInference { + public: + ShapeInference(int64_t graph_version, ModuleOp module, + bool propagate_caller_callee_constants, + ArrayRef ops_to_skip); + + LogicalResult ComputeInputsRequiredForOutput(ValuePort value_port, + ValuePortInputs* inputs) { + return ::mlir::TF::ComputeInputsRequiredForOutput( + value_port, + [this](const ValuePort& port) { + return results_.find(port) != results_.end(); + }, + inputs); + } + + Attribute ComputeOutputComponent(const ValuePort& value_port) { + if (auto known_attr = results_[value_port]) return known_attr; + auto attr = ::mlir::TF::ComputeOutputComponent( + value_port, [this](const ValuePort& port) { return results_[port]; }); + RecordValue(value_port, attr); + return attr; + } + + // Returns ShapeHandle if the op result could be computed as shape. + ShapeHandle ComputeOutputAsShape(OpResult result, InferenceContext* ic); + + void RecordValue(const ValuePort& value_port, Attribute value) { + LLVM_DEBUG(value_port.print(llvm::dbgs() << "\trecording ") + << value << "\n"); + results_[value_port] = value; + } + + // Infers shape of tf.While/tf.WhileRegion. If `shape_invariant` attribute is + // set, operand types are set as result types if associated body result types + // match the operand type (does not change per loop iteration). If operand and + // body result types are not the same, only handle types are propagated to + // result types. This is necessary to not incorrectly change result shapes + // when the While op will have a different result shape. Otherwise operand + // shapes are propagated to result shapes. + template + bool InferShapeForWhile(WhileOpTy op, TypeRange body_result_types); + + // Performs shape inference on the provided op and return true if the type of + // at least one result has been changed. + // A tf.Cast() is inserted for any uses that isn't in the TensorFlow dialect. + // `graph_version` indicates the current GraphDef compatibility versions + // (the versions field in graph.proto). + bool InferShapeForSingleOperation(Operation* op, int64_t max_iterations); + + // Infers shape on the provided region, including nested ones, iterate until + // fix point with a limit of max_iteration. + // Returns a failure() on error, otherwise returns true to indicate that it + // reached convergence, false otherwise. + FailureOr InferShapeUntilFixPoint(Region* region, + int64_t max_iterations); + + // Updates the serialized StableHLO modules of XlaCallModule ops whose shapes + // are refined. This function should be called after the shape refinement has + // finished, i.e. when InferShapeUntilFixPoint will no longer be called, to + // avoid re-serializing the modules multiple times. + // Returns whether it was able to propagate the shape to the StableHLO modules + // successfully. + LogicalResult PropagateShapeToStableHloModules(); + + // Updates input types and refine shapes inside body of functions that are + // attached to ControlFlow ops (If/While) or Calls. These functions include + // Then/Else branches of IfOp and Cond/Body functions of WhileOp. Functions + // attached to control flow share following common properties: + // 1) They are never reused, ie. having a single use in module. + // 2) Their input types match those of their parent ops (excluding inputs + // like predicate). + // For calls, functions can be reused across multiple call sites. In this case + // we propagate the types when all call sites have the same operand types. + // Returns a failure() on error, otherwise returns true to indicate that it + // reached convergence, false otherwise. + FailureOr PropagateShapeToFunctions(ModuleOp module, + TypeRange input_types, + ArrayRef functions, + int64_t max_iterations); + + // Propagates shapes to regions given the shapes of the inputs of the regions. + // All regions provided in `regions` are assumed to have inputs of type + // `input_types`. + // Returns a failure() on error, otherwise returns true to indicate that it + // reached convergence, false otherwise. + FailureOr PropagateShapeToRegions(TypeRange input_types, + ArrayRef regions, + int64_t max_iterations); + + // Shape propagation for call/control flow ops. + // Returns a failure() on error, otherwise returns true to indicate that it + // reached convergence, false otherwise. + FailureOr PropagateShapeIntoAttachedFunctions(Operation* op, + int64_t max_iterations); + + // Shape propagation for region based control flow. + // Returns a failure() on error, otherwise returns true to indicate that it + // reached convergence, false otherwise. + FailureOr PropagateShapeIntoAttachedRegions(Operation* op, + int64_t max_iterations); + + // Propagates any constant operand of call_op to the called function body's + // corresponding argument if the callee has only one use. + // + // TODO(b/154065712): Move this to a more general inter-procedural constant + // folding pass. + void PropagateConstantToCallee(CallOpInterface call_op, FuncOp func, + ModuleOp module); + + // Propagates any constant return value of the callee function to the call + // op's corresponding result. + void PropagateConstantFromCallee(CallOpInterface call_op, FuncOp func, + ModuleOp module); + + // Tries to compute the result of folding the op. This doesn't actually + // perform constant folding, it is just computes the equivalent constants. + // Returns whether it was able to compute constant values. + LogicalResult TryToFold(Operation* op); + + // Forcely assign operand types to result types (the i-th operand type will + // assign to i-th result type). Returns true if anything is changed. + bool ForceTypeForPassThroughOperands(Operation* op, OperandRange operands, + ResultRange results); + + // Makes result types match the operand types (the i-th result type will + // match the i-th operand type). Returns true if anything is changed. + bool RefineTypeForPassThroughOperands(Operation* op, OperandRange operands, + ResultRange results); + + // Makes result type's shape match the corresponding operand's shape. + // Returns whether any change was made. + bool RefineShapeForPassThroughOps(Operation* op); + + // Infers shape for necessary ops that are not in the TF dialect. Returns + // whether any result type changed. + bool InferShapeForNonTFDialectOperation(Operation* op); + + // Infers shape for function return type and returns whether changed. + LogicalResult InferShapeForFunctionReturnType(func::FuncOp func); + + // Enqueues function for processing. + void enqueue(func::FuncOp fn) { + LLVM_DEBUG(llvm::dbgs() + << "enqueue " << fn.getName() << " (" + << (queue_set_.count(fn) ? "already inserted" : "newly inserted") + << ")\n"); + if (queue_set_.insert(fn).second) queue_.push(fn); + } + + // Enqueues callers on functions. + void EnqueueCallers(func::FuncOp fn); + + // Returns the function at the front of the queue. + func::FuncOp front() { return queue_.front(); } + + // Returns whether work queue is empty. + bool EmptyQueue() const { return queue_.empty(); } + + // Returns function from the front of the work queue. + func::FuncOp pop_front() { + func::FuncOp ret = queue_.front(); + queue_.pop(); + queue_set_.erase(ret); + return ret; + } + + // Returns the current size of the queue. + std::queue::size_type QueueSize() const { + return queue_.size(); + } + + Dialect* const tf_dialect_; + + private: + // Returns whether the result of an operation could be updated to a new + // inferred type. Also inserts cast operation for uses that are incompatible + // with the new type. + bool UpdateTypeAndInsertIncompatibleUseCasts(Type new_type, Value result); + + // Refines the type of `result` of `op` using the type + // `potential_refined_type`. Return true if the type was changed. + bool RefineResultType(Operation* op, Value result, + Type potential_refined_type); + + // Infers the shape from a (Stateful)PartitionedCall operation by looking up + // the called function and propagating the return type. + bool InferShapeForCall(CallOpInterface call_op); + + bool InferShapeForCast(Operation* op); + + bool InferShapeForRestore(Operation* op); + + // Infers the shape IfOp outputs based on the shapes of the then and else + // function result types. + bool InferShapeForIf(IfOp op); + + // Infers the shape IfRegion outputs based on the shapes of the then and else + // yields. + bool InferShapeForIfRegion(IfRegionOp op); + + // Infers the shape CaseOp outputs based on the shapes of branch function + // result types. + bool InferShapeForCase(CaseOp op); + + // Infers the shape CaseRegion outputs based on the shapes of the branch + // yields. + bool InferShapeForCaseRegion(CaseRegionOp op); + + // Infers the shape CaseRegion outputs based on the embedded StableHLO module. + // Returns true if a return type was changed. + bool InferShapeForXlaCallModule(XlaCallModuleOp op); + + // Infers the shape of _XlaHostComputeMlir based on the host computation + // module. Returns true if a return type was changed. + bool InferShapeForXlaHostComputeMlir(_XlaHostComputeMlirOp op); + + // Infers the shape of function attached to XlaHostCompute. + // Returns true if a return type was changed. + bool InferShapeForFunctionAttachedToXlaHostCompute(XlaHostComputeOp op); + + // Infers the shape for MapDatasetOp and its associated function. Returns + // whether either op or function type was changed. + bool InferShapeForMapDataset(MapDatasetOp op, int64_t max_iterations); + + // Infers the shape for ReduceDatasetOp and its associated reduce function. + // Returns whether either op or function type was changed. + bool InferShapeForReduceDataset(ReduceDatasetOp op, int64_t max_iterations); + + // Infers the shape for TakeWhileDatasetOp and its associated predicate + // function. Returns whether either op or function type was changed. + bool InferShapeForTakeWhileDataset(TakeWhileDatasetOp op, + int64_t max_iterations); + + // Infers shape for dataset ops that have `M` input elements and `N` + // arguments, and also propagates the shape to the specified function (called + // only when function exists and has single use). + bool InferShapeForDatasetOpCommon(Operation* op, FuncOp f, + int64_t max_iterations); + + // Infers the shape of ops that create TensorList. Specifically, + // TensorListReserveOp, EmptyTensorListOp and TensorListFromTensor ops. It + // refines the element shape if all tensors written to the list across all + // mutations have identical static shape. + bool InferShapeForTensorListInitOps(Operation* op); + + // Conservatively infers shape of output tenorlist and item based on + // input tensorlist's element shape. + bool InferShapeForTensorListPopBackOp(TensorListPopBackOp op); + + // Infers the shape of VarHandleOp based on the uses of the VarHandleOp to + // update the subtypes of the resource type. + bool InferShapeForVarHandleOp(VarHandleOp op); + + // Infers the output shape of XlaConvV2Op based on the input shapes + bool InferShapeForXlaConvV2Op(XlaConvV2Op op); + + // Infers the output shape of XlaReduceWindowOp based on the input shapes. + bool InferShapeForXlaReduceWindowOp(XlaReduceWindowOp op); + + // Infers the output shape of XlaSelectAndScatterOp based on the input shapes. + bool InferShapeForXlaSelectAndScatterOp(XlaSelectAndScatterOp op); + + // Infers the output shape of XlaGatherOp based on the input shapes. + bool InferShapeForXlaGatherOp(XlaGatherOp op); + + bool RefineWithInferTypeOpInterface(InferTypeOpInterface infer_ti); + + // Returns all the callers of a function. + // Note: Usage of the return value of this function may not be interleaved + // with insertions to the callers map. This could occur if GetCallers is + // called with two separate functions, the 2nd one incurs a resize and then + // both first and 2nd stored callers are used. + ArrayRef GetCallers(func::FuncOp fn); + + // Mapping between ValuePort (which corresponds to an OpResult or smaller, + // e.g., first element of OpResult produced) to an Attribute if the ValuePort + // corresponds to a constant value. + ValuePortResultMap results_; + + // Map from a function to the callers of that function. + SymbolTableCollection symbol_table_; + SymbolUserMap symbol_users_; + + // Queue of functions being processed. + llvm::DenseSet queue_set_; + std::queue queue_; + + int64_t graph_version_; + + // Op types for which shape inference should be skipped. + llvm::SmallDenseSet ops_to_skip_; + + // TODO(b/154065712): Remove propagate_caller_callee_constants once using + // SCCP pass instead. + bool propagate_caller_callee_constants_; + + // XlaCallModule loader, which is used to deserialize the StableHLO module in + // each `XlaCallModule` op. Uses its own MLIRContext since the loader needs to + // load additional dialects, which is not allowed for the main context since + // shape inference may be called from a pass. + std::unique_ptr xla_call_module_context_; + DenseMap> + xla_call_module_loaders_; +}; + +ShapeInference::ShapeInference(int64_t graph_version, ModuleOp module, + bool propagate_caller_callee_constants, + ArrayRef ops_to_skip) + : tf_dialect_(module->getContext()->getLoadedDialect()), + symbol_users_(symbol_table_, module), + graph_version_(graph_version), + propagate_caller_callee_constants_(propagate_caller_callee_constants) { + xla_call_module_context_ = MakeMLIRContextWithThreading(); + for (const auto& op_type : ops_to_skip) { + ops_to_skip_.insert(op_type); + } + // Create symbol table for module. + symbol_table_.getSymbolTable(module); +} + +ArrayRef ShapeInference::GetCallers(func::FuncOp fn) { + return symbol_users_.getUsers(fn); +} + +void ShapeInference::EnqueueCallers(func::FuncOp fn) { + for (auto user : GetCallers(fn)) { + auto func = user->getParentOfType(); + if (func) enqueue(func); + } +} + +bool ShapeInference::UpdateTypeAndInsertIncompatibleUseCasts(Type new_type, + Value result) { + // No changes needed if the new type is unchanged. + if (new_type == result.getType()) return false; + + Operation* cast_op = nullptr; + // First insert cast back for uses that need a cast and then + // update the type. + bool enqueue_callers = false; + for (OpOperand& use : make_early_inc_range(result.getUses())) { + if (isa(use.getOwner())) { + enqueue_callers = true; + } else if (NeedsCastBack(use, tf_dialect_)) { + if (!cast_op) { + Operation* op = result.getDefiningOp(); + OpBuilder b(op); + b.setInsertionPointAfter(op); + cast_op = InsertCast(b, op->getLoc(), result.getType(), result); + if (!cast_op) return false; + } + use.set(Value(cast_op->getResult(0))); + } + } + + result.setType(new_type); + if (enqueue_callers) + EnqueueCallers(result.getDefiningOp()->getParentOfType()); + return true; +} + +bool ShapeInference::RefineResultType(Operation* op, Value result, + Type potential_refined_type) { + if (!CanRefineTypeWith(result.getType(), potential_refined_type)) + return false; + + return UpdateTypeAndInsertIncompatibleUseCasts(potential_refined_type, + result); +} + +// Infers the shape from a (Stateful)PartitionedCall operation by looking up +// the called function and propagating the return type. +bool ShapeInference::InferShapeForCall(CallOpInterface call_op) { + func::FuncOp func = dyn_cast_or_null( + call_op.resolveCallableInTable(&symbol_table_)); + if (!func) return false; + + DCOMMENT("Infer shape for call " << func.getName()); + Operation* op = call_op.getOperation(); + bool changed = false; + // Map each of the results of the call to the returned type of the + // function. + for (auto result : + zip(op->getResults(), func.getFunctionType().getResults())) { + changed = RefineResultType(op, std::get<0>(result), std::get<1>(result)) || + changed; + } + DCOMMENT(" - call " << func.getName() << "changed ? " << changed << "\n"); + + return changed; +} + +bool ShapeInference::InferShapeForCast(Operation* op) { + DCOMMENT_OP(op, "Inferring shape for "); + Value result = op->getResult(0); + if (!CanBeRefined(result.getType())) return false; + + // Avoid inserting a cast where no users types could be refined (e.g., where + // there would need to be a cast inserted for every user again). + if (llvm::all_of(result.getUses(), [this](OpOperand& use) { + return NeedsCastBack(use, tf_dialect_); + })) + return false; + + // Combine shape information including shape info in subtypes. + Type operand_type = op->getOperand(0).getType(); + Type result_type = result.getType(); + auto new_type = GetCastCompatibleType(operand_type, result_type); + if (!new_type) { + // Combine shape information when leaf element types are not the same, not + // including shape info in subtypes. + auto ranked_operand_type = mlir::dyn_cast(operand_type); + if (!ranked_operand_type) return false; + auto ranked_res_type = mlir::dyn_cast(result.getType()); + if (ranked_res_type && + ranked_operand_type.getShape() == ranked_res_type.getShape()) + return false; + + auto shaped_res_type = mlir::dyn_cast(result_type); + if (!shaped_res_type) return false; + new_type = tensorflow::GetTypeFromTFTensorShape( + ranked_operand_type.getShape(), shaped_res_type.getElementType()); + } + return UpdateTypeAndInsertIncompatibleUseCasts(new_type, result); +} + +bool ShapeInference::InferShapeForIf(IfOp op) { + DCOMMENT_OP(op.getOperation(), "Infer shape for if "); + bool changed = false; + auto then_results = + op.ResolveThenFunction(&symbol_table_).getFunctionType().getResults(); + auto else_results = + op.ResolveElseFunction(&symbol_table_).getFunctionType().getResults(); + for (auto it : llvm::zip(op.getResults(), then_results, else_results)) { + // If then and else types do not match, skip refinement for that result. + if (std::get<1>(it) != std::get<2>(it)) continue; + changed = RefineResultType(op, std::get<0>(it), std::get<1>(it)) || changed; + } + return changed; +} + +bool ShapeInference::InferShapeForIfRegion(IfRegionOp op) { + bool changed = false; + + Operation* then_yield = op.getThenBranch().front().getTerminator(); + Operation* else_yield = op.getElseBranch().front().getTerminator(); + for (auto result : zip(op.getResults(), then_yield->getOperandTypes(), + else_yield->getOperandTypes())) { + // If then and else types do not match, skip refinement for that result. + if (std::get<1>(result) != std::get<2>(result)) continue; + changed = RefineResultType(op, std::get<0>(result), std::get<1>(result)) || + changed; + } + return changed; +} + +bool ShapeInference::InferShapeForCase(CaseOp op) { + DCOMMENT_OP(op.getOperation(), "Infer shape for case "); + + llvm::SmallVector branch_result_types; + for (int i = 0; i < op.num_branches(); ++i) { + branch_result_types.push_back(op.ResolveBranchFunction(&symbol_table_, i) + .getFunctionType() + .getResults()); + } + + bool changed = false; + for (const auto& result : op.getResults()) { + llvm::DenseSet types; + for (const auto& branch_result_type : branch_result_types) { + types.insert(branch_result_type[result.getResultNumber()]); + } + if (types.size() == 1) { + changed = RefineResultType(op, result, *types.begin()) || changed; + } + } + return changed; +} + +bool ShapeInference::InferShapeForCaseRegion(CaseRegionOp op) { + bool changed = false; + for (const auto& result : op.getResults()) { + llvm::DenseSet types; + for (auto& branch : op.getBranches()) { + Operation* yield = branch.front().getTerminator(); + types.insert(yield->getOperandTypes()[result.getResultNumber()]); + } + if (types.size() == 1) { + changed = RefineResultType(op, result, *types.begin()) || changed; + } + } + return changed; +} + +bool ShapeInference::InferShapeForXlaCallModule(XlaCallModuleOp op) { + if (!llvm::any_of(op.getResultTypes(), CanBeRefined)) return false; + + tensorflow::XlaCallModuleLoader* loader; + if (auto it = xla_call_module_loaders_.find(op); + it != xla_call_module_loaders_.end()) { + loader = it->second.get(); + } else { + // Lazily parse XlaCallModule's embedded HLO module and cache the loader to + // avoid repeatedly parsing the module. + + std::vector disabled_checks; + for (auto attr : op.getDisabledChecks().getAsRange()) { + disabled_checks.push_back(attr.getValue().str()); + } + std::vector platforms; + for (auto attr : op.getPlatforms().getAsRange()) { + platforms.push_back(attr.getValue().str()); + } + + // It is a terrible idea to have local MLIR contexts so we need to + // register extensions here, again. + mlir::DialectRegistry registry; + registry.insert(); + mlir::func::registerAllExtensions(registry); + xla_call_module_context_->appendDialectRegistry(registry); + + auto l = tensorflow::XlaCallModuleLoader::Create( + xla_call_module_context_.get(), op.getVersion(), op.getModule(), + std::move(disabled_checks), std::move(platforms), + /*num_invocation_args=*/op.getArgs().size(), + op.getHasTokenInputOutput(), op.getUseShardyPartitioner()); + if (!l.ok()) { + llvm::errs() << "Parsing error in XlaCallModule: " + << l.status().ToString() << "\n"; + return false; + } + + it = xla_call_module_loaders_.insert({op, *std::move(l)}).first; + loader = it->second.get(); + } + + // Cannot pass `op.getArgs().getTypes()` to `loader->RefineDynamicShapes` + // because `op` and `loader` are using different MLIR contexts. See comments + // on `xla_call_module_context_` for details. + std::vector input_shapes; + input_shapes.reserve(op.getArgs().size()); + for (mlir::Type type : op.getArgs().getTypes()) { + input_shapes.push_back(xla::TypeToShape(type)); + } + + absl::Status status = loader->RefineDynamicShapes(input_shapes); + if (!status.ok()) { + // Do not return false here. + // + // RefineDynamicShapes returns ok only when it produces full static shapes. + // It may partially succeed by producing RankedTensor shapes with dynamic + // dimensions. Such info is still useful for the downstream. We don't need + // to abort here. + // TODO(b/316639984): improve RefineDynamicShapes return values to include + // these info. + VLOG(1) << "Failed during XlaCallModule shape refinement: " << status; + } + mlir::ResultRange op_results = op.getResults(); + // The main_outputs may include tokens that are not among the op_results; + mlir::TypeRange main_output_types = loader->OutputTypes(); + int nr_main_token_outputs = + llvm::count_if(main_output_types, tensorflow::IsTokenType); + if (op_results.size() != main_output_types.size() - nr_main_token_outputs) { + llvm::errs() << "XlaCallModule has " << op_results.size() + << " but the main function has " + << main_output_types.size() - nr_main_token_outputs + << " non-token ouputs"; + return false; + } + bool changed = false; + int next_op_result = 0; + for (auto output_type : main_output_types) { + if (tensorflow::IsTokenType(output_type)) continue; + auto output_type_ranked = mlir::dyn_cast(output_type); + if (output_type_ranked == nullptr) { + llvm::errs() << "Unsupported XlaCallModule result type: " << output_type + << "\n"; + return false; + } + auto result = op_results[next_op_result++]; + + // Build a new type object from `type` and `elem_type`. `type` is owned by + // `xla_call_module_context_` and should not be mixed with op's context. + auto new_type = RankedTensorType::get( + output_type_ranked.getShape(), getElementTypeOrSelf(result.getType())); + + changed = RefineResultType(op, result, new_type) || changed; + } + + return changed; +} + +LogicalResult ShapeInference::PropagateShapeToStableHloModules() { + for (auto& [op, loader] : xla_call_module_loaders_) { + if (!loader->IsOutputTypeRefined()) continue; + + FailureOr version = + stablehlo::getPortableArtifactVersion(op.getModule()); + if (failed(version)) { + return op.emitOpError() << "Failed to extract the VHLO version from the " + "serialized StableHLO module while attempting " + "to propagate shapes to the StableHLO module."; + } + + std::string bytecode; + llvm::raw_string_ostream os(bytecode); + if (failed(stablehlo::serializePortableArtifact(loader->module(), + version->toString(), os))) { + return op.emitOpError() + << "Failed to serialize StableHLO module while attempting to " + "propagate shapes to the StableHLO module."; + } + + op.setModule(bytecode); + } + return success(); +} + +bool ShapeInference::InferShapeForFunctionAttachedToXlaHostCompute( + XlaHostComputeOp op) { + const std::string kShapeInferenceGraph = "shape_inference_graph"; + if (!op->hasAttr(kShapeInferenceGraph)) { + return false; + } + + ModuleOp module = op->getParentOfType(); + func::FuncOp func = module.lookupSymbol( + op.getShapeInferenceGraphAttr().getRootReference()); + + if (func == nullptr) return false; + + std::vector<_XlaRecvAtHostOp> xla_recv_at_host_ops; + func.walk([&](_XlaRecvAtHostOp op) { xla_recv_at_host_ops.push_back(op); }); + if (xla_recv_at_host_ops.empty()) return false; + auto xla_recv_at_host_op = xla_recv_at_host_ops.front(); + + // Copy const op into func body and replace the uses of the corresponding args + OpBuilder builder(&func.front().front()); + for (auto arg : func.getArguments()) { + Value operand = op.getOperand(arg.getArgNumber()); + if (isa_and_nonnull(operand.getDefiningOp())) { + xla_recv_at_host_op.getResult(arg.getArgNumber()) + .replaceAllUsesWith( + builder.clone(*operand.getDefiningOp())->getResult(0)); + } + } + + // Update/use input shapes for function. + FunctionType func_type = func.getFunctionType(); + func.setType(FunctionType::get(func.getContext(), op.getOperandTypes(), + func_type.getResults())); + + // Run shape inference on the function. + if (failed( + PropagateShapeToRegions(op.getOperandTypes(), {&func.getBody()}, 10))) + return false; + if (failed(InferShapeForFunctionReturnType(func))) return false; + + return false; +} + +bool ShapeInference::InferShapeForXlaHostComputeMlir( + _XlaHostComputeMlirOp host_compute_op) { + // Extract the module and function. + // The '_XlaHostComputeMlir` verifier verifies that `host_mlir_module` + // attribute is well formed, so we just return in case of an error in + // extracting the host function since it should never occur. + StringAttr host_module = + host_compute_op->getAttrOfType("host_mlir_module"); + if (host_module.getValue().empty()) return false; + + mlir::OwningOpRef module_for_func; + func::FuncOp func = host_compute_op.GetHostFunc(&module_for_func); + + // Update/use input shapes for function. + FunctionType func_type = func.getFunctionType(); + func.setType(FunctionType::get(func.getContext(), + host_compute_op.getOperandTypes(), + func_type.getResults())); + + // Run shape inference on the function. + if (failed(PropagateShapeToRegions(host_compute_op.getOperandTypes(), + {&func.getBody()}, 10))) + return false; + if (failed(InferShapeForFunctionReturnType(func))) return false; + + bool changed = false; + // Use refined function return shape for XlaHostComputeMlirOp. + for (auto result : + zip(host_compute_op.getResults(), func.getFunctionType().getResults())) { + changed = RefineResultType(host_compute_op, std::get<0>(result), + std::get<1>(result)) || + changed; + } + + return changed; +} + +// Infer the shape of `Restore` and `RestoreV2` op based on the first +// `AssignVariableOp` that uses the result. This requires that the resource +// subtype inference is completed. +bool ShapeInference::InferShapeForRestore(Operation* op) { + DCOMMENT_OP(op, "Inferring shape for Restore,RestoreV2"); + // Currently only support single output. + if (op->getNumResults() != 1) return false; + if (!CanBeRefined(op->getResult(0).getType())) return false; + + llvm::SmallVector worklist; + llvm::append_range(worklist, op->getUsers()); + + // Look for any `AssignVariableOp` that uses the result of this op. + while (!worklist.empty()) { + mlir::Operation* const use = worklist.pop_back_val(); + + // Follow the `CastOp`/`IdentityOp`'s users to handle the `RestoreV2` -> + // (optionally `IdentityOp`) -> `CastOp` `AssignVariableOp` case. + if (llvm::isa(use)) { + llvm::append_range(worklist, use->getUsers()); + continue; + } + + TF::AssignVariableOp assign_op = llvm::dyn_cast(use); + if (!assign_op) { + continue; + } + auto subtypes = mlir::cast( + getElementTypeOrSelf(assign_op.getResource())) + .getSubtypes(); + if (subtypes.empty()) { + continue; + } + auto subtype = mlir::dyn_cast(subtypes.front()); + if (subtype == nullptr) { + continue; + } + // Preserve the dtype from the restore op even if `AssignVariableOp` uses a + // different dtype, which is possible when there's a `CastOp` between them. + subtype = subtype.clone( + mlir::cast(op->getResult(0).getType()).getElementType()); + // Update the result type of this op with the resource's type. We only use + // the resource subtype of the first user since shapes from all the users + // should be equal or compatible. + return UpdateTypeAndInsertIncompatibleUseCasts(subtype, op->getResult(0)); + } + return false; +} + +// Helper structure to capture shapes & types for Dataset input. +struct DatasetInput { + explicit operator bool() const { return shapes && types; } + + ArrayAttr shapes; + ArrayAttr types; +}; + +// Returns the input elements shapes and types for Dataset ops. +DatasetInput GetDatasetInput(Value value) { + // TODO(haoliang): add an interface for DatasetOp to avoid the following + // enumeration. + // Iteratively tracing upwards if parent op is `IdentityOp` or `IdentityNOp`. + while ( + llvm::isa_and_nonnull(value.getDefiningOp())) { + value = value.getDefiningOp()->getOperand( + mlir::cast(value).getResultNumber()); + } + + Operation* op = value.getDefiningOp(); + if (!llvm::isa_and_nonnull(op)) + return DatasetInput{nullptr, nullptr}; + + return DatasetInput{op->getAttrOfType("output_shapes"), + op->getAttrOfType("output_types")}; +} + +bool ShapeInference::InferShapeForDatasetOpCommon(Operation* op, FuncOp f, + int64_t max_iterations) { + int N = op->getNumOperands() - 1; + int M = f.getNumArguments() - N; + DCOMMENT_OP(op, "Inferring shape for with N = " << N << " and M = " << M); + + // Initialize with function input types. + auto input_types = llvm::to_vector<1>( + cast(f.getOperation()).getArgumentTypes()); + + DatasetInput input_elements = GetDatasetInput(op->getOperand(0)); + if (!input_elements) { + op->emitWarning("unexpected dataset input; skipping function refinement"); + return false; + } + + // Track if changed to skip enqueueing. + bool changed = false; + auto it = input_types.begin(); + // First set first M argument shapes & types. + for (int i = 0; i < M; ++i) { + Type t = GetType(input_elements.shapes[i], input_elements.types[i]); + t = TypeMeet(*it, t); + changed = changed || (t != *it); + *it++ = t; + } + // Now the remaining N from operand types. + for (auto t : llvm::drop_begin(op->getOperandTypes())) { + auto meet = TypeMeet(*it, t); + changed = changed || (meet != *it); + *it++ = meet; + } + if (!changed) return false; + + FailureOr res = PropagateShapeToFunctions( + op->getParentOfType(), input_types, {f}, max_iterations); + if (failed(res)) { + op->emitOpError("propagating shapes failed"); + return false; + } + return *res; +} + +bool ShapeInference::InferShapeForMapDataset(MapDatasetOp op, + int64_t max_iterations) { + // MapDatasetOp's relationship with its associated function is as + // follows: first M function params are dictated by the set + // output shapes and types, the next N are the last Ninputs from MapDataset + // op. The MapDataset op always has N+1 inputs. + // TODO(jpienaar): Avoid this lookup. + auto module = op->getParentOfType(); + auto f = module.lookupSymbol(op.getF()); + // Skip if function is not found or more than one caller. + if (!f || !llvm::hasSingleElement(GetCallers(f))) return false; + return InferShapeForDatasetOpCommon(op, f, max_iterations); +} + +bool ShapeInference::InferShapeForTakeWhileDataset(TakeWhileDatasetOp op, + int64_t max_iterations) { + // TakeWhileDatasetOp's relationship with its associated function is as + // follows: first M function params are dictated by the set + // output shapes and types, the next N are the last Ninputs from + // TakeWhileDataset op. The TakeWhileDataset op always has N+1 inputs. + // TODO(jpienaar): Avoid this lookup. + auto module = op->getParentOfType(); + auto f = module.lookupSymbol(op.getPredicate()); + // Skip if function is not found or more than one caller. + if (!f || !llvm::hasSingleElement(GetCallers(f))) return false; + return InferShapeForDatasetOpCommon(op, f, max_iterations); +} + +bool ShapeInference::InferShapeForReduceDataset(ReduceDatasetOp op, + int64_t max_iterations) { + // ReduceDatasetOp's relationship with its associated reduce function is + // described as follows: The reduce function will in general have (X + Y + Z) + // arguments, where X is the number of tensor components that represent the + // state, Y is the number of tensor components that represent the input + // elements, and Z is the number of tensor components that represent any + // captured arguments. Y is determined by the output_shapes of an op that + // defines the first operand of `op`. + + // TODO(jpienaar): Avoid this lookup. + auto module = op->getParentOfType(); + auto f = module.lookupSymbol(op.getF()); + + // Skip if function is not found or it has more than one caller. + if (!f || !llvm::hasSingleElement(GetCallers(f))) return false; + + DatasetInput input_elements = GetDatasetInput(op.getInputDataset()); + + const int num_states = op.getOutputShapes().size(); + const int num_captured_arguments = op.getNumOperands() - 1 - num_states; + + // If input_elements is undefined, we can still infer the shapes for the + // states and captured arguments. + int num_input_elements; + auto input_types = llvm::to_vector<1>( + cast(f.getOperation()).getArgumentTypes()); + if (input_elements) { + num_input_elements = input_elements.shapes.size(); + } else { + num_input_elements = + input_types.size() - num_states - num_captured_arguments; + } + + DCOMMENT_OP(op, + "Inferring shape for ReduceDataset with #states = " + << num_states << " , #input_elements = " << num_input_elements + << " , and #captured_arguments = " << num_captured_arguments); + if (num_states + num_input_elements + num_captured_arguments != + f.getNumArguments()) { + op->emitOpError( + "propagating shapes for ReduceDataset failed due to inconsistent " + "number of arguments"); + return false; + } + + // Track if changed to skip enqueueing. + bool changed = false; + auto it = input_types.begin(); + + // Set the first num_states arguments shapes & types from the state. + for (int i = 0; i < num_states; ++i) { + Type t = GetType(op.getOutputShapes()[i], op.getOutputTypes()[i]); + t = TypeMeet(*it, t); + changed = changed || (t != *it); + *it++ = t; + } + + // Second set the following num_input_elements arguments from + // repeat_dataset_op. Skip propagating shape if input_elements is + // undefined. + for (int i = 0; i < num_input_elements; ++i) { + if (input_elements) { + Type t = GetType(input_elements.shapes[i], input_elements.types[i]); + t = TypeMeet(*it, t); + changed = changed || (t != *it); + *it++ = t; + } else { + it++; + } + } + + // Last set the remaining num_captured_arguments from op. + for (auto t : llvm::drop_begin(op.getOperandTypes(), 1 + num_states)) { + auto meet = TypeMeet(*it, t); + changed = changed || (meet != *it); + *it++ = meet; + } + + if (!changed) return false; + + FailureOr res = + PropagateShapeToFunctions(module, input_types, {f}, max_iterations); + if (failed(res)) { + op->emitOpError("Propagating shapes for ReduceDataset failed"); + return false; + } + return *res; +} + +bool ShapeInference::InferShapeForTensorListInitOps(Operation* op) { + DCOMMENT_OP(op, "Inferring shape for TensorListInitOps."); + Value handle = op->getResult(0); + Value initial_element_shape = GetElementShapeOperand(op); + RankedTensorType element_type; + if (auto tl_from_tensor = dyn_cast(op)) { + // For TensorListFromTensor op we can infer element shape by dropping the + // first dimension of input tensor. + element_type = DropFirstDimension(tl_from_tensor.getTensor().getType()); + if (!element_type || !element_type.hasStaticShape()) return false; + } + if (!CanInferTensorListElementType(handle, initial_element_shape, + &element_type)) { + DCOMMENT("InferShapeForListInitOps " << op << " could not infer"); + return false; + } + DCOMMENT("InferShapeForListInitOps " << *op << " could be inferred " + << element_type); + if (!element_type || !element_type.hasStaticShape()) return false; + auto variant_type = VariantType::get(element_type, op->getContext()); + auto tensor_type = tensorflow::GetTypeFromTFTensorShape({}, variant_type); + bool changed = RefineResultType(op, handle, tensor_type); + if (changed) DCOMMENT_OP(op, "Modified after shape inference:"); + return changed; +} + +bool ShapeInference::InferShapeForTensorListPopBackOp(TensorListPopBackOp op) { + // The first Operand is assumed to be a TensorType around a variant with a + // single subtype (e.g. tensor>>). We will + // copy this type to the first result, and copy the singular variant subtype + // to the second result (tensor<2xi32>). + DCOMMENT_OP(op, "Inferring shape for TensorListPopBackOp."); + + auto src_list_handle_t = + mlir::dyn_cast_or_null(op.getOperand(0).getType()); + if (!src_list_handle_t) return false; + + // Copy of operand tensorlist type. + TensorType dst_list_handle_t = + src_list_handle_t.clone(src_list_handle_t.getElementType()); + auto variant_element_t = + mlir::dyn_cast_or_null(dst_list_handle_t.getElementType()); + if (!variant_element_t || variant_element_t.getSubtypes().size() != 1) + return false; + + // Underlying TensorType from variant that represents a shape signature + // compatible with all elements in the tensorlist. + TensorType list_handle_element_t = variant_element_t.getSubtypes()[0]; + + if (!RefineResultType(op, op->getResult(0), dst_list_handle_t)) { + DCOMMENT("InferShapeForTensorListPopBackOp could not propogate result 0" + << op); + return false; + } + if (!RefineResultType(op, op->getResult(1), list_handle_element_t)) { + DCOMMENT("InferShapeForTensorListPopBackOp could not propogate result 1" + << op); + return false; + } + return true; +} + +bool ShapeInference::InferShapeForVarHandleOp(VarHandleOp op) { + DCOMMENT_OP(op, "Inferring shape for VarHandleOp"); + + Value resource = op.getResource(); + if (!CanBeRefined(resource.getType())) return false; + + // Make sure there are only use cases from the `AssignVariableOp` and + // `ReadVariableOp`. For other cases, we can skip to be conservative. + for (auto& use : make_early_inc_range(resource.getUses())) { + Operation* def = use.getOwner(); + if (!llvm::isa(def) && !llvm::isa(def)) { + return false; + } + } + + bool changed = false; + + // Look for any `AssignVariableOp` and `ReadVariableOp` that uses the value of + // this op. + for (auto& use : make_early_inc_range(resource.getUses())) { + Operation* def = use.getOwner(); + Value value; + if (AssignVariableOp assign_op = dyn_cast(def)) { + value = assign_op.getValue(); + } else if (ReadVariableOp read_op = dyn_cast(def)) { + value = read_op.getValue(); + } else { + llvm_unreachable("unexpected operator type"); + } + + TensorType resource_subtype = mlir::cast(value.getType()); + ResourceType resource_type = + ResourceType::get({resource_subtype}, op.getContext()); + UnrankedTensorType new_resource_type = + UnrankedTensorType::get(resource_type); + + Type refined_type = TypeMeet(resource.getType(), new_resource_type); + if (refined_type == resource.getType()) continue; + resource.setType(refined_type); + changed = true; + } + + return changed; +} + +// Helper function for creating a Window proto from user-supplied data. +// Returns std::nullopt if the user-supplied data was invalid. +std::optional InferWindowFromDimensions( + llvm::SmallVector window_dimensions, + llvm::SmallVector window_strides, + llvm::SmallVector> padding, + llvm::SmallVector lhs_dilation, + llvm::SmallVector rhs_dilation) { + const auto verify_size = [&](const size_t x, const char* x_name) { + if (x == 0 || x == window_dimensions.size()) { + return true; + } else { + llvm::errs() + << "Window has different number of window dimensions than of " + << x_name + << "\nNumber of window dimensions: " << window_dimensions.size() + << "\nNumber of " << x_name << ": " << x << "\n"; + return false; + } + }; + + if (!(verify_size(window_dimensions.size(), "window_dimensions") && + verify_size(window_strides.size(), "window strides") && + verify_size(padding.size(), "padding entries") && + verify_size(lhs_dilation.size(), "lhs dilation factors") && + verify_size(rhs_dilation.size(), "rhs dilation factors"))) + return std::nullopt; + + xla::Window window; + for (size_t i = 0; i < window_dimensions.size(); i++) { + auto dim = window.add_dimensions(); + dim->set_size(window_dimensions[i]); + if (!window_strides.empty()) { + dim->set_stride(window_strides[i]); + } else { + dim->set_stride(1); + } + if (!padding.empty()) { + dim->set_padding_low(padding[i].first); + dim->set_padding_high(padding[i].second); + } else { + dim->set_padding_low(0); + dim->set_padding_high(0); + } + if (!lhs_dilation.empty()) { + dim->set_base_dilation(lhs_dilation[i]); + } else { + dim->set_base_dilation(1); + } + if (!rhs_dilation.empty()) { + dim->set_window_dilation(rhs_dilation[i]); + } else { + dim->set_window_dilation(1); + } + dim->set_window_reversal(false); + } + return window; +} + +std::optional InferWindowOutputShape( + const ShapedType& base_shape, const xla::Window& window, + Type element_type) { + if (window.dimensions_size() != base_shape.getRank()) { + llvm::errs() << "Window has dimension " << window.dimensions_size() + << " but base shape has dimension " << base_shape.getRank() + << "\n"; + return std::nullopt; + } + + std::vector output_dimensions(window.dimensions_size()); + std::vector output_is_dynamic(window.dimensions_size()); + for (int64_t i = 0; i < window.dimensions_size(); ++i) { + const auto& dim = window.dimensions(i); + if (dim.size() <= 0) { + llvm::errs() << "Window " << window.DebugString() + << " has a non-positive dimension.\n"; + return std::nullopt; + } + if (dim.stride() <= 0) { + llvm::errs() << "Window " << window.DebugString() + << " has a non-positive stride.\n"; + return std::nullopt; + } + if (dim.base_dilation() < 1) { + llvm::errs() << "Window " << window.DebugString() + << " has a non-positive base area dilation factor.\n"; + return std::nullopt; + } + if (dim.window_dilation() < 1) { + llvm::errs() << "Window " << window.DebugString() + << " has a non-positive window dilation factor.\n"; + return std::nullopt; + } + + if (base_shape.isDynamicDim(i)) { + output_dimensions[i] = ShapedType::kDynamic; + } else { + const int64_t dilated_base = xla::window_util::DilatedBound( + base_shape.getDimSize(i), dim.base_dilation()); + const int64_t padded_dilated_base = + dim.padding_low() + dilated_base + dim.padding_high(); + const int64_t dilated_window = + xla::window_util::DilatedBound(dim.size(), dim.window_dilation()); + + output_dimensions[i] = xla::window_util::StridedBound( + padded_dilated_base, dilated_window, dim.stride()); + } + } + + return tensorflow::GetTypeFromTFTensorShape(output_dimensions, element_type); +} + +bool ShapeInference::InferShapeForXlaReduceWindowOp(XlaReduceWindowOp op) { + DCOMMENT_OP(op, "Inferring shape for XlaReduceWindowOp"); + + bool changed = false; + + auto input_ty = mlir::cast(op.getInput().getType()); + DenseElementsAttr window_dimensions, window_strides, base_dilations, + window_dilations, padding; + if (input_ty.hasStaticShape() && + matchPattern(op.getWindowDimensions(), m_Constant(&window_dimensions)) && + matchPattern(op.getWindowStrides(), m_Constant(&window_strides)) && + matchPattern(op.getBaseDilations(), m_Constant(&base_dilations)) && + matchPattern(op.getWindowDilations(), m_Constant(&window_dilations)) && + matchPattern(op.getPadding(), m_Constant(&padding))) { + llvm::SmallVector window_dimensions_vec, window_strides_vec, + base_dilations_vec, window_dilations_vec; + llvm::SmallVector> padding_pairs( + padding.getNumElements() / 2); + + for (auto i = 0; i < window_dimensions.size(); ++i) { + window_dimensions_vec.push_back( + window_dimensions.getValues()[i].getInt()); + } + + for (auto i = 0; i < window_strides.size(); ++i) { + window_strides_vec.push_back( + window_strides.getValues()[i].getInt()); + } + + for (auto i = 0; i < base_dilations.size(); ++i) { + base_dilations_vec.push_back( + base_dilations.getValues()[i].getInt()); + } + + for (auto i = 0; i < window_dilations.size(); ++i) { + window_dilations_vec.push_back( + window_dilations.getValues()[i].getInt()); + } + + for (auto i = 0; i < padding_pairs.size(); ++i) { + padding_pairs[i] = {padding.getValues()[i * 2].getInt(), + padding.getValues()[i * 2 + 1].getInt()}; + } + + auto window = InferWindowFromDimensions( + window_dimensions_vec, window_strides_vec, padding_pairs, + base_dilations_vec, window_dilations_vec); + if (!window) { + op->emitOpError("failed to create window"); + } + auto output_shape = InferWindowOutputShape( + input_ty, window.value(), + mlir::cast(op.getInitValue().getType()).getElementType()); + + if (!output_shape) { + op->emitOpError("failed to infer output shape"); + } + + changed = RefineResultType(op.getOperation(), op.getResult(), + output_shape.value()); + } + + return changed; +} + +bool ShapeInference::InferShapeForXlaSelectAndScatterOp( + XlaSelectAndScatterOp op) { + DCOMMENT_OP(op, "Inferring shape for XlaSelectAndScatterOp"); + + auto operand_shape = mlir::cast(op.getOperand().getType()); + auto source_shape = mlir::cast(op.getSource().getType()); + DenseElementsAttr window_dimensions, window_strides, padding; + if (operand_shape.hasRank() && source_shape.hasRank() && + matchPattern(op.getWindowDimensions(), m_Constant(&window_dimensions)) && + matchPattern(op.getWindowStrides(), m_Constant(&window_strides)) && + matchPattern(op.getPadding(), m_Constant(&padding))) { + llvm::SmallVector window_dimensions_vec, window_strides_vec, + base_dilations_vec, window_dilations_vec; + llvm::SmallVector> padding_pairs( + padding.getNumElements() / 2); + + for (auto i = 0; i < window_dimensions.size(); ++i) { + window_dimensions_vec.push_back( + window_dimensions.getValues()[i].getInt()); + } + + for (auto i = 0; i < window_strides.size(); ++i) { + window_strides_vec.push_back( + window_strides.getValues()[i].getInt()); + } + + for (auto i = 0; i < padding_pairs.size(); ++i) { + padding_pairs[i] = {padding.getValues()[i * 2].getInt(), + padding.getValues()[i * 2 + 1].getInt()}; + } + + auto window = InferWindowFromDimensions( + window_dimensions_vec, window_strides_vec, padding_pairs, {}, {}); + if (!window) { + op->emitOpError("failed to create window"); + } + auto window_result_shape = InferWindowOutputShape( + operand_shape, window.value(), operand_shape.getElementType()); + + if (!window_result_shape) { + op->emitOpError("failed to infer window result shape"); + } + + if (window_result_shape.value() != source_shape) { + op->emitOpError( + "Source shape does not match the shape of window-reduced operand."); + } + } + + return RefineResultType(op.getOperation(), op.getResult(), + op.getOperand().getType()); +} + +bool ShapeInference::InferShapeForXlaGatherOp(XlaGatherOp op) { + xla::Shape input_shape = xla::TypeToShape(op.getOperand().getType()); + if (input_shape == xla::Shape() || input_shape.is_unbounded_dynamic()) + return false; + + xla::Shape start_indices_shape = + xla::TypeToShape(op.getStartIndices().getType()); + if (start_indices_shape == xla::Shape()) return false; + + xla::GatherDimensionNumbers gather_dim_numbers; + if (!gather_dim_numbers.ParseFromString(op.getDimensionNumbers().str())) + return false; + + DenseIntElementsAttr slice_sizes_attr; + if (DenseIntElementsAttr attr; + matchPattern(op.getSliceSizes(), m_Constant(&attr))) { + slice_sizes_attr = attr; + } else if (const auto it = results_.find(ValuePort(op.getSliceSizes())); + it != results_.end() && + llvm::isa_and_nonnull(it->second)) { + slice_sizes_attr = llvm::cast(it->second); + } else { + return false; + } + + llvm::SmallVector slice_sizes; + for (const auto& attr : slice_sizes_attr.getValues()) { + slice_sizes.push_back(attr.getSExtValue()); + } + + auto output_shape = xla::ShapeInference::InferGatherShape( + input_shape, start_indices_shape, gather_dim_numbers, slice_sizes); + if (!output_shape.ok()) { + op->emitError() << output_shape.status().message(); + return false; + } + + auto refined_type = xla::ConvertShapeToType( + *output_shape, mlir::Builder(op)); + if (!refined_type.ok()) { + op->emitError() << refined_type.status().message(); + return false; + } + + return RefineResultType(op, op.getOutput(), *refined_type); +} + +std::optional InferXlaConvOutputShape( + llvm::SmallVector input_tensor_dims, + llvm::SmallVector kernel_tensor_dims, + llvm::SmallVector window_strides, + llvm::SmallVector> paddings, + llvm::SmallVector lhs_dilations, + llvm::SmallVector rhs_dilations, int64_t batch_group_count, + xla::ConvolutionDimensionNumbers dnums, Type element_type) { + auto num_spatial_dims = input_tensor_dims.size() - 2; + std::vector output_dims(input_tensor_dims.size()); + + auto input_batch = input_tensor_dims[dnums.input_batch_dimension()]; + auto kernel_output_feature = + kernel_tensor_dims[dnums.kernel_output_feature_dimension()]; + output_dims[dnums.output_batch_dimension()] = input_batch / batch_group_count; + DCOMMENT("inferrd output batch dimension is " + << output_dims[dnums.output_batch_dimension()]); + output_dims[dnums.output_feature_dimension()] = kernel_output_feature; + DCOMMENT("inferrd output output_feature_dimension is " + << output_dims[dnums.output_feature_dimension()]); + + std::vector input_spatial_dims; + llvm::SmallVector window_spatial_dims; + for (auto i = 0; i < num_spatial_dims; ++i) { + input_spatial_dims.push_back( + input_tensor_dims[dnums.input_spatial_dimensions(i)]); + window_spatial_dims.push_back( + kernel_tensor_dims[dnums.kernel_spatial_dimensions(i)]); + } + + ShapedType base_shape = + tensorflow::GetTypeFromTFTensorShape(input_spatial_dims, element_type); + + auto window = + InferWindowFromDimensions(window_spatial_dims, window_strides, paddings, + lhs_dilations, rhs_dilations); + + auto output_shape = + InferWindowOutputShape(base_shape, window.value(), element_type); + + for (auto i = 0; i < num_spatial_dims; ++i) { + output_dims[dnums.output_spatial_dimensions(i)] = + output_shape.value().getShape()[i]; + DCOMMENT("inferrd output spatial dimension " + << i << " at dimension numebr " + << dnums.output_spatial_dimensions(i) << " is " + << output_dims[dnums.output_spatial_dimensions(i)]); + } + return tensorflow::GetTypeFromTFTensorShape(output_dims, element_type); +} + +// TODO(hanxiongwang): The logic in this function need move to Op Verify method +// when dependecy issue of adding header file +// "third_party/tensorflow/compiler/xla/xla_data.pb.h" into +// "third_party/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc" is +// resolved +LogicalResult PrecheckForXlaConvV2Op(XlaConvV2Op op) { + auto input_tensor = op.getLhs(); + auto kernel_tensor = op.getRhs(); + auto window_strides = op.getWindowStrides(); + auto padding = op.getPadding(); + auto lhs_dilation = op.getLhsDilation(); + auto rhs_dilation = op.getRhsDilation(); + auto feature_group_count = op.getFeatureGroupCount(); + int64_t batch_group_count = op.getBatchGroupCount(); + + auto input_args_have_static_shape = [&]() -> bool { + return mlir::cast(input_tensor.getType()).hasStaticShape() && + mlir::cast(kernel_tensor.getType()).hasStaticShape() && + mlir::cast(window_strides.getType()).hasStaticShape() && + mlir::cast(padding.getType()).hasStaticShape() && + mlir::cast(lhs_dilation.getType()).hasStaticShape() && + mlir::cast(rhs_dilation.getType()).hasStaticShape() && + mlir::cast(feature_group_count.getType()) + .hasStaticShape(); + }; + + // Return failure when one of the input args has not a static shape + if (!input_args_have_static_shape()) { + return failure(); + } + + auto input_tensor_shape = + mlir::cast(input_tensor.getType()).getShape(); + auto kernel_tensor_shape = + mlir::cast(kernel_tensor.getType()).getShape(); + + if (input_tensor_shape.size() <= 2) { + return op.emitOpError() + << "input tensor argument is " << input_tensor_shape.size() + << " which is invalid, since input tensor argument must has a " + << "rank greater than 2.\n"; + } + + if (kernel_tensor_shape.size() <= 2) { + return op.emitOpError() + << "kernel tensor argument is " << kernel_tensor_shape.size() + << " which is invalid, since kernel tensor argument must has a " + << "rank greater than 2.\n"; + } + + if (input_tensor_shape.size() != kernel_tensor_shape.size()) { + return op.emitOpError() << "both input tensor and kernel tensor must " + << "have same number of dimensions.\n"; + } + + DenseElementsAttr feature_group_count_attr; + xla::ConvolutionDimensionNumbers dnums; + dnums.ParseFromString(op.getDimensionNumbersAttr().getValue().str()); + if (dnums.input_spatial_dimensions_size() != + dnums.kernel_spatial_dimensions_size()) { + return op.emitOpError() << "Both arguments to convolution must have " + << "same number of dimensions.\n"; + } + + if (dnums.input_spatial_dimensions_size() != + dnums.output_spatial_dimensions_size()) { + return op.emitOpError() << "Both input and output of convolution must have " + << "same number of dimensions.\n"; + } + if (!matchPattern(feature_group_count, + m_Constant(&feature_group_count_attr))) { + return success(); + } + + auto feature_group_count_val = + feature_group_count_attr.getValues()[0].getInt(); + auto input_features = input_tensor_shape[dnums.input_feature_dimension()]; + auto input_batch = input_tensor_shape[dnums.input_batch_dimension()]; + auto kernel_input_features = + kernel_tensor_shape[dnums.kernel_input_feature_dimension()]; + auto kernel_output_features = + kernel_tensor_shape[dnums.kernel_output_feature_dimension()]; + + if (feature_group_count_val <= 0) { + return op.emitOpError() + << "feature_group_count must be a positive number, got " + << feature_group_count_val; + } + + if (batch_group_count <= 0) { + return op.emitOpError() + << "batch_group_count must be a positive number, got " + << batch_group_count; + } + if (batch_group_count > 1 && feature_group_count_val > 1) { + return op.emitOpError() + << "both batch_group_count " << batch_group_count + << "and feature_group_count " << feature_group_count_val + << " cannot be greater than 1"; + } + if (kernel_output_features % batch_group_count != 0) { + return op.emitOpError() + << "Expected output feature dimension size (value " + << kernel_output_features + << ") to be a multiple of batch group count " << batch_group_count; + } + if (input_features % feature_group_count_val != 0 || + input_features / feature_group_count_val != kernel_input_features) { + return op.emitOpError() + << "Expected the size of kernel_input_features (value " + << kernel_input_features + << ") in rhs times feature_group_count (value " + << feature_group_count_val + << ") in lhs should equal the size of the z dimension (value " + << input_features << ") in lhs.\n"; + } + if (kernel_output_features % feature_group_count_val > 0) { + return op.emitOpError() << "Expected output feature dimension (value " + << kernel_output_features << ") to be divisible by " + << "feature_group_count (value " + << feature_group_count_val << ").\n"; + } + if (input_batch % batch_group_count != 0) { + return op.emitOpError() + << "Expected input batch dimension (value " << input_batch + << " ) to be divisible by batch_group_count (value " + << batch_group_count << "); "; + } + return success(); +} + +bool ShapeInference::InferShapeForXlaConvV2Op(XlaConvV2Op op) { + DCOMMENT_OP(op, "Inferring shape for XlaConvV2Op"); + + bool changed = false; + + if (PrecheckForXlaConvV2Op(op).failed()) { + return changed; + } + + auto input_tensor = op.getLhs(); + auto kernel_tensor = op.getRhs(); + auto window_strides = op.getWindowStrides(); + auto padding = op.getPadding(); + auto lhs_dilation = op.getLhsDilation(); + auto rhs_dilation = op.getRhsDilation(); + int64_t batch_group_count = op.getBatchGroupCount(); + + DenseIntElementsAttr window_strides_attr, padding_attr, lhs_dilation_attr, + rhs_dilation_attr; + if (matchPattern(window_strides, m_Constant(&window_strides_attr)) && + matchPattern(padding, m_Constant(&padding_attr)) && + matchPattern(lhs_dilation, m_Constant(&lhs_dilation_attr)) && + matchPattern(rhs_dilation, m_Constant(&rhs_dilation_attr))) { + llvm::SmallVector input_tensor_dims_vec, kernel_tensor_dims_vec, + window_strides_vec, lhs_dilations_vec, rhs_dilations_vec; + llvm::SmallVector> padding_pairs( + padding_attr.getNumElements() / 2); + xla::ConvolutionDimensionNumbers dnums; + dnums.ParseFromString(op.getDimensionNumbersAttr().getValue().str()); + + auto input_tensor_shape = + mlir::cast(input_tensor.getType()); + for (auto i = 0; i < input_tensor_shape.getShape().size(); ++i) { + DCOMMENT("Input Tensor Shape " << i << "th is " + << input_tensor_shape.getShape()[i]); + input_tensor_dims_vec.push_back(input_tensor_shape.getShape()[i]); + } + + auto kernel_tensor_shape = + mlir::cast(kernel_tensor.getType()); + for (auto i = 0; i < kernel_tensor_shape.getShape().size(); ++i) { + DCOMMENT("Kernel tensor Shape" << i << "th is " + << kernel_tensor_shape.getShape()[i]); + kernel_tensor_dims_vec.push_back(kernel_tensor_shape.getShape()[i]); + } + + for (const llvm::APInt& i : window_strides_attr) { + window_strides_vec.push_back(i.getSExtValue()); + } + + for (auto i = 0; i < padding_pairs.size(); ++i) { + padding_pairs[i] = { + padding_attr.getValues()[i * 2].getInt(), + padding_attr.getValues()[i * 2 + 1].getInt()}; + } + + for (const llvm::APInt& i : lhs_dilation_attr) { + lhs_dilations_vec.push_back(i.getSExtValue()); + } + + for (const llvm::APInt& i : rhs_dilation_attr) { + rhs_dilations_vec.push_back(i.getSExtValue()); + } + + Type input_tensor_element_type = input_tensor_shape.getElementType(); + Type result_element_type = op.getType().getElementType(); + Type element_type = input_tensor_element_type.getIntOrFloatBitWidth() >= + result_element_type.getIntOrFloatBitWidth() + ? input_tensor_element_type + : result_element_type; + auto output_shape = InferXlaConvOutputShape( + input_tensor_dims_vec, kernel_tensor_dims_vec, window_strides_vec, + padding_pairs, lhs_dilations_vec, rhs_dilations_vec, batch_group_count, + dnums, element_type); + + if (output_shape.value()) { + changed = RefineResultType(op.getOperation(), op.getResult(), + output_shape.value()); + return changed; + } + } + return changed; +} + +bool ShapeInference::RefineWithInferTypeOpInterface( + InferTypeOpInterface infer_ti) { + Operation* op = infer_ti.getOperation(); + if (none_of(op->getResultTypes(), CanBeRefined)) { + LLVM_DEBUG(llvm::dbgs() << "Skipping inference for statically shaped op '" + << op->getName() << "'.\n"); + return false; + } + + SmallVector inferred; + LogicalResult res = infer_ti.inferReturnTypes( + op->getContext(), op->getLoc(), op->getOperands(), + op->getAttrDictionary(), op->getPropertiesStorage(), op->getRegions(), + inferred); + if (failed(res)) { + op->emitOpError("failed to refine type as inference failed"); + return false; + } + + if (inferred == op->getResultTypes()) return false; + + // Map each of the results of the call to the returned type of the + // function. + bool changed = false; + for (auto [result, inferred_type] : zip(op->getResults(), inferred)) { + auto result_type = result.getType(); + auto new_type = inferred_type; + if (!llvm::isa(op)) { + // TODO: b/361179755 - Remove when folding issue is resolved or proper + // shape inference is confirmed for zeros like. + new_type = TypeMeet(inferred_type, result_type); + } + if (new_type == result_type) { + continue; + } + if (!UpdateTypeAndInsertIncompatibleUseCasts(new_type, result)) { + continue; + } + changed = true; + } + return changed; +} + +ShapeHandle ShapeInference::ComputeOutputAsShape(OpResult result, + InferenceContext* ic) { + LLVM_DEBUG(result.print(llvm::dbgs() << "\nEvaluate partially ")); + auto rt = mlir::dyn_cast(result.getType()); + if (!rt || !rt.hasStaticShape() || rt.getRank() != 1) return {}; + int dim_size = rt.getDimSize(0); + + // Worklist to direct partial evaluation. + SmallVector worklist; + + // Simple evaluator that attempts to partially evaluate the input value even + // if unable to evaluate the complete output. Below follows a simple stack + // based evaluation where it queries what operands/part of operands need to + // be evaluated and attempting to partially evaluate those operands. It does + // so by pushing the operands that need to be required on to the worklist + // before enqueuing the operation requiering those values. + std::vector dims(dim_size, ic->UnknownDim()); + for (unsigned int i = 0, e = dims.size(); i != e; ++i) { + LLVM_DEBUG(llvm::dbgs() << "\nConsidering output dim " << i << "\n"); + + worklist.push_back( + ValuePort{result.getOwner(), {result.getResultNumber(), i}}); + while (!worklist.empty()) { + auto front = worklist.pop_back_val(); + LLVM_DEBUG(front.print(llvm::dbgs() << "\nWorklist front ")); + + SmallVector inputs; + auto res = ComputeInputsRequiredForOutput(front, &inputs); + if (failed(res)) { + // Abort if unable to find which required inputs need to be computed. + worklist.clear(); + break; + } + + if (!inputs.empty()) { + // Enqueue required computation followed by its required operands in + // stack. + worklist.push_back(std::move(front)); + for (auto& it : inputs) worklist.push_back(std::move(it)); + continue; + } + + auto ret = ComputeOutputComponent(front); + if (!ret) continue; + + LLVM_DEBUG(ret.print(llvm::dbgs() << "\ncomputed result = ")); + + // If worklist is empty, then this is the root query op. + if (worklist.empty()) { + LLVM_DEBUG(llvm::dbgs() << "[root node]\n"); + if (auto dea = mlir::dyn_cast(ret)) { + if (dea.getNumElements() != 1) { + LLVM_DEBUG(llvm::dbgs() << "Unexpected number of elements\n"); + return {}; + } + int64_t val = (*dea.getValues().begin()).getSExtValue(); + dims[i] = ic->MakeDim(val); + } + } + } + } + return ic->MakeShape(dims); +} + +bool ShapeInference::ForceTypeForPassThroughOperands(Operation* op, + OperandRange operands, + ResultRange results) { + bool changed = false; + for (auto entry : llvm::zip(operands, results)) { + Type operand_type = std::get<0>(entry).getType(); + Value result = std::get<1>(entry); + TensorType result_type = dyn_cast(result.getType()); + if (result_type == operand_type) continue; + + if (!UpdateTypeAndInsertIncompatibleUseCasts(operand_type, result)) + continue; + changed = true; + } + return changed; +} + +bool ShapeInference::RefineTypeForPassThroughOperands(Operation* op, + OperandRange operands, + ResultRange results) { + bool changed = false; + for (auto entry : llvm::zip(operands, results)) { + Type operand_type = std::get<0>(entry).getType(); + Value result = std::get<1>(entry); + TensorType result_type = mlir::cast(result.getType()); + Type inferred_type = TypeMeet(result_type, operand_type); + if (result_type == inferred_type) continue; + + if (!UpdateTypeAndInsertIncompatibleUseCasts(inferred_type, result)) + continue; + changed = true; + } + return changed; +} + +bool ShapeInference::RefineShapeForPassThroughOps(Operation* op) { + DCOMMENT_OP(op, "Pass through op"); + bool changed = false; + for (auto entry : llvm::zip(op->getOperands(), op->getResults())) { + Value operand = std::get<0>(entry); + Value result = std::get<1>(entry); + Type inferred_type = TypeMeet(result.getType(), operand.getType()); + if (result.getType() == inferred_type) continue; + if (!UpdateTypeAndInsertIncompatibleUseCasts(inferred_type, result)) + continue; + changed = true; + } + return changed; +} + +bool ShapeInference::InferShapeForNonTFDialectOperation(Operation* op) { + if (auto graph_op = dyn_cast(op)) { + return ForceTypeForPassThroughOperands(graph_op.GetFetch(), + graph_op.GetFetch().getFetches(), + op->getResults()); + } + if (auto island_op = dyn_cast(op)) { + return ForceTypeForPassThroughOperands(island_op.GetYield(), + island_op.GetYield().getFetches(), + op->getResults()); + } + if (auto iter_sink = dyn_cast(op)) { + auto iter_source = cast( + iter_sink.getToken().getDefiningOp()); + return RefineTypeForPassThroughOperands( + op, iter_sink.getOperands().drop_front().take_front(), + iter_source.getResults()); + } + if (auto launch_op = dyn_cast(op)) { + auto terminator = launch_op.GetBody().getTerminator(); + return RefineTypeForPassThroughOperands(op, terminator->getOperands(), + op->getResults()); + } + if (auto cluster_op = dyn_cast(op)) { + auto terminator = cluster_op.GetBody().getTerminator(); + return RefineTypeForPassThroughOperands(op, terminator->getOperands(), + op->getResults()); + } + if (op->hasTrait()) + return RefineShapeForPassThroughOps(op); + if (auto call = dyn_cast(op)) return InferShapeForCall(call); + if (isa(op)) return InferShapeForCast(op); + return false; +} + +// Finds element type to be used for result from operand, with special handling +// for handle types. +Type GetElementTypeFromOperand(TensorType operand_type, + TensorType result_type) { + auto operand_handle_type = + mlir::dyn_cast(operand_type.getElementType()); + if (!operand_handle_type) return result_type.getElementType(); + auto result_handle_type = + mlir::cast(result_type.getElementType()); + if (operand_handle_type.GetSubtypes().empty() || + !result_handle_type.GetSubtypes().empty()) + return result_type.getElementType(); + return operand_handle_type; +} + +// Checks if one tensor type can refine another type for tf.While/ +// tf.WhileRegion. If rank differs or static dimensions can be lost, the other +// type cannot be used for refinement. +bool CanWhileTypeBeRefinedWith(TensorType current_type, + TensorType potential_refined_type) { + if (!current_type.hasRank()) return true; + if (!potential_refined_type.hasRank()) return false; + if (current_type.getRank() != potential_refined_type.getRank()) return false; + for (auto dim : + llvm::zip(current_type.getShape(), potential_refined_type.getShape())) { + int64_t current_dim = std::get<0>(dim); + int64_t potential_refined_dim = std::get<1>(dim); + if (current_dim != potential_refined_dim && + current_dim != ShapedType::kDynamic) + return false; + } + return true; +} + +template +bool ShapeInference::InferShapeForWhile(WhileOpTy op, + TypeRange body_result_types) { + if (!op.getShapeInvariant()) + return RefineTypeForPassThroughOperands(op, op.getInput(), op.getOutput()); + + bool changed = false; + for (auto entry : + zip(op.getInput().getTypes(), op.getOutput(), body_result_types)) { + Value result = std::get<1>(entry); + TensorType body_result_type = mlir::cast(std::get<2>(entry)); + auto result_type = mlir::cast(result.getType()); + + Type potential_refined_type; + if (CanWhileTypeBeRefinedWith(result_type, body_result_type)) { + Type element_type = + GetElementTypeFromOperand(body_result_type, result_type); + potential_refined_type = CreateTensorType( + body_result_type.hasRank() ? body_result_type.getShape() + : std::optional>(), + element_type); + } else { + TensorType operand_type = mlir::cast(std::get<0>(entry)); + Type element_type = GetElementTypeFromOperand(operand_type, result_type); + potential_refined_type = CreateTensorType( + result_type.hasRank() ? result_type.getShape() + : std::optional>(), + element_type); + } + changed |= RefineResultType(op, result, potential_refined_type); + } + return changed; +} + +bool ShapeInference::InferShapeForSingleOperation(Operation* op, + int64_t max_iterations) { + LLVM_DEBUG(op->print(llvm::dbgs() << "InferShapeForSingleOperation for "); + llvm::dbgs() << "\n"); + assert(tf_dialect_ == op->getDialect()); + // The shape function of these ops sometimes does not propagate subtypes + // (handle shapes) for resource and variant types. We use a simple passthrough + // to make sure they are preserved in the output. + if (isa(op)) { + return RefineTypeForPassThroughOperands(op, op->getOperands(), + op->getResults()); + } + + // The shape inference function for `ReduceDatasetOp` should always be + // executed regardless of whether the result type can be refined. + if (auto reduce_dataset_op = dyn_cast(op)) { + // TODO(jpienaar): The output type of these ops need to be refined. + return InferShapeForReduceDataset(reduce_dataset_op, max_iterations); + } + + // If no result for this op needs shape inference, we have a fast-path return. + // But if the type is a resource/variant, we do not skip it because we might + // not have the handle shapes. + if (none_of(op->getResultTypes(), CanBeRefined)) { + if (auto host_compute_op = dyn_cast(op)) { + LLVM_DEBUG(llvm::dbgs() << "Keep inference for statically shaped op '" + << op->getName() << "'.\n"); + } else { + LLVM_DEBUG(llvm::dbgs() << "Skipping inference for statically shaped op '" + << op->getName() << "'.\n"); + return false; + } + } + + if (isa(op)) return InferShapeForRestore(op); + + // Handle call operations by looking up callee and inferring return shape as + // needed. + if (auto call = dyn_cast(op)) return InferShapeForCall(call); + + // tf.Cast is only inferred if it has at least one user in the TF dialect or + // feeding into the function return. This is necessary to avoid inserting + // casts which cannot be refined. + if (isa(op)) return InferShapeForCast(op); + + // Handle IfOp here by inferring the shape from the else/then function + // results. Since `output_shapes` is a derived attribute, avoid going down the + // TF InferenceContext path as IfOp shape inference is implemented as just + // a lookup of the output_shapes attribute. + if (auto if_op = dyn_cast(op)) return InferShapeForIf(if_op); + + // Handle IfRegion operations by inferring return shape from the then and else + // branches. + if (auto if_region = dyn_cast(op)) + return InferShapeForIfRegion(if_region); + + if (auto case_op = dyn_cast(op)) return InferShapeForCase(case_op); + + if (auto case_region = dyn_cast(op)) + return InferShapeForCaseRegion(case_region); + + if (auto while_op = dyn_cast(op)) + return InferShapeForWhile( + while_op, while_op.body_function().getFunctionType().getResults()); + + if (auto while_region = dyn_cast(op)) + return InferShapeForWhile( + while_region, + while_region.getBody().front().getTerminator()->getOperandTypes()); + + if (auto xla_call_module = dyn_cast(op)) { + return InferShapeForXlaCallModule(xla_call_module); + } + + if (auto host_compute_op = dyn_cast<_XlaHostComputeMlirOp>(op)) { + return InferShapeForXlaHostComputeMlir(host_compute_op); + } + + if (auto host_compute_op = dyn_cast(op)) { + return InferShapeForFunctionAttachedToXlaHostCompute(host_compute_op); + } + + // TODO(jpienaar): Extract function input arg constraint interface. + // TODO(jpienaar): Unify the shape propagation to functions using interface. + if (auto map_dataset_op = dyn_cast(op)) { + // TODO(jpienaar): The output type of these ops need to be refined. + return InferShapeForMapDataset(map_dataset_op, max_iterations); + } + + if (auto takewhile_dataset_op = dyn_cast(op)) { + // TODO(jpienaar): The output type of these ops need to be refined. + return InferShapeForTakeWhileDataset(takewhile_dataset_op, max_iterations); + } + + // Handle TensorList init operations by inferring shape from TensorList write + // operations. If we are unable to refine element shape here, proceed to use + // the InferenceContext below to get more precise shapes. + if (IsTensorListInitOp(op) && InferShapeForTensorListInitOps(op)) return true; + + if (auto pop_back = dyn_cast(op)) { + return InferShapeForTensorListPopBackOp(pop_back); + } + + if (auto var_handle_op = dyn_cast(op)) { + return InferShapeForVarHandleOp(var_handle_op); + } + + if (auto xla_reduce_window_op = dyn_cast(op)) { + return InferShapeForXlaReduceWindowOp(xla_reduce_window_op); + } + + if (auto xla_select_and_scatter_op = dyn_cast(op)) { + return InferShapeForXlaSelectAndScatterOp(xla_select_and_scatter_op); + } + + if (auto xla_conv_v2_op = dyn_cast(op)) { + return InferShapeForXlaConvV2Op(xla_conv_v2_op); + } + + if (auto xla_gather_op = dyn_cast(op)) { + return InferShapeForXlaGatherOp(xla_gather_op); + } + + // Return operand as a constant attribute. + auto operand_as_constant_fn = [&](Value operand) { + ValuePort vp(operand); + Attribute attr = ComputeOutputComponent(vp); + if (!attr && matchPattern(operand, m_Constant(&attr))) + RecordValue(vp, attr); + return attr; + }; + + // Return op result as a shape. + auto op_result_as_shape_fn = [&](InferenceContext& context, + OpResult op_result) { + return ComputeOutputAsShape(op_result, &context); + }; + + // Return result element type at `index`. + auto result_element_type_fn = [&](int index) { + return mlir::cast(op->getResult(index).getType()) + .getElementType(); + }; + + llvm::SmallVector inferred_return_shapes; + if (failed(InferReturnTypeComponentsForTFOp( + /*location=*/std::nullopt, op, graph_version_, operand_as_constant_fn, + op_result_as_shape_fn, result_element_type_fn, + inferred_return_shapes))) + return false; + + // Update the shape for each of the operation result if the InferenceContext + // has more precise shapes recorded. + bool changed = false; + for (auto result : llvm::zip(op->getResults(), inferred_return_shapes)) { + Value op_result = std::get<0>(result); + if (!CanBeRefined(op_result.getType())) continue; + + ShapedTypeComponents inferred = std::get<1>(result); + TensorType inferred_type; + if (inferred.hasRank()) { + inferred_type = + RankedTensorType::get(inferred.getDims(), inferred.getElementType()); + + } else { + inferred_type = UnrankedTensorType::get(inferred.getElementType()); + } + inferred_type = + mlir::cast(TypeMeet(op_result.getType(), inferred_type)); + if (op_result.getType() == inferred_type) continue; + if (!UpdateTypeAndInsertIncompatibleUseCasts(inferred_type, op_result)) + continue; + changed = true; + } + + if (changed) DCOMMENT_OP(op, "Modified after shape inference:"); + return changed; +} + +FailureOr ShapeInference::PropagateShapeToFunctions( + ModuleOp module, TypeRange input_types, ArrayRef functions, + int64_t max_iterations) { + bool any_failure = false; + bool any_nonconvergence = false; + // If shape propagation fails for one function, return failure, but do not + // early exit and attempt to propagate shapes for all provided functions to + // have a best-effort propagation. + for (func::FuncOp func : functions) { + DCOMMENT("Propating shape to " << func.getName()); + ArrayRef callers = GetCallers(func); + if (!llvm::hasSingleElement(callers) && + !llvm::all_of(callers.drop_front(), [&](Operation* caller) { + /// TODO(aminim): this is overly conservative as some operations + /// (like TPUPartitionedCallOp) may have extra operands that aren't + /// propagated to the callee. + return isa(caller) && + std::equal(caller->getOperandTypes().begin(), + caller->getOperandTypes().end(), + callers.front()->getOperandTypes().begin()); + })) { + if (llvm::any_of(callers, [](Operation* op) { + return isa(op); + })) + func.emitWarning(formatv( + "expected control flow function @{0} to have exactly 1 use, " + "found {1}.", + func.getName(), callers.size())); + + continue; + } + FunctionType func_type = func.getFunctionType(); + func.setType(FunctionType::get(func.getContext(), input_types, + func_type.getResults())); + + FailureOr failure_or_converged = + PropagateShapeToRegions(input_types, {&func.getBody()}, max_iterations); + if (failed(failure_or_converged)) { + any_failure = true; + continue; + } + any_nonconvergence = any_nonconvergence || !failure_or_converged.value(); + if (failed(InferShapeForFunctionReturnType(func))) any_failure = true; + } + if (any_failure) return failure(); + return any_nonconvergence; +} + +FailureOr ShapeInference::PropagateShapeToRegions( + TypeRange input_types, ArrayRef regions, int64_t max_iterations) { + DCOMMENT("\tPropagating shapes to regions"); + bool any_failure = false; + bool any_nonconvergence = false; + // If shape propagation fails for one region, return failure, but do not + // early exit and attempt to propagate shapes for all provided regions to + // have a best-effort propagation. + for (auto region : regions) { + // Refine region arguments. + Block& entry = region->front(); + assert(llvm::size(input_types) == entry.getNumArguments()); + for (auto it : llvm::zip(entry.getArguments(), input_types)) { + BlockArgument arg = std::get<0>(it); + Type type = std::get<1>(it); + arg.setType(type); + } + + // Propagate shapes into the region. + FailureOr failure_or_converged = + InferShapeUntilFixPoint(region, max_iterations); + if (failed(failure_or_converged)) + any_failure = true; + else if (!failure_or_converged.value()) + any_nonconvergence = true; + } + if (any_failure) return failure(); + return any_nonconvergence; +} + +void ShapeInference::PropagateConstantToCallee(CallOpInterface call_op, + func::FuncOp func, + ModuleOp module) { + auto callers = GetCallers(func); + if (!llvm::hasSingleElement(callers)) return; + + OpBuilder builder(&func.front().front()); + Operation* op = call_op.getOperation(); + // If this is the only caller, and an operand is a constant, propagate + // the constant value inside the function. + for (auto arg : func.getArguments()) { + auto operand = op->getOperand(arg.getArgNumber()); + if (propagate_caller_callee_constants_) { + if (isa_and_nonnull(operand.getDefiningOp())) { + arg.replaceAllUsesWith( + builder.clone(*operand.getDefiningOp())->getResult(0)); + } + continue; + } + + auto known_constant = ComputeOutputComponent(ValuePort(operand)); + if (!known_constant) continue; + LLVM_DEBUG(call_op.print(llvm::dbgs() << "Propagate to calee: "); + known_constant.print(llvm::dbgs() << " constant "); + llvm::dbgs() << "\n"); + RecordValue(ValuePort(arg), known_constant); + } +} + +void ShapeInference::PropagateConstantFromCallee(CallOpInterface call_op, + func::FuncOp func, + ModuleOp module) { + // If the return value is a constant, use the constant as the value of + // the call return. + Operation* op = call_op.getOperation(); + OpBuilder builder(op); + builder.setInsertionPointAfter(op); + for (auto retval : + llvm::enumerate(func.front().getTerminator()->getOperands())) { + if (propagate_caller_callee_constants_) { + auto retval_op = retval.value().getDefiningOp(); + if (isa_and_nonnull(retval_op)) { + op->getResult(retval.index()) + .replaceAllUsesWith(builder.clone(*retval_op)->getResult(0)); + } + continue; + } + + ValuePort vp(retval.value()); + if (auto known_constant = ComputeOutputComponent(vp)) { + LLVM_DEBUG(known_constant.print(llvm::dbgs() << "Propagate constant "); + call_op.print(llvm::dbgs() << "from "); llvm::dbgs() << "\n"); + RecordValue(ValuePort(op->getResult(retval.index())), known_constant); + } + } +} + +bool RankedAndSameRank(TensorType lhs, TensorType rhs) { + return lhs.hasRank() && rhs.hasRank() && lhs.getRank() == rhs.getRank(); +} + +// Creates a compatible RankedTensorType where mismatched dimensions are +// replaced with dynamic sizes. +RankedTensorType GetCompatibleRankedTensorType(RankedTensorType lhs, + RankedTensorType rhs) { + assert(lhs.getRank() == rhs.getRank()); + llvm::SmallVector dims; + dims.reserve(lhs.getRank()); + for (auto dim : llvm::zip(lhs.getShape(), rhs.getShape())) { + int64_t lhs_dim = std::get<0>(dim); + if (lhs_dim == std::get<1>(dim)) { + dims.push_back(lhs_dim); + } else { + dims.push_back(ShapedType::kDynamic); + } + } + return tensorflow::GetTypeFromTFTensorShape( + dims, GetElementTypeFromOperand(lhs, rhs)); +} + +// Finds compatible types to propagate into functions/regions of a shape +// invariant tf.While/tf.WhileRegion. If operand and result types are the same, +// that type is returned. If operand and result types are of the same rank, a +// compatible type with matching dimensions is used. Otherwise functions/regions +// arguments are returned but with the handle type from the operand type. +llvm::SmallVector GetWhileCompatibleTypes( + TypeRange operand_types, TypeRange result_types, + TypeRange region_argument_types) { + llvm::SmallVector types; + types.reserve(operand_types.size()); + for (auto entry : + llvm::zip(operand_types, result_types, region_argument_types)) { + auto operand_type = mlir::cast(std::get<0>(entry)); + auto result_type = mlir::cast(std::get<1>(entry)); + if (operand_type == result_type) { + types.push_back(operand_type); + } else if (RankedAndSameRank(operand_type, result_type)) { + auto potential_refined_type = GetCompatibleRankedTensorType( + mlir::cast(operand_type), + mlir::cast(result_type)); + types.push_back(potential_refined_type); + } else { + auto region_argument_type = mlir::cast(std::get<2>(entry)); + Type element_type = GetElementTypeFromOperand( + mlir::cast(operand_type), region_argument_type); + Type potential_refined_type = CreateTensorType( + region_argument_type.hasRank() ? region_argument_type.getShape() + : std::optional>(), + element_type); + types.push_back(potential_refined_type); + } + } + return types; +} + +FailureOr ShapeInference::PropagateShapeIntoAttachedFunctions( + Operation* op, int64_t max_iterations) { + ModuleOp module = op->getParentOfType(); + if (auto if_op = dyn_cast(op)) { + DCOMMENT("Propagating shapes into If"); + return PropagateShapeToFunctions( + module, if_op.getInput().getTypes(), + {if_op.ResolveThenFunction(&symbol_table_), + if_op.ResolveElseFunction(&symbol_table_)}, + max_iterations); + } else if (auto case_op = dyn_cast(op)) { + SmallVector branches; + case_op.get_branch_functions(branches); + return PropagateShapeToFunctions(module, case_op.getInput().getTypes(), + branches, max_iterations); + } else if (auto while_op = dyn_cast(op)) { + // If `shape_invariant` is set, operand shapes cannot be simply propagated + // to result shapes as the op may have different intermediate shapes (such + // While ops can have different result shapes from operand shapes). + // Compatible shapes must be determined before propagating them. + if (while_op.getShapeInvariant()) { + auto compatible_types = GetWhileCompatibleTypes( + while_op.getInput().getTypes(), while_op.getOutput().getTypes(), + while_op.ResolveBodyFunction(&symbol_table_) + .getFunctionType() + .getInputs()); + return PropagateShapeToFunctions( + module, compatible_types, + {while_op.ResolveCondFunction(&symbol_table_), + while_op.ResolveBodyFunction(&symbol_table_)}, + max_iterations); + } + return PropagateShapeToFunctions( + module, while_op.getInput().getTypes(), + {while_op.ResolveCondFunction(&symbol_table_), + while_op.ResolveBodyFunction(&symbol_table_)}, + max_iterations); + } else if (auto call_op = dyn_cast(op)) { + if (auto func = dyn_cast( + call_op.resolveCallableInTable(&symbol_table_))) { + PropagateConstantToCallee(call_op, func, module); + FailureOr failure_or_converged = PropagateShapeToFunctions( + module, call_op.getArgOperands().getTypes(), {func}, max_iterations); + if (failed(failure_or_converged)) return failure(); + PropagateConstantFromCallee(call_op, func, module); + return failure_or_converged; + } + } else if (isa(op) || + isa(op) || + isa(op) || + isa(op)) { + auto propagate_shape_to = [&](mlir::SymbolRefAttr func_sym) { + auto func = llvm::cast( + mlir::SymbolTable::lookupSymbolIn(module, func_sym)); + mlir::SmallVector types; + for (auto type : func.getFunctionType().getInputs()) { + types.push_back(tensorflow::GetTypeFromTFTensorShape( + {}, getElementTypeOrSelf(type))); + } + return PropagateShapeToFunctions(module, types, {func}, max_iterations); + }; + + if (auto xla_reduce_window_op = dyn_cast(op)) { + return propagate_shape_to(xla_reduce_window_op.getComputation()); + } + if (auto xla_select_and_scatter_op = + dyn_cast(op)) { + return propagate_shape_to(xla_select_and_scatter_op.getSelect()) + .value() && + propagate_shape_to(xla_select_and_scatter_op.getScatter()).value(); + } else if (auto xla_variadic_reduce_v2_op = + dyn_cast(op)) { + return propagate_shape_to(xla_variadic_reduce_v2_op.getReducer()); + } else if (auto xla_variadic_sort_op = + dyn_cast(op)) { + return propagate_shape_to(xla_variadic_sort_op.getComparator()); + } + } + + // TODO(ycao): Implement support for Call op, including function reuse. + + return true; +} + +FailureOr ShapeInference::PropagateShapeIntoAttachedRegions( + Operation* op, int64_t max_iterations) { + if (auto while_op = dyn_cast(op)) { + // If `shape_invariant` is set, operand shapes cannot be simply propagated + // to result shapes as the op may have different intermediate shapes (such + // While ops can have different result shapes from operand shapes). + // Compatible shapes must be determined before propagating them. + if (while_op.getShapeInvariant()) { + auto compatible_types = GetWhileCompatibleTypes( + while_op.getInput().getTypes(), while_op.getOutput().getTypes(), + while_op.getBody().getArgumentTypes()); + return PropagateShapeToRegions(compatible_types, + {&while_op.getCond(), &while_op.getBody()}, + max_iterations); + } + return PropagateShapeToRegions(while_op.getInput().getTypes(), + {&while_op.getCond(), &while_op.getBody()}, + max_iterations); + } + return true; +} + +LogicalResult ShapeInference::TryToFold(Operation* op) { + LLVM_DEBUG(op->print(llvm::dbgs() << "TryToFold "); llvm::dbgs() << "\n"); + // If any output result is known, then the op probably has been computed + // before. + if (op->getNumResults() > 0 && results_[ValuePort(op->getResult(0))]) + return success(); + + SmallVector constant_operands(op->getNumOperands()); + SmallVector fold_results; + + // Check to see if any operands to the operation is constant and whether + // the operation knows how to constant fold itself. + bool some_unknown = false; + for (int i = 0, e = op->getNumOperands(); i != e; ++i) { + if (!(constant_operands[i] = + ComputeOutputComponent(ValuePort(op->getOperand(i))))) + some_unknown = true; + } + + // Attempt to constant fold the operation. + auto abstract_op = op->getRegisteredInfo(); + LogicalResult folded = failure(); + if (abstract_op) { + folded = abstract_op->foldHook(op, constant_operands, fold_results); + } + // Attempt dialect fallback if op's fold hook failed. + if (failed(folded)) { + Dialect* dialect = op->getDialect(); + if (!dialect) return failure(); + // Only attempt TF dialect fallback if there are no unknown operands. + if (some_unknown && dialect == tf_dialect_) return failure(); + auto* interface = dialect->getRegisteredInterface(); + if (!interface) return failure(); + + if (failed(interface->fold(op, constant_operands, fold_results))) + return failure(); + } + + for (auto result : zip(op->getResults(), fold_results)) { + auto fold_result = std::get<1>(result); + Attribute attr = nullptr; + if ((attr = fold_result.dyn_cast())) { + RecordValue(ValuePort(std::get<0>(result)), attr); + } else { + auto value = fold_result.get(); + assert(value.getType() == std::get<0>(result).getType() && + "folder produced value of incorrect type"); + if ((attr = ComputeOutputComponent(ValuePort(value)))) { + DCOMMENT("\t\tValue Result mapped to " << attr); + RecordValue(ValuePort(std::get<0>(result)), attr); + } else { + DCOMMENT("\t\tValue result unmapped, consider value type:" << value); + RefineResultType(op, std::get<0>(result), value.getType()); + } + } + + if (ElementsAttr eattr = mlir::dyn_cast_or_null(attr)) { + if (std::get<0>(result).getType() == eattr.getType()) continue; + + (void)UpdateTypeAndInsertIncompatibleUseCasts(eattr.getType(), + std::get<0>(result)); + } + } + + return success(); +} + +LogicalResult ShapeInference::InferShapeForFunctionReturnType( + func::FuncOp func) { + LLVM_DEBUG(llvm::dbgs() << "Inferring return type for: " << func.getName() + << "\n"); + + // Find any return ops. + SmallVector return_ops; + for (Block& block : func) { + if (auto return_op = dyn_cast(block.getTerminator())) { + return_ops.push_back(return_op); + } + } + + // Skip functions without a return, but don't flag as failure here. + if (return_ops.empty()) return success(); + + // Right now we only handle the case of a single return op. + // To handle multiple return ops, we would need to look at all their shapes + // and come up with a common shape and insert appropriate casts. + if (return_ops.size() != 1) return failure(); + + // Find the return type. + auto return_op = return_ops.front(); + + // Manually fold tf.Cast that precedes the return instruction and only differs + // in shape refinement level. + bool changed = false; + for (OpOperand& arg_op : return_op.getOperation()->getOpOperands()) { + Operation* arg_defining_op = arg_op.get().getDefiningOp(); + if (isa_and_nonnull(arg_defining_op)) { + Value input = arg_defining_op->getOperand(0); + Value result = arg_defining_op->getResult(0); + Type meet = TypeMeet(result.getType(), input.getType()); + if (meet == result.getType()) continue; + + LLVM_DEBUG({ + llvm::errs() << "\tfolding & updating return type "; + result.getType().print(llvm::errs()); + input.getType().print(llvm::errs() << " to "); + llvm::errs() << "\n"; + }); + + // Shape inference should not change the element type. + if (HasCompatibleElementTypes(input.getType(), result.getType()) && + meet == input.getType()) { + arg_op.set(input); + } else { + OpBuilder b(return_op.getOperation()); + auto new_cast_op = InsertCast(b, return_op.getLoc(), meet, input); + if (!new_cast_op) return failure(); + arg_op.set(new_cast_op->getResult(0)); + } + if (result.use_empty()) arg_defining_op->erase(); + changed = true; + } + } + + DCOMMENT("Updating function type"); + func.setType(FunctionType::get(func.getContext(), func.getArgumentTypes(), + return_op.getOperandTypes())); + + if (changed) EnqueueCallers(func); + return success(); +} + +FailureOr ShapeInference::InferShapeUntilFixPoint( + Region* region, int64_t max_iterations) { + bool changed = true; + + // TODO(aminim): we could have a more efficient traversal by guiding the + // traversal with a worklist and reconsider only the nodes for which an + // operand type was inferred. This would need to be careful if working on a + // region that would not be isolated. + for (int iteration = 0; iteration < max_iterations && changed; ++iteration) { + changed = false; + LLVM_DEBUG(llvm::dbgs() + << "Shape inference, iteration " << iteration << "\n"); + auto res = region->walk([&](Operation* op) { + auto abstract_op = op->getRegisteredInfo(); + if (abstract_op && ops_to_skip_.contains(abstract_op->getTypeID())) { + LLVM_DEBUG(llvm::dbgs() + << "Skipping shape inference for explicitly skipped op '" + << op->getName() << "'.\n"); + return WalkResult::advance(); + } + + DCOMMENT_OP(op, "Inferring for"); + if (auto infer_ti = dyn_cast(op)) { + DCOMMENT("\tRefinining with type op interface"); + changed |= RefineWithInferTypeOpInterface(infer_ti); + return WalkResult::advance(); + } + + if (op->getDialect() != tf_dialect_) { + DCOMMENT("\tInfer non-TF dialect"); + changed |= InferShapeForNonTFDialectOperation(op); + return WalkResult::advance(); + } + + // Before attempting inference, just try to compute the folded + // value/shape. + if (succeeded(TryToFold(op)) && + // Folding can "succeed" and yet not all types be refined. In such + // cases we still want to give a try at `InferShapeForSingleOperation` + none_of(op->getResultTypes(), CanBeRefined)) + return WalkResult::advance(); + + // Best-effort shape inference in attached functions. Do not return + // failure even if it doesn't get to fixed point, but propagate "real" + // failure. + if (failed(PropagateShapeIntoAttachedFunctions(op, max_iterations))) { + op->emitWarning() << "unable to refine shape of attached function " + "arguments and bodies"; + return WalkResult::interrupt(); + } + + if (failed(PropagateShapeIntoAttachedRegions(op, max_iterations))) { + op->emitWarning() << "unable to refine shape of attached region " + "arguments and bodies"; + return WalkResult::interrupt(); + } + + changed |= InferShapeForSingleOperation(op, max_iterations); + return WalkResult::advance(); + }); + if (res.wasInterrupted()) return failure(); + } + + if (changed) { + region->getParentOp()->emitWarning() + << "shape inference did not reach stable state after " << max_iterations + << " iterations"; + } + return !changed; +} + +static FailureOr InferShapeForFunction(ShapeInference& context, + func::FuncOp func, + int64_t max_iterations) { + FailureOr failure_or_converged = + context.InferShapeUntilFixPoint(&func.getBody(), max_iterations); + if (failed(failure_or_converged) || !failure_or_converged.value()) + return failure_or_converged; + // TODO(b/156276510): Verify that it is always fine to refine a function's + // return type, as long as we do not change the argument shapes. + if (failed(context.InferShapeForFunctionReturnType(func))) return failure(); + return true; +} + +absl::StatusOr>> ParseArgumentShapes( + absl::string_view input_shapes) { + SmallVector> parsed_shapes; + if (input_shapes.empty()) { + return parsed_shapes; + } + + std::vector>> shapes; + TF_RETURN_IF_ERROR(::tensorflow::ParseNodeShapes(input_shapes, shapes)); + + for (const auto& shape : shapes) { + if (!shape) { + return absl::AbortedError("Missing input argument shapes"); + } + parsed_shapes.push_back(SmallVector(shape->begin(), shape->end())); + } + return parsed_shapes; +} + +FailureOr InferShapeForFunction(func::FuncOp func, + ArrayRef> arg_shapes, + int64_t graph_version, + int64_t max_iterations, + ArrayRef ops_to_skip) { + ShapeInference context(graph_version, func->getParentOfType(), + /*propagate_caller_callee_constants=*/true, + ops_to_skip); + if (arg_shapes.empty()) { + return InferShapeForFunction(context, func, max_iterations); + } + + FunctionType func_type = func.getFunctionType(); + bool needs_refinement = false; + SmallVector new_arg_types; + new_arg_types.reserve(func_type.getNumInputs()); + + // Update argument types in-place using the provided arg_shapes. + for (size_t i = 0; i < func_type.getNumInputs(); ++i) { + ArrayRef shape = arg_shapes[i]; + Type element_type; + if (auto input_ty = + mlir::dyn_cast(func_type.getInput(i))) { + if (input_ty.getRank() != shape.size()) { + return failure(); + } + element_type = input_ty.getElementType(); + } else { + auto unranked_input_ty = + mlir::dyn_cast(func_type.getInput(i)); + if (!unranked_input_ty) { + return failure(); + } + element_type = unranked_input_ty.getElementType(); + } + + auto new_arg_type = GetNewArgType(func_type.getInput(i), shape, + element_type, func.getContext()); + + if (new_arg_type != func_type.getInput(i)) { + // If the new type is more detailed, trigger shape inference. + func.getArgument(i).setType(new_arg_type); + needs_refinement = true; + } + new_arg_types.push_back(new_arg_type); + } + + if (!needs_refinement) return true; + + FailureOr failure_or_converged = + context.InferShapeUntilFixPoint(&func.getBody(), max_iterations); + if (failed(failure_or_converged) || !failure_or_converged.value()) + return failure_or_converged; + + if (failed(context.InferShapeForFunctionReturnType(func))) return failure(); + func.setType(FunctionType::get(func.getContext(), new_arg_types, + func.getFunctionType().getResults())); + + return true; +} + +FailureOr InferModuleShape(ModuleOp module, int64_t max_iterations, + ArrayRef ops_to_skip, + ArrayRef> input_shapes, + bool enable_stablehlo_propagation) { + auto producer_or = tensorflow::GetTfGraphProducerVersion(module); + if (!producer_or.ok()) { + // TODO(jpienaar): Keeping the existing behavior for now but this could + // be relaxed. + LLVM_DEBUG(llvm::dbgs() + << "Skipping inference; " << producer_or.status().ToString()); + return true; + } + int64_t producer = producer_or.value(); + + // TODO(jpienaar): Clean up propagate_NextIterationSinkOp_callee_constants if + // it is no longer needed. + ShapeInference context(producer, module, + /*propagate_caller_callee_constants=*/false, + ops_to_skip); + auto main = module.lookupSymbol("main"); + // Error if no main to refine with input shapes + if (!main && !input_shapes.empty()) { + return module->emitError( + "Input shapes provided but no `main` function found."); + } + + // Add main function to head of queue, refine input shapes if provided + if (main) { + if (!input_shapes.empty()) { + FailureOr failure_or_converged = + InferShapeForFunction(main, input_shapes, producer, + /*max_iterations=*/10, ops_to_skip); + if (failed(failure_or_converged) || !failure_or_converged.value()) + return failure_or_converged; + } + context.enqueue(main); + } + for (auto func : module.getOps()) context.enqueue(func); + // Arbitrarily upper bound the maximum number of functions that get processed + // just to avoid pathological cases. + auto max_iteration = context.QueueSize() * 4; + while (!context.EmptyQueue()) { + func::FuncOp func = context.front(); + FailureOr failure_or_converged = + InferShapeForFunction(context, func, max_iterations); + if (failed(failure_or_converged) || !failure_or_converged.value()) + return failure_or_converged; + context.pop_front(); + + if ((--max_iteration) == 0) { + emitWarning(UnknownLoc::get(module.getContext())) + << "shape inference did not reach stable state after " + << max_iteration << " iterations"; + return false; + } + } + + // Propagate shapes to StableHLO modules, if enabled and there are any + // XlaCallModule ops whose StableHLO modules were refined. + if (enable_stablehlo_propagation) { + if (failed(context.PropagateShapeToStableHloModules())) return failure(); + } + + return true; +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h new file mode 100644 index 0000000000000000000000000000000000000000..f6ec1237c200572491c858980f4b82e82873f451 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h @@ -0,0 +1,91 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SHAPE_INFERENCE_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SHAPE_INFERENCE_H_ + +#include + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project + +namespace mlir { +namespace TF { + +inline constexpr char kMLIRContextSingleThreadVar[] = + "TF_USE_SINGLE_THREAD_MLIR_CONTEXT"; + +// Returns whether type can be further refined. +bool CanBeRefined(Type type); + +// Returns a new arg type based on the shape and element type. If there are +// dynamic bounds attribute to the arg, update the bounds based on the shape +// as well. +Type GetNewArgType(Type old_arg_type, ArrayRef shape, + Type element_type, mlir::MLIRContext* context); + +// Refines all the shapes in a module, skipping the inference for all ops +// whose type is in ops_to_skip. +// Returns a failure() on error, otherwise returns true to indicate that it +// reached convergence, false otherwise. +// If input shapes are provided, first refines the `main` function using +// InferShapeForFunction. +// If enable_stablehlo_propagation is true, then the refined StableHLO ops +// shapes will be propagated by re-serializing StableHLO modules in the +// XlaCallModule ops. propagated. +FailureOr InferModuleShape(ModuleOp module, int64_t max_iterations = 10, + ArrayRef ops_to_skip = {}, + ArrayRef> input_shapes = {}, + bool enable_stablehlo_propagation = false); + +// Given a tensorflow NodeShape string, returns a vector of argument shapes +// that can be used with InferShapeForFunction. +// TF NodeShape uses `,` to separate dimensions, and `:` to separate arguments. +// Ex: 1,2:3,4,5:6,? --> [[1, 2], [3, 4, 5], [6, ?]] +absl::StatusOr>> ParseArgumentShapes( + absl::string_view input_shapes); + +// Given a list of refined shapes matching the function arguments of func, runs +// shape inference over the function to propagate this updated information, +// skipping the inference for all ops whose type is in ops_to_skip. +// If arg_shapes are empty, then argument shapes will be left unchanged. +// Note: This affects the entire module, and changes are not just scoped to the +// function being inferred. +// Returns a failure() on error, otherwise returns true to indicate that it +// reached convergence, false otherwise. +FailureOr InferShapeForFunction(func::FuncOp func, + ArrayRef> arg_shapes, + int64_t graph_version, + int64_t max_iterations = 10, + ArrayRef ops_to_skip = {}); + +// Create a MLIRContext based on the threading setup in the env var. +std::unique_ptr MakeMLIRContextWithThreading(); + +} // namespace TF + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SHAPE_INFERENCE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..f97103c5ed2945fa5c8e9ae7bc50c317be6fb87d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_pass.cc @@ -0,0 +1,89 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_TENSORFLOWSHAPEINFERENCEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// This transformation pass propagate shapes on the TensorFlow graph. +// It is a ModulePass in order to be able to change function types. +class ShapeInference + : public impl::TensorFlowShapeInferencePassBase { + public: + ShapeInference() = default; + explicit ShapeInference(ArrayRef> input_shapes, + bool enable_stablehlo_propagation) + : input_shapes_(input_shapes) { + enable_stablehlo_propagation_ = enable_stablehlo_propagation; + } + void runOnOperation() override { + // Parse `input_arg_shapes_` if provided (test only) + SmallVector> input_shapes_vec; + absl::StatusOr>> parsed_shapes; + if (!input_arg_shapes_.empty()) { + parsed_shapes = ParseArgumentShapes(input_arg_shapes_); + if (!parsed_shapes.ok()) { + getOperation().emitError() << parsed_shapes.status().message(); + return signalPassFailure(); + } + input_shapes_vec = SmallVector>{parsed_shapes->begin(), + parsed_shapes->end()}; + input_shapes_ = input_shapes_vec; + } + + auto failure_or_converged = + InferModuleShape(getOperation(), max_iterations_, /*ops_to_skip=*/{}, + input_shapes_, enable_stablehlo_propagation_); + if (failed(failure_or_converged)) return signalPassFailure(); + if (!failure_or_converged.value()) { + getOperation().emitError() + << "shape inference pass did not reach convergence after " + << max_iterations_; + return signalPassFailure(); + } + } + + private: + ArrayRef> input_shapes_; +}; +} // namespace + +// TODO: b/393025760 - Consider enabling stablehlo propagation by default. +std::unique_ptr> CreateTFShapeInferencePass( + ArrayRef> input_shapes, + bool enable_stablehlo_propagation) { + return std::make_unique(input_shapes, + enable_stablehlo_propagation); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..de9f78e239bf8d7ee7fccd629cf9515687b79210 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/shape_inference_test.cc @@ -0,0 +1,39 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" + +#include + +#include +#include "mlir/IR/MLIRContext.h" // from @llvm-project + +namespace mlir { +namespace TF { +namespace { + +TEST(ShapeInferenceTest, CreateMultiThreadedMLIRContext) { + std::unique_ptr ctx = MakeMLIRContextWithThreading(); + EXPECT_TRUE(ctx->isMultithreadingEnabled()); +} + +TEST(ShapeInferenceTest, CreateSingleThreadedMLIRContext) { + setenv(kMLIRContextSingleThreadVar, "true", 1); + std::unique_ptr ctx = MakeMLIRContextWithThreading(); + EXPECT_FALSE(ctx->isMultithreadingEnabled()); +} + +} // namespace +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sink_constant.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sink_constant.cc new file mode 100644 index 0000000000000000000000000000000000000000..de6f7b5217c58ddc83f2baf7b973ea33c8797f80 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sink_constant.cc @@ -0,0 +1,106 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Debug.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +#define DEBUG_TYPE "tf-executor-sink-constant" + +namespace mlir { +namespace TFDevice { + +namespace { +using ::mlir::TF::ConstOp; + +#define GEN_PASS_DEF_CLUSTERCONSTANTSINKINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class ClusterConstantSinkingPass + : public impl::ClusterConstantSinkingPassBase { + public: + explicit ClusterConstantSinkingPass( + llvm::function_ref filter) + : filter_(filter) {} + + void runOnOperation() override { + getOperation().walk([filter = filter_](tf_device::ClusterOp cluster) { + LLVM_DEBUG(llvm::dbgs() << "Visit " << *cluster.getOperation() << "\n"); + // For each launch op, we find the values used that come from a constant + // defined above and sink these constants in the region body. + // The sunk_constant map keeps a mapping from a ConstOp defined above to + // a sunk clone of it. This allows for reusing a sunk constant with + // multiple uses in the region. + llvm::DenseMap sunk_constant; + Region &body = cluster.getBody(); + visitUsedValuesDefinedAbove(body, [&](OpOperand *use) { + Value constant = use->get(); + auto const_op = dyn_cast_or_null(constant.getDefiningOp()); + if (!const_op) return; + + // Filter constants using user provided predicate function. + if (filter && !filter(cluster, const_op.getValue())) return; + + // We found a constant, try to insert it in the map and re-use its + // cloned value if any. + auto map_entry = sunk_constant.try_emplace(constant, nullptr); + if (!map_entry.second) { + // This constant has already been cloned into the region, reuse it. + use->set(map_entry.first->getSecond().getResult()); + LLVM_DEBUG(llvm::dbgs() << "Re-use sunk constant " << use->get() + << "\n in " << use->get() << "\n"); + if (constant.use_empty()) const_op.erase(); + return; + } + if (constant.hasOneUse()) { + LLVM_DEBUG(llvm::dbgs() << "Moved constant " << constant << "\n"); + const_op.getOperation()->moveBefore(&body.begin()->front()); + return; + } + map_entry.first->getSecond() = const_op.clone(); + body.begin()->getOperations().insert(body.begin()->begin(), + map_entry.first->getSecond()); + use->set(map_entry.first->getSecond().getResult()); + LLVM_DEBUG(llvm::dbgs() << "Sunk cloned constant " << use->get() + << "\n in " << use->get() << "\n"); + }); + }); + } + + private: + llvm::function_ref filter_; +}; + +} // anonymous namespace + +std::unique_ptr> CreateClusterConstantSinkingPass( + llvm::function_ref filter) { + return std::make_unique(filter); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..fa76a5b71b279348c4c59c80d9abfeec893d86a7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/BUILD @@ -0,0 +1,117 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir:__subpackages__", + "//tensorflow/compiler/mlir/tensorflow/transforms:__pkg__", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/internal:__pkg__", + ], + licenses = ["notice"], +) + +gentbl_cc_library( + name = "sparsecore_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"sparsecore_passes.h.inc": [ + "-gen-pass-decls", + "-name=SparseCore", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "sparsecore_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "sparsecore_passes", + hdrs = [ + "sparsecore_passes.h", + ], + textual_hdrs = [ + "sparsecore_passes.h.inc", + ], + deps = [ + ":embedding_pipelining", + ":embedding_program_key", + ":embedding_sequencing", + ":sparsecore_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "embedding_pipelining", + srcs = ["embedding_pipelining.cc"], + hdrs = [ + "sparsecore_passes.h", + ], + deps = [ + ":sparsecore_passes_inc_gen", + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "embedding_sequencing", + srcs = ["embedding_sequencing.cc"], + hdrs = [ + "sparsecore_passes.h", + ], + deps = [ + ":sparsecore_passes_inc_gen", + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "embedding_program_key", + srcs = ["embedding_program_key.cc"], + hdrs = [ + "sparsecore_passes.h", + ], + deps = [ + ":sparsecore_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/mlir_hlo", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_pipelining.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_pipelining.cc new file mode 100644 index 0000000000000000000000000000000000000000..46e00abf2e165b23688a69e613e6fbd0e17b7d19 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_pipelining.cc @@ -0,0 +1,2289 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/****************************************************************************** +This pass implements automated pipelining for TPU embeddings defined using +the TF2 Embedding API. This is designed for applications that have an +embedding lookup on the SparseCore, followed by one or more dense layers on +TensorCores, optionally followed by a backward pass (training update) with +more ops on the SparseCore. Ops are broken up into: + 1. SC forward pass + 2. TC forward/backward pass + 3. SC backward pass + 4. non-TPU loop counter updates +These 4 functions are then staggered so as to enable parallel execution. + +In pseudocode, the algorithm is as follows: + +// Start step 0 +C_0 = cond(args_0) +N_0 = non_tpu(args_0) +if (C_0) { + F_0 = forward(args_0, N_0) + T_0 = core_tpu(args_0, N_0, F_0) + // B_0 = backward() is not evaluated here. +} + +args_1 = update_args(args_0, N_0, T_0) + +// Start step 1 +C_1 = cond(args_1) +N_1 = non_tpu(args_1) +if (C_1) { + F_1 = forward(args_1, N_1) + // T_1 = core_tpu() is not evaluated here. + // B_1 = backward() is not evaluated here. +} + +// Partial update of args. We expect this to be sufficient +// for evaluating cond(). +args_2a = update_args(args_1, N_1) // NO T_1 here + +// Conditional for step 2 +C_2 = cond(args_2) + +new_while_body (new_args) { // starts at i==2 + // Finish step i-2 + B_im2 = backward(args_im2, N_im2, F_im2, T_im2) + + // Advance step i-1 + T_im1 = core_tpu(args_im1, N_im1, F_im1) + + // Finish the update of args_2 + args_i = args_2b = update_args(args_2a, T_im1) + + // Start step i + N_i = non_tpu(args_i) + F_i = forward(args_i, N_i) + + // Conditional update + args_ip1 = update_args(args_i, N_i) // T_i is lagged. + C_ip1 = cond(args_ip1) + + return (...) +} +// Note: the tf.while conditional is based on Ci which is initially C2. The +// tf.while op returns the inputs unmodified if the initial conditional is +// false. Thus, the following special cases hold for N <= 2: +// N==0 | N==1 | N==2 | N==3 +// ----------------------------- +// C_nm2 == C_0 -> false | true | true | true +// C_nm1 == C_1 -> false | false | true | true + +// Finish step N-2 +if (C_nm2) { + backward(args_nm2, N_nm2, F_nm2, T_nm2) +} + +// Finish step N-1 +if (C_nm1) { + T_nm1 = core_tpu(args_nm1, N_nm1, F_nm1) + backward(args_nm1, N_nm1, F_nm1, T_nm1) +} + +// To match the original, un-pipelined while loop, we need to return the +// correct results from the pipelined version. Nominally, we'd like to do +// this: +// if ( NOT(C_nm2) ) { +// return args_nm2 +// } else if (NOT(C_nm1)) { +// return args_nm1 +// } else { +// return args_n +// } +// but we don't have if/else-if operators. We can convert this to a CaseOp. +// Note, if C_nm1==true and C_nm2 must also be true. +branch_index = int(C_nm2) + int(C_nm1) +selected_results = switch(branch_index) { + case 0: return args_nm2 + case 1: return args_nm1 + case 2: return args_n +} +return selected_results +******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +// #include "smartass/brain/ops/flogs_ops.h" +#include "absl/log/log.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Inliner.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +#define GEN_PASS_DEF_EMBEDDINGPIPELININGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h.inc" + +static constexpr char kEmbeddingPipelining[] = "_embedding_pipelining"; +static constexpr char kEmbeddingPipeliningInlineAttr[] = + "_embedding_pipelining_inline"; +static constexpr char kEmbeddingForward[] = "forward"; +static constexpr char kEmbeddingBackward[] = "backward"; +static constexpr char kEmbeddingForwardSequential[] = "forward_sequential"; +static constexpr char kEmbeddingBackwardSequential[] = "backward_sequential"; +static constexpr char kDevice[] = "device"; +static constexpr char kLower[] = "_lower_using_switch_merge"; +static constexpr llvm::StringRef kTpuCompilationStatus = + "_tpu_compilation_status"; + +namespace mlir { +namespace TFDevice { +namespace { + +bool IsResourceType(Type val_type) { + if (auto tensor_type = mlir::dyn_cast(val_type)) { + if (mlir::isa(tensor_type.getElementType())) { + return true; + } + } + return false; +} + +struct EmbeddingPipeliningPass + : public ::impl::EmbeddingPipeliningPassBase { + void getDependentDialects(mlir::DialectRegistry& registry) const override { + registry.insert(); + } + + void runOnOperation() override; +}; + +bool UseEmbeddingPipelining(ModuleOp& module) { + // Enable automated pipelining pass unless: + // 1. The user disables it via flag, or + // 2. The graph contains TF.Summary ops. Graphs like this typically only run + // for a single step which doesn't work in pipelining. + + if (tensorflow::GetBuildXlaOpsPassFlags() + ->tf_xla_disable_full_embedding_pipelining) { + LOG(INFO) << "Embedding pipelining disabled via flag."; + return false; + } + + if (tensorflow::GetBuildXlaOpsPassFlags() + ->tf_xla_disable_full_embedding_pipelining_with_summaries) { + // Detect summaries by looking for key Ops in the graph. It would be better + // to do this via operator attributes rather than looking for a specific op. + WalkResult walk_result = + module.walk([&](TF::WriteSummaryOp op) -> WalkResult { + return WalkResult::interrupt(); + }); + if (walk_result.wasInterrupted()) { + LOG(WARNING) << "TF summaries detected - disabling embedding pipelining."; + return false; + } + } + LOG(INFO) << "Embedding pipelining rewrite enabled."; + return true; +} + +StringAttr GetReplicationAttr(mlir::Operation* op) { + return op->getAttrOfType(TF::kReplicationInfoAttr); +} + +StringAttr GetReplicationAttr(TF::TPUCompilationResultOp op) { + // Special case for getting the replication region for + // TPUCompilationResultsOp. + return op->getAttrOfType(kTpuCompilationStatus); +} + +// Replaces the replication region attribute if it already exists. +void UpdateReplicationAttr(Operation* op, StringAttr attr) { + if (op->hasAttr(TF::kReplicationInfoAttr)) { + op->setAttr(TF::kReplicationInfoAttr, attr); + } +} + +// Replaces the replication region attribute if it already exists. +void UpdateReplicationAttr(TF::TPUCompilationResultOp& op, StringAttr attr) { + // Special case for getting the replication region for + // TPUCompilationResultsOp. + if (op->hasAttr(kTpuCompilationStatus)) { + op->setAttr(kTpuCompilationStatus, attr); + } +} + +// A helper class to inline TF::StatefulPartitionedCall ops +struct Inliner : public InlinerInterface { + Inliner(OpBuilder& builder, SymbolTable& symbol_table) + : InlinerInterface(builder.getContext()), + builder(builder), + symbol_table(symbol_table) {} + + bool isLegalToInline(Operation* call, Operation* callable, + bool wouldBeCloned) const override { + return true; + } + bool isLegalToInline(Region* dest, Region* src, bool wouldBeCloned, + IRMapping& valueMapping) const override { + return true; + } + bool isLegalToInline(Operation* op, Region* dest, bool wouldBeCloned, + IRMapping& valueMapping) const override { + return true; + } + + // Don't recursively analyze operations, because they can all be "inlined". + bool shouldAnalyzeRecursively(Operation* op) const override { return true; } + + LogicalResult UnifyReplicationInfo(func::FuncOp func) { + auto new_repl_info = + builder.getStringAttr(func.getSymName().str() + "_repl_info"); + for (auto& op : func.getRegion().getOps()) { + if (auto compile_op = llvm::dyn_cast(op)) { + UpdateReplicationAttr(compile_op, new_repl_info); + } else { + UpdateReplicationAttr(&op, new_repl_info); + } + } + return LogicalResult::success(); + } + + // After inlining, there will likely be some instances where a + // TPUReplicatedInput feeds directly into a TPUReplicatedOutput. Find such + // pairs and remove them. + LogicalResult RemoveOutputInputPairs(func::FuncOp func) { + llvm::SetVector ops_to_erase; + // Inlining can result in multiple TPUCompilationResultOp and + // TPUReplicateMetadataOp ops. Only keep one, the first will do fine. + TF::TPUCompilationResultOp compile_op = nullptr; + for (auto op : func.getRegion().getOps()) { + if (compile_op == nullptr) { + compile_op = op; + } else { + ops_to_erase.insert(op); + } + } + // If there's no outside compilation, we can exit early because this isn't + // a TPU function. + if (compile_op == nullptr) { + return LogicalResult::success(); + } + + TF::TPUReplicateMetadataOp metadata_op = nullptr; + for (auto op : func.getRegion().getOps()) { + if (metadata_op == nullptr) + metadata_op = op; + else + ops_to_erase.insert(op); + } + if (metadata_op == nullptr) { + func->emitError( + "Expected to find TPUReplicateMetadataOps but found none."); + return LogicalResult::failure(); + } + + for (auto output_op : + func.getRegion().getOps()) { + bool outputs_are_returned = false; + TF::TPUReplicatedInputOp input_op = nullptr; + // Only visit each user of the results once. + llvm::SetVector seen_users; + for (auto user : output_op->getUsers()) { + if (!seen_users.insert(user)) continue; + if (llvm::isa(user)) { + if (input_op != nullptr) { + func->emitError( + "Found multiple TPUReplicatedInput ops but only expected 1."); + return LogicalResult::failure(); + } + input_op = llvm::dyn_cast(user); + } + if (llvm::isa(user)) { + outputs_are_returned = true; + } + } + if (input_op == nullptr) continue; + + // If we found matching input ops, we can remove the TPUReplicatedInput + // ops and replace their result values with the inputs to the matching + // TPUReplicatedOutput op. + replaceAllUsesInRegionWith(input_op.getResult(), output_op.getOperand(), + func.getRegion()); + ops_to_erase.insert(input_op); + + // If the outputs aren't also returned from this function, then we can + // remove the TPUReplicatedOutput op as well. In some cases we'll + // still need these ops. + if (!outputs_are_returned) ops_to_erase.insert(output_op); + } + for (auto op : ops_to_erase) op->erase(); + + return LogicalResult::success(); + } + + LogicalResult RemoveDuplicateReplication(func::FuncOp func) { + llvm::SetVector ops_to_erase; + llvm::MapVector cache; + for (auto input_op : func.getRegion().getOps()) { + // We're only expecting a single input argument to be replicated. + if (input_op->getNumOperands() > 1) continue; + Value operand = input_op->getOperand(0); + if (!llvm::isa(operand)) continue; + BlockArgument arg = llvm::dyn_cast(operand); + + // See if we've run across this TPUReplicatedInputOp before. + if (!cache.insert({arg, input_op}).second) { + // We've seen this before. Replace this instance with the cached op. + for (auto p : + llvm::zip(input_op->getResults(), cache[arg]->getResults())) { + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + func.getRegion()); + } + ops_to_erase.insert(input_op); + } + } + for (auto op : ops_to_erase) op->erase(); + return LogicalResult::success(); + } + + LogicalResult PatchCollectiveGatherInstanceKey(func::FuncOp func) { + // We're expecting the original model to have a single CollectiveGatherV2Op + // that gets split into 3 copies in the start_step_0, start_step_1 and + // new_while_body functions. We use global iter id to set the instance key. + // Therefore, we add an offset to the output of the global_iter_id_op to + // make sure the instance keys are unique among them we replace the original + // instance key as: + // global_iter_id = c + original_global_iter_id + // where c = -2, -1 or 0 depending on which function it's being replaced in. + static int64_t offset_value = -2; + for (auto global_iter_id_op : + func.getRegion().getOps()) { + auto loc = global_iter_id_op->getLoc(); + builder.setInsertionPointAfter(global_iter_id_op); + auto offset = TF::ConstOp::create( + builder, loc, builder.getI64IntegerAttr(offset_value)); + auto new_global_iter_id = TF::AddV2Op::create( + builder, loc, global_iter_id_op->getResultTypes(), + global_iter_id_op->getResult(0), offset->getResult(0)); + global_iter_id_op->getResult(0).replaceAllUsesExcept( + new_global_iter_id->getResult(0), new_global_iter_id); + std::vector attr_names = { + TF::kReplicationInfoAttr.str(), "_xla_compile_device_type", + kEmbeddingPipelining, "_xla_outside_compilation", "device"}; + for (const auto& attr_name : attr_names) { + if (!global_iter_id_op->hasAttr(attr_name)) continue; + offset->setAttr(attr_name, global_iter_id_op->getAttr(attr_name)); + new_global_iter_id->setAttr(attr_name, + global_iter_id_op->getAttr(attr_name)); + } + // Make the next function to get inlined use a different offset. + ++offset_value; + } + return LogicalResult::success(); + } + + // Find any StatefulPartitionedCalls and inline their contents in this func. + LogicalResult InlineCallsInFunc(func::FuncOp func, + bool inline_all_funcs = false) { + llvm::SetVector ops_to_erase; + InlinerConfig config; + for (auto caller : + func.getRegion().getOps()) { + if (!inline_all_funcs && + !caller->hasAttr(kEmbeddingPipeliningInlineAttr)) { + continue; + } + Operation* symbol = symbol_table.lookup(caller.getF()); + if (symbol == nullptr) { + func.emitError() << "Symbol not found in SymbolTable: " + << caller.getF(); + return LogicalResult::failure(); + } + if (!llvm::isa(symbol)) { + func.emitError() << "Invalid callee: " << caller.getF(); + return LogicalResult::failure(); + } + auto callee = + llvm::dyn_cast(symbol_table.lookup(caller.getF())); + auto& src_region = callee.getRegion(); + auto result = inlineCall(*this, config.getCloneCallback(), caller, callee, + &src_region, true); + if (failed(result)) { + func.emitError("Inliner failed"); + return result; + } + ops_to_erase.insert(caller); + } + for (auto op : ops_to_erase) op->erase(); + + auto result = PatchCollectiveGatherInstanceKey(func); + if (failed(result)) return result; + + result = UnifyReplicationInfo(func); + if (failed(result)) return result; + + result = RemoveOutputInputPairs(func); + if (failed(result)) return result; + + result = RemoveDuplicateReplication(func); + if (failed(result)) return result; + + return LogicalResult::success(); + } + + private: + OpBuilder& builder; + SymbolTable& symbol_table; +}; + +LogicalResult EliminateResourceLoops(OpBuilder& builder, + SymbolTable& symbol_table, + func::FuncOp func) { + // Examine all StatefulPartitionedCall ops that have resources as return + // types. If the returned resource traces back to an input argument for the + // SPC, then replace uses of the returned copy with the original input. + // + // Note: This does not descend through nested SCPs. + auto ComesFromBlockArgNumber = [](Value val) -> int { + while (true) { + if (auto block_arg = llvm::dyn_cast(val)) { + return block_arg.getArgNumber(); + } + if (auto identity_op = + llvm::dyn_cast(val.getDefiningOp())) { + val = identity_op.getOperand(); + } else { + return -1; + } + } + }; + + for (auto call_op : + func.getRegion().getOps()) { + for (int i = 0; i < call_op->getNumResults(); ++i) { + if (IsResourceType(call_op->getResult(i).getType())) { + Operation* symbol = symbol_table.lookup(call_op.getF()); + if (symbol == nullptr) { + func.emitError() << "Symbol not found in SymbolTable: " + << call_op.getF(); + return LogicalResult::failure(); + } + if (!llvm::isa(symbol)) { + func.emitError() << "Invalid callee: " << call_op.getF(); + return LogicalResult::failure(); + } + auto callee = + llvm::dyn_cast(symbol_table.lookup(call_op.getF())); + func::ReturnOp return_op = *callee.getOps().begin(); + auto val = return_op.getOperand(i); + auto block_arg_number = ComesFromBlockArgNumber(val); + if (block_arg_number >= 0) { + replaceAllUsesInRegionWith(call_op->getResult(i), + call_op->getOperand(block_arg_number), + func.getRegion()); + } + } + } + } + return LogicalResult::success(); +} + +struct Callers { + TF::StatefulPartitionedCallOp forward; + TF::StatefulPartitionedCallOp core_tpu; + TF::StatefulPartitionedCallOp backward; + TF::StatefulPartitionedCallOp non_tpu; +}; + +template +std::vector GetValueTypes(const InputContainer& input) { + // Convert a list of mlir::Value's into a list of mlir::Type's + std::vector types; + types.reserve(input.size()); + for (auto val : input) types.push_back(val.getType()); + return types; +} + +bool IsTPUOp(mlir::Operation* op) { + return op->hasAttr(TF::kReplicationInfoAttr); +} + +template +void Append(Vector& a, const Container& b) { + a.insert(a.end(), b.begin(), b.end()); +} + +template +void Append(Vector& a, const Vector& b) { + a.insert(a.end(), b.begin(), b.end()); +} + +int64_t GetNumOps(func::FuncOp func) { + int64_t num_ops = 0; + for (auto it = func.begin(); it != func.end(); ++it) ++num_ops; + return num_ops; +} + +std::vector ResultsAsVector(Operation* op) { + std::vector vec; + vec.reserve(op->getNumResults()); + for (auto res : op->getResults()) vec.push_back(res); + return vec; +} + +void SetBasicBlockAttributes(OpBuilder& builder, Operation* op) { + op->setAttr(kDevice, builder.getStringAttr("")); + op->setAttr(kLower, builder.getBoolAttr(true)); +} + +std::vector ResultsAsVector(Operation* op, int begin, int num) { + int end = begin + num; + std::vector vec; + vec.reserve(end - begin); + for (int i = begin; i < end; ++i) vec.push_back(op->getResult(i)); + return vec; +} + +void GatherOpsForExtraction(mlir::SetVector* operations, + const mlir::SetVector& ops_to_avoid, + bool predecessors, bool successors) { + // Walk the input and output dependencies of the Ops in `operations` to form + // the closer of Ops needed to evaluate 'operations'. Input dependencies are + // walked if 'predecessors' is true and output dependencies are walked if + // 'successors' is true. In either case, if a discoverd Op is in the + // 'ops_to_avoid' set, then the dependency walking is terminated. + llvm::SetVector ops_to_process(*operations); + llvm::SetVector new_ops; + + while (!ops_to_process.empty()) { + for (Operation* op : ops_to_process) { + if (predecessors) { + for (Value operand : op->getOperands()) { + // Stop at the block boundary. + if (mlir::isa(operand)) continue; + + Operation* predecessor = operand.getDefiningOp(); + if (!operations->contains(predecessor) && + !ops_to_avoid.contains(predecessor)) { + new_ops.insert(operand.getDefiningOp()); + operations->insert(operand.getDefiningOp()); + } + } + } + if (successors) { + for (mlir::Operation* successor : op->getUsers()) { + // Don't include the return op + if (llvm::isa(successor)) continue; + + if (!operations->contains(successor) && + !ops_to_avoid.contains(successor)) { + new_ops.insert(successor); + operations->insert(successor); + } + } + } + } + ops_to_process.swap(new_ops); + new_ops.clear(); + } +} + +TF::StatefulPartitionedCallOp MakeFuncCaller(mlir::OpBuilder& builder, + const Location& loc, + func::FuncOp func, + const ArrayRef& operands, + bool flag_for_inlining) { + // Constructs a tf.StatefulPartitionedCall to the function provided in 'func' + // using the operands in 'operands'. Assumes the insertion point on builder is + // already set. + auto symbol = + mlir::SymbolRefAttr::get(builder.getContext(), func.getSymName()); + auto result_types = func.getResultTypes(); + auto caller = TF::StatefulPartitionedCallOp::create( + builder, loc, result_types, operands, /*arg_attrs=*/nullptr, + /*res_attrs=*/nullptr, symbol, + /*config=*/builder.getStringAttr(""), + /*config_proto=*/builder.getStringAttr(""), + /*executor_type=*/builder.getStringAttr("")); + caller.setFAttr(symbol); + + // Set an attribute that our inliner will look for when choosing which + // TF::StatefulPartitionedCallOps to inline. + if (flag_for_inlining) + caller->setAttr(kEmbeddingPipeliningInlineAttr, builder.getBoolAttr(true)); + return caller; +} + +func::FuncOp CreateFnWithSignature(ModuleOp module, SymbolTable& symbol_table, + const llvm::SetVector& inputs, + const llvm::SetVector& outputs, + const std::string& name) { + // Creates an empty func.FuncOp with a signature compatible with 'inputs' + // (operands) and 'outputs' (results). + OpBuilder builder(module); + auto in_types = GetValueTypes(inputs); + auto out_types = GetValueTypes(outputs); + builder.setInsertionPointToEnd(&module.getBodyRegion().back()); + auto func_op = + func::FuncOp::create(builder, module.getLoc(), name, + builder.getFunctionType(in_types, out_types)); + func_op.setPrivate(); + symbol_table.insert(func_op); + return func_op; +} + +TF::StatefulPartitionedCallOp EncapsulateOpsInFunc( + OpBuilder& builder, SymbolTable& symbol_table, + const llvm::SetVector& ops, + const llvm::SetVector& inputs, const llvm::SetVector& outputs, + func::FuncOp parent_func, ModuleOp module, const std::string& name, + bool flag_for_inlining) { + // Moves all of the Operations in 'ops' into a newly created func.FuncOp + // function named 'name' and replaces the original ops with a call to the + // newly created function using a tf.StatefulPartitionedCall. Here, + // 'parent_func' is the function that holds the original set of ops. + // Note, 'inputs' and 'outputs' are the predetermined set of values that + // should become the operands and return values, respectively. + auto saved_insertion_point = builder.saveInsertionPoint(); + func::FuncOp new_func = + CreateFnWithSignature(module, symbol_table, inputs, outputs, name); + + // This preserves the order of the ops that was in the original parent + // function. This is critical for preserving correctness in the presence of + // resource variables and stateful functions. + std::vector topological_order; + for (Operation& op : parent_func.getOps()) + if (ops.contains(&op)) topological_order.push_back(&op); + + // Create the partitioned call + builder.restoreInsertionPoint(saved_insertion_point); + auto caller = MakeFuncCaller(builder, module.getLoc(), new_func, + inputs.getArrayRef(), flag_for_inlining); + + Block* block = new_func.addEntryBlock(); + for (Operation* op : topological_order) op->moveBefore(block, block->end()); + + // Replace the 'inputs' values with the new function's arguments. + for (auto p : llvm::zip(inputs, new_func.getArguments())) + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + new_func.getBody()); + + builder.setInsertionPointToEnd(block); + func::ReturnOp::create(builder, parent_func.getLoc(), outputs.getArrayRef()); + + // Replace the original 'outputs' values with the result of the call to the + // new function. + for (auto p : llvm::zip(outputs, caller->getResults())) + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + parent_func.getBody()); + + return caller; +} + +void UpdateAndInsertTPUOps(TF::StatefulPartitionedCallOp caller, + TF::TPUReplicateMetadataOp metadata_op, + TF::TPUCompilationResultOp compilation_op, + StringAttr old_group) { + // Adds the TPUReplicateMetatdataOp and TPUCompilationResultOp ops to the + // function called by the provided 'caller'. + mlir::CallInterfaceCallable callable = caller.getCallableForCallee(); + mlir::SymbolRefAttr sym = callable.dyn_cast(); + auto func = llvm::dyn_cast( + mlir::SymbolTable::lookupNearestSymbolFrom(caller, sym)); + OpBuilder builder(func.getBody()); + + StringAttr new_group = builder.getStringAttr( + absl::StrCat(old_group.getValue().str(), caller.getF().str())); + + builder.insert(metadata_op.clone()); + for (Operation& op : func.getOps()) { + if (!IsTPUOp(&op)) continue; + op.setAttr(TF::kReplicationInfoAttr, new_group); + } + TF::TPUCompilationResultOp new_result = compilation_op.clone(); + new_result->setAttr(kTpuCompilationStatus, new_group); + builder.insert(new_result); +} + +template +LogicalResult FindAndExcludeOp(func::FuncOp func, + const StringAttr& replication_attr, + llvm::SetVector& merged_set, + OpType& found_op) { + // Find the TPUReplicationMetadata or TPUCompilationResult ops which will be + // cloned/inserted into each region. We add them to the merged_set so that + // they're ignored when extracting the four main functions. + found_op = nullptr; + for (OpType op : func.getOps()) { + if (found_op != nullptr) { + func.emitOpError() << "number of " << found_op.getOperationName() + << " in loop body is not 1"; + return LogicalResult::failure(); + } + if (GetReplicationAttr(op) != replication_attr) { + op.emitOpError() << "is not part of the replication region " + << replication_attr << " vs " << GetReplicationAttr(op); + return LogicalResult::failure(); + } + found_op = op; + merged_set.insert(found_op); + } + return LogicalResult::success(); +} + +LogicalResult FindOwningWhileOp(func::FuncOp body_func, ModuleOp module, + TF::WhileOp& while_op) { + // Given a while loop body function 'body_func', find the tf.While Op that + // uses it. + auto uses_optional = body_func.getSymbolUses(module); + if (!uses_optional.has_value()) { + body_func.emitOpError() << "no use of while loop body"; + return LogicalResult::failure(); + } + while_op = nullptr; + for (auto& use : uses_optional.value()) { + if (llvm::isa(use.getUser())) { + if (while_op != nullptr) { + use.getUser()->emitOpError() << "multiple users of function."; + return LogicalResult::failure(); + } else { + while_op = llvm::cast(use.getUser()); + } + } else { + use.getUser()->emitOpError() << "non while use of function."; + return LogicalResult::failure(); + } + } + // TODO(bfontain): If the while op is not present we could just split things + // or we wait until the compiler supports multiple regions? + if (while_op == nullptr) { + body_func.emitOpError() << "unable to find while body user."; + return LogicalResult::failure(); + } + return LogicalResult::success(); +} + +LogicalResult FindForwardPassOps(OpBuilder& builder, + llvm::SetVector& forward_pass_ops, + llvm::SetVector& backward_pass_ops, + llvm::SetVector& merged_set, + func::FuncOp loop_body_func, + const int num_replicas) { + // Find all the ops that are to be included in the 'sc_forward' function which + // will be executed on the SparseCore. Note, 'forward_pass_ops' is initially + // seeded with ops from the input MLIR graph that have the + // _embedding_pipelining="forward" attribute which is set by the TF2 Embedding + // API. + // + // When outputs of the forward pass function are used outside of it, we'll + // need to insert a TPUReplicatedOutput Op and include that in the + // forward_pass_ops. And if that usage is also on the TPU (either TensorCore + // or SparseCore) we'll need to insert a matching TPUReplicatedInput. We do + // this before the Ops are removed from the original function/graph so that + // function operands and return values are handled automatically. + + // First, walk the op dependencies. + GatherOpsForExtraction(&forward_pass_ops, merged_set, /*predecessors=*/true, + /*successors=*/false); + + // Locate which variable inputs are part of the forwards pass. These will + // also be used in the backwards pass. We need to create a 'private' copy + // of the TpuReplicatedInput for for the fowards pass if there are users + // outside the pass. Note that in the case of the backwards pass existing + // this will be the case. + // This means that when we have put all out sections together some resource + // inputs will have multiple TPUReplicateInput nodes, so we will need a final + // pass to merge these together into the earliest copy. + llvm::SetVector forward_variable_inputs; + + // Validate that the only resource inputs that are read by ops in + // forward_pass_ops are dataset and variable ops. + int64_t resource_count = 0; + for (auto argument : loop_body_func.getArguments()) { + // Check that all resource arguments are either fed to iterator get next + // or a TPUReplicatedInput with is_packed. + + if (IsResourceType(argument.getType())) { + resource_count++; + bool is_variable = false; + bool is_non_variable = false; + bool use_in_forward = false; + bool use_in_not_forward = false; + for (auto user : argument.getUsers()) { + if (llvm::isa(user)) continue; + if (!forward_pass_ops.contains(user)) { + use_in_not_forward = true; + } else { + use_in_forward = true; + } + if (TF::TPUReplicatedInputOp input = + llvm::dyn_cast(user)) { + if (!input.getIsPacked()) { + input.emitOpError() << "unexpected variable input, not packed"; + return LogicalResult::failure(); + } + + if (is_variable) { + input.emitOpError() << "unexpected multiple TPUReplicatedInputOp " + << "for single argument"; + return LogicalResult::failure(); + } + is_variable = true; + } else { + is_non_variable = true; + } + } + if (use_in_forward && use_in_not_forward) { + loop_body_func.emitOpError() + << "resource input " << argument.getArgNumber() + << " is used both in the forwards and not forward passes dataset"; + return LogicalResult::failure(); + } + if (is_non_variable && is_variable) { + loop_body_func.emitOpError() + << "resource input " << argument.getArgNumber() + << " is used both as a variable and not a variable"; + return LogicalResult::failure(); + } + if (is_variable && use_in_forward) + forward_variable_inputs.insert(argument.getArgNumber()); + } + } + + VLOG(3) << "Found " << forward_variable_inputs.size() + << " variables used in forward pass of " << resource_count + << " total resource inputs"; + + // Clone the TPUReplicatedInputs. + int64_t cloned_inputs = 0; + for (int64_t index : forward_variable_inputs) { + Value argument = loop_body_func.getArgument(index); + // Uses of this argument should only be the return and the + // TPUReplicateInputOp. This is checked by the loop above. + Operation* input_ptr = nullptr; + for (Operation* user : argument.getUsers()) { + if (llvm::isa(user)) { + input_ptr = user; + break; + } + } + TF::TPUReplicatedInputOp input = + llvm::cast(input_ptr); + + // Validate that all users of the TPUReplicatedInput are ReadVariable + // or AssignVariable ops and check if any are outside the forwards pass. + bool duplicate_needed = false; + for (Operation* next_user : input.getOutput().getUsers()) { + if (!llvm::isa(next_user) && + !llvm::isa(next_user)) { + next_user->emitOpError() + << "unexpected user of output of TPUReplicatedInputOp"; + return LogicalResult::failure(); + } + if (!forward_pass_ops.contains(next_user)) duplicate_needed = true; + } + if (!duplicate_needed) continue; + + cloned_inputs++; + builder.setInsertionPointAfter(input); + forward_pass_ops.remove(input); + + TF::TPUReplicatedInputOp private_input = input.clone(); + builder.insert(private_input); + forward_pass_ops.insert(private_input); + + input.getOutput().replaceUsesWithIf( + private_input.getOutput(), [&](OpOperand& use) { + return forward_pass_ops.contains(use.getOwner()); + }); + } + + VLOG(3) << "Cloned " << cloned_inputs << " TPUReplicatedInputOps"; + + // Add TPUReplicatedInput/TPUReplicatedOutput pairs along each edge. + llvm::SetVector new_forward_ops; + for (Operation* op : forward_pass_ops) { + // TODO(bfontain): Should validate that all the TPU ops are in the same + // replication region. + if (!IsTPUOp(op)) continue; + for (Value result : op->getResults()) { + std::vector> out_of_region_use; + for (OpOperand& use : result.getUses()) { + auto use_owner = use.getOwner(); + // TODO(bfontain): Error check here, if the use.getOwner() is not a TPU + // then this op must be a TPUReplicatedOutputOp. + if (IsTPUOp(use_owner) && !forward_pass_ops.contains(use_owner)) + out_of_region_use.push_back( + std::make_pair(use_owner, use.getOperandNumber())); + } + if (out_of_region_use.empty()) continue; + builder.setInsertionPointAfter(op); + std::vector types(num_replicas, result.getType()); + TF::TPUReplicatedOutputOp replicated_output = + TF::TPUReplicatedOutputOp::create(builder, op->getLoc(), + TypeRange(types), result); + new_forward_ops.insert(replicated_output); + // TODO(bfontain): Check for other attributes. + replicated_output->setAttr(kDevice, builder.getStringAttr("")); + TF::TPUReplicatedInputOp input = TF::TPUReplicatedInputOp::create( + builder, op->getLoc(), result.getType(), + replicated_output.getResults()); + input->setAttr(kDevice, builder.getStringAttr("")); + mlir::Value new_value = input.getOutput(); + + if (mlir::isa( + result.getDefiningOp())) { + TF::TPUAnnotateTensorsWithDynamicShapeOp annotate_op = + TF::TPUAnnotateTensorsWithDynamicShapeOp::create( + builder, op->getLoc(), result.getType(), new_value, + result.getDefiningOp()->getAttrs()); + for (auto [operation, index] : out_of_region_use) { + if (!backward_pass_ops.contains(operation)) { + operation->emitOpError() + << "expect all dynamic inputs consumed by backwards pass."; + return LogicalResult::failure(); + } + } + + backward_pass_ops.insert(annotate_op); + new_value = annotate_op->getResult(0); + } + for (auto [operation, index] : out_of_region_use) + operation->setOperand(index, new_value); + } + } + + VLOG(3) << "Inserted " << new_forward_ops.size() << " TPU Input/Output ops."; + forward_pass_ops.insert(new_forward_ops.begin(), new_forward_ops.end()); + return LogicalResult::success(); +} + +LogicalResult FindBackwardPassOps( + OpBuilder& builder, llvm::SetVector& backward_pass_ops, + llvm::SetVector& merged_set, const int num_replicas) { + // Find all the ops that are to be included in the 'sc_backward' function + // which will be executed on the SparseCore. Note, 'backward_pass_ops' is + // initially seeded with ops from the input MLIR graph that have the + // _embedding_pipelining="backward" attribute which is set by the TF2 + // Embedding API. + // + // Since we're inserting a replication boundary around the backward pass + // function, we'll also need to make sure TPUReplicatedInputOp and + // TPUReplicatedOutputOp ops are inserted as necessary. + + // First, walk the Ops dependencies. + GatherOpsForExtraction(&backward_pass_ops, merged_set, /*predecessors=*/false, + /*successors=*/true); + + VLOG(3) << "Found " << backward_pass_ops.size() << " backwards pass ops."; + + // If any inputs are to the backward_pass_ops region are direct + // TPUReplicatedInput ops, then include (if this is the only use) or + // clone the op. This will be the case for all Read/Assign variable ops. + + llvm::SetVector to_clone; + llvm::SetVector to_insert; + + for (Operation* op : backward_pass_ops) { + for (OpOperand& input_value : op->getOpOperands()) { + Operation* predecessor_op = input_value.get().getDefiningOp(); + if (TF::TPUReplicatedInputOp input = + llvm::dyn_cast(predecessor_op)) { + if (to_clone.contains(input) || to_insert.contains(input)) continue; + // Check if all uses in backwards pass. + bool all_in_backwards = true; + for (Operation* user : input->getUsers()) + if (!backward_pass_ops.contains(user)) all_in_backwards = false; + if (all_in_backwards) + to_insert.insert(input); + else + to_clone.insert(input); + } + } + } + backward_pass_ops.insert(to_insert.begin(), to_insert.end()); + for (TF::TPUReplicatedInputOp input : to_clone) { + builder.setInsertionPointAfter(input); + TF::TPUReplicatedInputOp private_input = input.clone(); + builder.insert(private_input); + backward_pass_ops.insert(private_input); + input.getOutput().replaceUsesWithIf( + private_input.getOutput(), [&](OpOperand& use) { + return backward_pass_ops.contains(use.getOwner()); + }); + } + + VLOG(2) << " cloned " << to_clone.size() << " and inserted " + << to_insert.size() << " TPUReplicatedInput ops"; + + // For all other inputs that go from TPU op to TPU op, insert the + // TPUOutput/Input pair. + + // Add TPUReplicatedInput/TPUReplicatedOutput pairs along each edge. + // TODO(bfontain): Should be merged with the above loop. + llvm::SetVector values_to_add_nodes; + + for (Operation* op : backward_pass_ops) { + // TODO(bfontain): Should validate that all the TPU ops are in the same + // replication region. + // If the op is already a replicated input, no need to to anything. + if (!IsTPUOp(op) || llvm::isa(op)) continue; + for (OpOperand& input_value : op->getOpOperands()) + // TODO(bfontain): Error check here, this line should never be false, + // since we skip the TF::TPUReplicatedInputOp case. + if (IsTPUOp(input_value.get().getDefiningOp()) && + !backward_pass_ops.contains(input_value.get().getDefiningOp())) + values_to_add_nodes.insert(input_value.get()); + } + + for (Value value : values_to_add_nodes) { + builder.setInsertionPointAfter(value.getDefiningOp()); + std::vector types(num_replicas, value.getType()); + Location loc = value.getDefiningOp()->getLoc(); + TF::TPUReplicatedOutputOp output = TF::TPUReplicatedOutputOp::create( + builder, loc, TypeRange(types), value); + // TODO(bfontain): Check for other attributes. + output->setAttr(kDevice, builder.getStringAttr("")); + TF::TPUReplicatedInputOp input = TF::TPUReplicatedInputOp::create( + builder, loc, value.getType(), output.getResults()); + input->setAttr(kDevice, builder.getStringAttr("")); + value.replaceUsesWithIf(input.getOutput(), [&](OpOperand& use) { + return backward_pass_ops.contains(use.getOwner()); + }); + backward_pass_ops.insert(input); + } + + VLOG(2) << " inserted " << values_to_add_nodes.size() + << " TPUReplicatedInput/Output pairs"; + return LogicalResult::success(); +} + +LogicalResult FindCoreTPUOps( + llvm::SetVector& core_tpu_ops, + const llvm::SetVector& forward_pass_ops, + const llvm::SetVector& backward_pass_ops, + const llvm::SetVector& merged_set, + func::FuncOp loop_body_func) { + // Find all of the Ops that are part of the forward/backward pass but aren't + // targeting the SparseCore. Note that we need to include some non-TPU ops + // that flow out of the forward pass function. Otherwise, they would get + // absorbed into the non_tpu function which breaks the pipelining + // decomposition strategy. + // + // Find all the outputs of the forward pass that aren't fed into the backward + // pass. + for (Operation* op : forward_pass_ops) { + for (Value res : op->getResults()) { + for (auto user : res.getUsers()) { + if (!forward_pass_ops.contains(user) && + !backward_pass_ops.contains(user)) { + core_tpu_ops.insert(user); + } + } + } + } + + // Gather all TPU ops marked for compilation in this while loop body that also + // are not in one of the two other sets. + for (Operation& op : loop_body_func.getOps()) { + // Find all TPU ops that don't belong to the forward or backward pass. + if (merged_set.contains(&op) || llvm::isa(op) || + !IsTPUOp(&op) || op.hasAttr(kEmbeddingPipelining)) + continue; + // TODO(bfontain): only collect those ops in a fixed TPUReplica. + core_tpu_ops.insert(&op); + } + + GatherOpsForExtraction(&core_tpu_ops, merged_set, /*predecessors=*/true, + /*successors=*/true); + + // TODO(patn): Verify that all the ops here fall between the forward pass + // and backward pass ops (i.e., not before the forward pass or after the + // backward pass). + return LogicalResult::success(); +} + +LogicalResult FindNonTPUOps(llvm::SetVector& non_tpu_ops, + const llvm::SetVector& merged_set, + func::FuncOp loop_body_func) { + // Find all of the left over Ops after the sc_forward, sc_backward and + // core_tpu ops have been identified. What's left are just the ops necessary + // for updating loop counters etc. + llvm::SetVector non_tpu_args; + for (Operation& op : loop_body_func.getOps()) { + if (merged_set.contains(&op) || llvm::isa(op) || + op.hasAttr(kEmbeddingPipelining)) + continue; + // Note, there should be no TPU ops left at this point. If this trips, + // there's likely a bug in this pass. + if (IsTPUOp(&op)) { + loop_body_func.emitOpError() + << "Unexpcted TPU op found while identifying non-TPU ops."; + return LogicalResult::failure(); + } + non_tpu_ops.insert(&op); + } + + // Validate that remainder_ops takes and returns a subset of the loop carried + // args. This will basically be our set increment fn. + for (Operation* op : non_tpu_ops) + for (Value input : op->getOperands()) + if (BlockArgument arg = llvm::dyn_cast(input)) + // TODO(bfontain): Check that this is actually an argument to the loop + // body. + non_tpu_args.insert(arg.getArgNumber()); + + // All funcs have a return op so this should be safe. + func::ReturnOp return_op = *loop_body_func.getOps().begin(); + + for (OpOperand& operand : return_op->getOpOperands()) { + if (non_tpu_args.contains(operand.getOperandNumber())) { + if (BlockArgument argument = + llvm::dyn_cast(operand.get())) { + if (argument.getArgNumber() != operand.getOperandNumber()) { + return_op.emitOpError() + << "non TPU ops do not divide state into two pieces."; + return LogicalResult::failure(); + } + } else if (!non_tpu_ops.contains(operand.get().getDefiningOp())) { + return_op.emitOpError() + << "non TPU ops do not divide state into two pieces."; + return LogicalResult::failure(); + } + } + } + return LogicalResult::success(); +} + +LogicalResult ExtractOpsAsFunc( + OpBuilder& builder, ModuleOp module, SymbolTable& symbol_table, + llvm::SetVector& ops, StringAttr replication_attr, + TF::TPUReplicateMetadataOp metadata_op, + TF::TPUCompilationResultOp compilation_op, func::FuncOp parent_func, + const std::string& func_name, TF::StatefulPartitionedCallOp* caller, + bool flag_for_inlining) { + // Move the given set of 'ops' into it's own function and replace them with a + // call to that function ('caller'). if 'metadata_op' and 'compilation_op' are + // non-null, also insert those (i.e., target the resulting function to the + // TPU). Here, 'parent_func' is the func.FuncOp that owns the ops in 'ops'. + // + // Returns in 'caller' a tf.StatefulPartitionedCallOp that calls the function + // that was extracted.. + + // Find the input edges to form the set of operands to the new function call. + llvm::SetVector inputs; + for (Operation* op : ops) { + for (Value operand : op->getOperands()) { + Operation* defining_op = operand.getDefiningOp(); + if (!ops.contains(defining_op)) inputs.insert(operand); + } + } + // Find the output edges to form the set of resutls of the new function call. + llvm::SetVector results; + for (Operation* op : ops) { + for (auto result : op->getResults()) { + for (const OpOperand& operand : result.getUsers()) { + if (!ops.contains(operand.getOwner())) { + results.insert(result); + break; + } + } + } + } + llvm::SetVector outputs; + for (auto output : results) outputs.insert(output); + auto tf_caller = + EncapsulateOpsInFunc(builder, symbol_table, ops, inputs, outputs, + parent_func, module, func_name, flag_for_inlining); + if (!ops.empty() && metadata_op != nullptr && compilation_op != nullptr) + UpdateAndInsertTPUOps(tf_caller, metadata_op, compilation_op, + replication_attr); + *caller = tf_caller; + return LogicalResult::success(); +} + +LogicalResult FindSourceTPUReplicatedOutput( + Value val, TF::TPUReplicatedOutputOp& rep_out) { + Operation* op = val.getDefiningOp(); + if (auto src = llvm::dyn_cast(op)) { + rep_out = src; + return LogicalResult::success(); + } + if (auto src = llvm::dyn_cast(op)) { + return FindSourceTPUReplicatedOutput(src->getOperand(0), rep_out); + } + op->emitOpError() << "Value did not come from a TPUReplicatedOutput op: " + << val; + return LogicalResult::failure(); +} + +int FindReturnIndex(Value val) { + const int not_found = -1; + for (auto user : val.getUsers()) { + if (auto ret_op = llvm::dyn_cast(user)) { + for (auto index = 0; index < ret_op->getNumOperands(); ++index) { + if (val == ret_op->getOperand(index)) { + return index; + } + } + } + if (auto ident_op = llvm::dyn_cast(user)) { + auto index = FindReturnIndex(ident_op->getResult(0)); + if (index != not_found) return index; + } + } + return not_found; +} + +// Skip the assertions because they currently create problematic dependencies. +constexpr bool kDoAssertions = true; + +void AddAssertion(OpBuilder& builder, Location& loc, Value cond, + const std::string& message) { + if (!kDoAssertions) return; + auto shape_type = + RankedTensorType::get({1}, builder.getType()); + auto msg = + TF::ConstOp::create(builder, loc, + DenseStringElementsAttr::get( + shape_type, llvm::ArrayRef{message})); + TF::AssertOp::create(builder, loc, cond, msg.getResult()); +} + +LogicalResult StartStep0(OpBuilder& builder, Location& loc, + SymbolTable& symbol_table, + TF::TPUReplicateMetadataOp& metadata_op, + TF::TPUCompilationResultOp& compilation_op, + Value& cond_value, Callers& callers, + const std::vector& loop_operands_nm0, + TF::StatefulPartitionedCallOp& caller) { + const std::string name = "start_step_0"; + + AddAssertion(builder, loc, cond_value, + "[StartStep0] Auto-pipelining requires at least two steps."); + auto insertion_point = builder.saveInsertionPoint(); + + func::FuncOp orig_parent_func = + callers.backward->getParentOfType(); + + const std::vector& operands = loop_operands_nm0; + + // Input types will be the same as the original loop body. + std::vector input_types = GetValueTypes(operands); + + // Determine the results types. + // Return ALL outputs, respecting the provided order of the Operations. This + // makes it straightforward for users of this function to map the return + // values. + llvm::SetVector ops; + ops.insert(callers.forward); + ops.insert(callers.core_tpu); + std::vector result_map; + result_map.reserve(callers.forward->getNumResults() + + callers.core_tpu->getNumResults()); + int result_pos = 0; + for (auto res : callers.forward->getResults()) { + bool is_output = false; + for (auto user : res.getUsers()) { + if (!ops.contains(user)) { + is_output = true; + break; + } + } + result_map.push_back(is_output ? result_pos++ : -1); + } + std::vector result_types; + Append(result_types, callers.forward->getResultTypes()); + Append(result_types, callers.core_tpu->getResultTypes()); + + // Create the function based on input and result types and values. + auto func_type = + mlir::FunctionType::get(builder.getContext(), input_types, result_types); + func::FuncOp then_func = func::FuncOp::create(loc, name, func_type); + then_func.setPrivate(); + symbol_table.insert(then_func); + mlir::OpBuilder func_builder = + mlir::OpBuilder::atBlockBegin(then_func.addEntryBlock()); + + // This must match the concatenation order in 'operands' above. + IRMapping ir_map; + int pos = 0; + for (auto orig : orig_parent_func.getArguments()) + ir_map.map(orig, then_func.getArgument(pos++)); + + // Clone the specified ops into the new function. + auto new_forward = func_builder.insert(callers.forward->clone(ir_map)); + for (auto p : + llvm::zip(callers.core_tpu->getResults(), new_forward->getResults())) + ir_map.map(std::get<0>(p), std::get<1>(p)); + auto new_core_tpu = func_builder.insert(callers.core_tpu->clone(ir_map)); + + // Add the function return; + std::vector results; + Append(results, new_forward->getResults()); + Append(results, new_core_tpu->getResults()); + func::ReturnOp::create(func_builder, loc, results); + + // Inline any StatefulPartitionCall Ops. + auto result = Inliner(builder, symbol_table).InlineCallsInFunc(then_func); + if (failed(result)) return result; + + builder.restoreInsertionPoint(insertion_point); + caller = MakeFuncCaller(builder, loc, then_func, operands, + /*flag_for_inlining=*/false); + return LogicalResult::success(); +} + +LogicalResult StartStep1(OpBuilder& builder, Location& loc, + SymbolTable& symbol_table, + TF::TPUReplicateMetadataOp& metadata_op, + TF::TPUCompilationResultOp& compilation_op, + Value& cond_value, Callers& callers, + const std::vector& loop_operands_1, + TF::StatefulPartitionedCallOp& caller) { + const std::string name = "start_step_1"; + + AddAssertion(builder, loc, cond_value, + "[StartStep1] Auto-pipelining requires at least two steps."); + + auto insertion_point = builder.saveInsertionPoint(); + func::FuncOp orig_parent_func = + callers.backward->getParentOfType(); + + const std::vector& operands = loop_operands_1; + + // Input types will be the same as the original loop body. + std::vector input_types = GetValueTypes(operands); + + // Determine the results types. + // Return ALL outputs, respecting the provided order of the Operations. This + // makes it straightforward for users of this function to map the return + // values. + auto result_types = callers.forward->getResultTypes(); + + // Create the function based on input and result types and values. + auto func_type = + mlir::FunctionType::get(builder.getContext(), input_types, result_types); + func::FuncOp then_func = func::FuncOp::create(loc, name, func_type); + then_func.setPrivate(); + symbol_table.insert(then_func); + mlir::OpBuilder func_builder = + mlir::OpBuilder::atBlockBegin(then_func.addEntryBlock()); + + // This must match the concatenation order in 'operands' above. + IRMapping ir_map; + int pos = 0; + for (auto orig : orig_parent_func.getArguments()) + ir_map.map(orig, then_func.getArgument(pos++)); + + // Clone the specified ops into the new function. + auto new_forward = func_builder.insert(callers.forward->clone(ir_map)); + + // Add the function return; + func::ReturnOp::create(func_builder, loc, new_forward->getResults()); + + // Inline any StatefulPartitionCall Ops. + auto result = Inliner(builder, symbol_table).InlineCallsInFunc(then_func); + if (failed(result)) return result; + + builder.restoreInsertionPoint(insertion_point); + caller = MakeFuncCaller(builder, loc, then_func, operands, + /*flag_for_inlining=*/false); + return LogicalResult::success(); +} + +LogicalResult FinishStepNm2(OpBuilder& builder, Location& loc, + SymbolTable& symbol_table, + TF::TPUReplicateMetadataOp& metadata_op, + TF::TPUCompilationResultOp& compilation_op, + Value& cond_value, Callers& callers, + const std::vector& loop_operands_nm2, + const std::vector& forward_res_nm2, + const std::vector& core_tpu_res_nm2, + TF::StatefulPartitionedCallOp& caller) { + const std::string name = "finish_step_nm2"; + + AddAssertion(builder, loc, cond_value, + "[FinishStepNm2] Auto-pipelining requires at least two steps."); + + auto insertion_point = builder.saveInsertionPoint(); + func::FuncOp orig_parent_func = + callers.backward->getParentOfType(); + + std::vector operands = loop_operands_nm2; + Append(operands, forward_res_nm2); + Append(operands, core_tpu_res_nm2); + + // Input types will be the same as the original loop body. + std::vector input_types = GetValueTypes(operands); + + // Determine the results types. + // Return ALL outputs, respecting the provided order of the Operations. This + // makes it straightforward for users of this function to map the return + // values. + auto result_types = callers.backward->getResultTypes(); + + // Create the function based on input and result types and values. + auto func_type = + mlir::FunctionType::get(builder.getContext(), input_types, result_types); + func::FuncOp then_func = func::FuncOp::create(loc, name, func_type); + then_func.setPrivate(); + symbol_table.insert(then_func); + mlir::OpBuilder func_builder = + mlir::OpBuilder::atBlockBegin(then_func.addEntryBlock()); + + // This must match the concatenation order in 'operands' above. + IRMapping ir_map; + int pos = 0; + for (auto orig : orig_parent_func.getArguments()) + ir_map.map(orig, then_func.getArgument(pos++)); + for (auto orig : callers.forward->getResults()) + ir_map.map(orig, then_func.getArgument(pos++)); + for (auto orig : callers.core_tpu->getResults()) + ir_map.map(orig, then_func.getArgument(pos++)); + + // Clone the specified ops into the new function. + auto new_backward = func_builder.insert(callers.backward->clone(ir_map)); + + // Add the function return; + func_builder.setInsertionPointAfter(new_backward); + func::ReturnOp::create(func_builder, loc, new_backward->getResults()); + + // Inline any StatefulPartitionCall Ops. + auto result = Inliner(builder, symbol_table).InlineCallsInFunc(then_func); + if (failed(result)) return result; + + builder.restoreInsertionPoint(insertion_point); + caller = MakeFuncCaller(builder, loc, then_func, operands, + /*flag_for_inlining=*/false); + return LogicalResult::success(); +} + +LogicalResult FinishStepNm1(OpBuilder& builder, Location& loc, + SymbolTable& symbol_table, + TF::TPUReplicateMetadataOp& metadata_op, + TF::TPUCompilationResultOp& compilation_op, + Value& cond_value, Callers& callers, + const std::vector& loop_operands_nm1, + const std::vector& forward_res_nm1, + TF::StatefulPartitionedCallOp& caller) { + const std::string name = "finish_step_nm1"; + + AddAssertion(builder, loc, cond_value, + "Auto-pipelining requires at least two steps."); + + auto insertion_point = builder.saveInsertionPoint(); + func::FuncOp orig_parent_func = + callers.backward->getParentOfType(); + + std::vector operands = loop_operands_nm1; + Append(operands, forward_res_nm1); + + // Input types will be the same as the original loop body. + std::vector input_types = GetValueTypes(operands); + + // Determine the results types. + // Return ALL outputs, respecting the provided order of the Operations. This + // makes it straightforward for users of this function to map the return + // values. + std::vector result_types; + Append(result_types, callers.core_tpu->getResultTypes()); + Append(result_types, callers.backward->getResultTypes()); + + // Create the function based on input and result types and values. + auto func_type = + mlir::FunctionType::get(builder.getContext(), input_types, result_types); + func::FuncOp then_func = func::FuncOp::create(loc, name, func_type); + then_func.setPrivate(); + symbol_table.insert(then_func); + mlir::OpBuilder func_builder = + mlir::OpBuilder::atBlockBegin(then_func.addEntryBlock()); + + // This must match the concatenation order in 'operands' above. + IRMapping ir_map; + int pos = 0; + for (auto orig : orig_parent_func.getArguments()) + ir_map.map(orig, then_func.getArgument(pos++)); + for (auto orig : callers.forward->getResults()) + ir_map.map(orig, then_func.getArgument(pos++)); + + // Clone the specified ops into the new function. + auto new_core_tpu = func_builder.insert(callers.core_tpu->clone(ir_map)); + for (auto p : + llvm::zip(callers.core_tpu->getResults(), new_core_tpu->getResults())) + ir_map.map(std::get<0>(p), std::get<1>(p)); + auto new_backward = func_builder.insert(callers.backward->clone(ir_map)); + // Add the function return; + std::vector results; + Append(results, new_core_tpu->getResults()); + Append(results, new_backward->getResults()); + func::ReturnOp::create(func_builder, loc, results); + + // Inline any StatefulPartitionCall Ops. + auto result = Inliner(builder, symbol_table).InlineCallsInFunc(then_func); + if (failed(result)) return result; + + builder.restoreInsertionPoint(insertion_point); + caller = MakeFuncCaller(builder, loc, then_func, operands, + /*flag_for_inlining=*/false); + return LogicalResult::success(); +} + +LogicalResult MakeForwardOperands(Operation* forward_caller, + Operation* non_tpu_caller, + const std::vector& loop_operands, + const std::vector& non_tpu_res, + std::vector& f_operands) { + f_operands.clear(); + f_operands.reserve(forward_caller->getNumOperands()); + for (auto operand : forward_caller->getOperands()) { + if (llvm::isa(operand)) { + // Pull this from the original operands to the original while op. + auto arg = llvm::cast(operand); + f_operands.push_back(loop_operands[arg.getArgNumber()]); + continue; + } + auto src = operand.getDefiningOp(); + auto res = llvm::cast(operand); + if (src == non_tpu_caller) { + f_operands.push_back(non_tpu_res[res.getResultNumber()]); + } else { + forward_caller->emitOpError() + << "Unknown op source for operand " << operand; + return LogicalResult::failure(); + } + } + return LogicalResult::success(); +} + +LogicalResult MakeCoreTPUOperands(Operation* core_tpu_caller, + Operation* non_tpu_caller, + Operation* forward_caller, + const std::vector& loop_operands, + const std::vector& non_tpu_res, + const std::vector& forward_res, + std::vector& t_operands) { + t_operands.clear(); + t_operands.reserve(core_tpu_caller->getNumOperands()); + for (auto operand : core_tpu_caller->getOperands()) { + if (llvm::isa(operand)) { + // Pull this from the original operands to the original while op. + auto arg = llvm::cast(operand); + t_operands.push_back(loop_operands[arg.getArgNumber()]); + continue; + } + auto src = operand.getDefiningOp(); + auto res = llvm::cast(operand); + if (src == non_tpu_caller) { + t_operands.push_back(non_tpu_res[res.getResultNumber()]); + } else if (src == forward_caller) { + t_operands.push_back(forward_res[res.getResultNumber()]); + } else { + core_tpu_caller->emitOpError() << "Unknown op source for operand " + << operand << ": " << src->getName(); + return LogicalResult::failure(); + } + } + return LogicalResult::success(); +} + +LogicalResult MakeBackwardOperands(Operation* forward_caller, + Operation* core_tpu_caller, + Operation* backward_caller, + const std::vector& loop_operands, + const std::vector& forward_res, + const std::vector& core_tpu_res, + std::vector& b_operands) { + b_operands.clear(); + b_operands.reserve(backward_caller->getNumOperands()); + for (auto operand : backward_caller->getOperands()) { + if (llvm::isa(operand)) { + // Pull this from the original operands to the original while op. + auto arg = llvm::cast(operand); + b_operands.push_back(loop_operands[arg.getArgNumber()]); + continue; + } + auto src = operand.getDefiningOp(); + auto res = llvm::cast(operand); + if (src == forward_caller) { + b_operands.push_back(forward_res[res.getResultNumber()]); + } else if (src == core_tpu_caller) { + b_operands.push_back(core_tpu_res[res.getResultNumber()]); + } else { + // Note: we're expecting no edges from non_tpu() to backward(). + backward_caller->emitOpError() << "Unknown op source for operand " + << operand << ": " << src->getName(); + return LogicalResult::failure(); + } + } + return LogicalResult::success(); +} + +LogicalResult MakeNonTPUOperands(Operation* non_tpu_caller, + const std::vector& loop_operands, + std::vector& n_operands) { + n_operands.clear(); + n_operands.reserve(non_tpu_caller->getNumOperands()); + for (auto operand : non_tpu_caller->getOperands()) { + if (llvm::isa(operand)) { + auto arg = llvm::cast(operand); + n_operands.push_back(loop_operands[arg.getArgNumber()]); + continue; + } + // This shouldn't happen: + auto src = operand.getDefiningOp(); + non_tpu_caller->emitOpError() << "Unknown op source for operand " << operand + << ": " << src->getName(); + return LogicalResult::failure(); + } + return LogicalResult::success(); +} + +Operation* LiftNonTpuFuncCaller(mlir::OpBuilder& builder, + Operation* orig_non_tpu_caller, + const std::vector& operands) { + // Use this to clone an op and lift it outside its parent function. The + // original while body is unchanged. Example: + // Original: + // %x = tf.while(%a, %b) + // ... + // while_body: + // call(f=@sc_fw, %arg0, %arg1) + // Lifted: + // call(f=@sc_fw, %a, %b) + // %x = tf.while(%a, %b) + // ... + func::FuncOp orig_parent_func = + orig_non_tpu_caller->getParentOfType(); + IRMapping ir_map; + ir_map.map(orig_parent_func.getArguments(), operands); + Operation* new_caller = builder.clone(*orig_non_tpu_caller, ir_map); + return new_caller; +} + +void EmbeddingPipeliningPass::runOnOperation() { + LOG(INFO) << "EmbeddingPipeliningPass::runOnOperation()"; + ModuleOp module = getOperation(); + + // We only use one of the EmbeddingPipelining and EmbeddingSequencing passes. + if (!UseEmbeddingPipelining(module)) return; + + SymbolTable symbol_table(module); + + llvm::SetVector forward_pass_ops; + llvm::SetVector backward_pass_ops; + + // Find all ops that we know compose the embedding forward and backward pass. + // These ops are only tagged if one enables the + // `pipeline_execution_with_tensor_core` flag in the mid-level API. + bool sequencing_requested = false; + WalkResult walk_result = module.walk([&](Operation* op) -> WalkResult { + if (op->hasAttr(kEmbeddingPipelining)) { + const std::string region = + op->getAttrOfType(kEmbeddingPipelining).getValue().str(); + if (region == kEmbeddingForward) { + forward_pass_ops.insert(op); + op->removeAttr(kEmbeddingPipelining); + } else if (region == kEmbeddingBackward) { + backward_pass_ops.insert(op); + op->removeAttr(kEmbeddingPipelining); + } else if (region == kEmbeddingForwardSequential || + region == kEmbeddingBackwardSequential) { + sequencing_requested = true; + } else { + return op->emitOpError() + << "embedding op has unknown " << kEmbeddingPipelining + << " attribute value " << region << "."; + } + } + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return signalPassFailure(); + + if (sequencing_requested) { + LOG(INFO) << "EmbeddingSequencingPass requested, skipping " + "EmbeddingPipeliningPass"; + return; + } + + // If there are no forward pass ops, there is no SC, so we end early. + if (forward_pass_ops.empty()) { + if (backward_pass_ops.empty()) { + LOG(INFO) << "no pipelining ops found"; + return; + } else { + (*backward_pass_ops.begin())->emitOpError() + << "embedding backwards pass op with no forwards pass ops."; + return signalPassFailure(); + } + } + + // Ensure that all ops are in the same region, and have the same replication + // info. + // TODO(bfontain): Allow for multiple regions/loops in one module. + // TODO(patn): move this pass after cluster formation to remove the + // complexity with replication info and metadata, cluster checking and + // generalizing to multiple TPU clusters. + Region* region = (*forward_pass_ops.begin())->getParentRegion(); + StringAttr replication_attr = GetReplicationAttr(*forward_pass_ops.begin()); + llvm::SmallVector checkset(forward_pass_ops.getArrayRef()); + checkset.append(backward_pass_ops.begin(), backward_pass_ops.end()); + for (Operation* op : checkset) { + if (op->getParentRegion() != region) { + op->emitOpError() << "embedding ops in two different regions"; + return signalPassFailure(); + } + if (GetReplicationAttr(op) != replication_attr) { + op->emitOpError() << "embedding ops with different replication info " + << replication_attr << " vs " << GetReplicationAttr(op); + return signalPassFailure(); + } + } + + // TODO(bfontain): Check that the region here is the region + // of the loop body func. + // Find the FuncOp for the surrounding while loop body. + auto loop_body_func = + (*forward_pass_ops.begin())->getParentOfType(); + + // merged_set will keep track of which ops are to be avoided when gather ops + // for inclusion into the four extracted functions. + llvm::SetVector merged_set; + + // Find the TPUReplicationMetadata and TPUCompilationResult ops and delete + // them. These will be cloned/inserted into each region. + TF::TPUReplicateMetadataOp metadata_op; + auto result = FindAndExcludeOp(loop_body_func, replication_attr, merged_set, + metadata_op); + if (failed(result)) return signalPassFailure(); + const int num_replicas = metadata_op.getNumReplicas(); + + TF::TPUCompilationResultOp compilation_op; + result = FindAndExcludeOp( + loop_body_func, replication_attr, merged_set, compilation_op); + if (failed(result)) return signalPassFailure(); + + TF::WhileOp orig_while_op = nullptr; + result = FindOwningWhileOp(loop_body_func, module, orig_while_op); + if (failed(result)) return signalPassFailure(); + Location loc = orig_while_op->getLoc(); + + OpBuilder builder(module); + + // A special fix for models that pass resources into helper functions and + // return the same resource (after passing it through multiple identity ops). + // Some subsequent ops use the original resource and others use the returned + // version. Pipelining splits these uses across loop iterations resulting in + // terrible things. + result = EliminateResourceLoops(builder, symbol_table, loop_body_func); + if (failed(result)) return signalPassFailure(); + + result = FindForwardPassOps(builder, forward_pass_ops, backward_pass_ops, + merged_set, loop_body_func, num_replicas); + if (failed(result)) return signalPassFailure(); + merged_set.insert(forward_pass_ops.begin(), forward_pass_ops.end()); + + result = + FindBackwardPassOps(builder, backward_pass_ops, merged_set, num_replicas); + if (failed(result)) return signalPassFailure(); + merged_set.insert(backward_pass_ops.begin(), backward_pass_ops.end()); + + llvm::SetVector core_tpu_ops; + result = FindCoreTPUOps(core_tpu_ops, forward_pass_ops, backward_pass_ops, + merged_set, loop_body_func); + if (failed(result)) return signalPassFailure(); + merged_set.insert(core_tpu_ops.begin(), core_tpu_ops.end()); + + llvm::SetVector non_tpu_ops; + result = FindNonTPUOps(non_tpu_ops, merged_set, loop_body_func); + if (failed(result)) return signalPassFailure(); + merged_set.insert(non_tpu_ops.begin(), non_tpu_ops.end()); + + LOG(INFO) << "Forwards pass " << forward_pass_ops.size() + << " ops, backwards pass " << backward_pass_ops.size() + << " ops, core " << core_tpu_ops.size() + << " ops. Total = " << merged_set.size() << " of " + << GetNumOps(loop_body_func); + + builder.setInsertionPointAfter(*non_tpu_ops.begin()); + TF::StatefulPartitionedCallOp non_tpu_caller = nullptr; + result = + ExtractOpsAsFunc(builder, module, symbol_table, non_tpu_ops, + replication_attr, nullptr, nullptr, loop_body_func, + "non_tpu", &non_tpu_caller, /*flag_for_inlining=*/false); + if (failed(result)) return signalPassFailure(); + + builder.setInsertionPointAfter(non_tpu_caller); + TF::StatefulPartitionedCallOp forward_caller = nullptr; + result = ExtractOpsAsFunc(builder, module, symbol_table, forward_pass_ops, + replication_attr, metadata_op, compilation_op, + loop_body_func, "sc_forward", &forward_caller, + /*flag_for_inlining=*/true); + if (failed(result)) return signalPassFailure(); + + // Create tpu_core function + builder.setInsertionPointAfter(forward_caller); + TF::StatefulPartitionedCallOp core_tpu_caller = nullptr; + result = ExtractOpsAsFunc(builder, module, symbol_table, core_tpu_ops, + replication_attr, metadata_op, compilation_op, + loop_body_func, "core_tpu", &core_tpu_caller, + /*flag_for_inlining=*/true); + if (failed(result)) return signalPassFailure(); + + builder.setInsertionPointAfter(core_tpu_caller); + TF::StatefulPartitionedCallOp backward_caller = nullptr; + result = ExtractOpsAsFunc(builder, module, symbol_table, backward_pass_ops, + replication_attr, metadata_op, compilation_op, + loop_body_func, "sc_backward", &backward_caller, + /*flag_for_inlining=*/true); + if (failed(result)) return signalPassFailure(); + + Callers orig_callers; + orig_callers.forward = forward_caller; + orig_callers.backward = backward_caller; + orig_callers.core_tpu = core_tpu_caller; + orig_callers.non_tpu = non_tpu_caller; + + // The output of the original while op also serves as subsequent input to + // the same function so input_signature == output_signature. Figure out the + // mapping from the result of each of the four functions into the result + // vector. + auto orig_return_op = *loop_body_func.getOps().begin(); + std::map loop_arg_update_map_non_tpu; + std::map loop_arg_update_map_core_tpu; + for (int ret_pos = 0; ret_pos < orig_return_op->getNumOperands(); ++ret_pos) { + auto operand = orig_return_op->getOperand(ret_pos); + auto def_op = operand.getDefiningOp(); + auto result = mlir::dyn_cast(operand); + if (def_op == non_tpu_caller) { + loop_arg_update_map_non_tpu[result.getResultNumber()] = ret_pos; + } else if (def_op == core_tpu_caller) { + loop_arg_update_map_core_tpu[result.getResultNumber()] = ret_pos; + } else if (def_op == forward_caller) { + loop_body_func->emitOpError( + "Unexpected loop carried variable dependency on sc_forward"); + return signalPassFailure(); + } else if (def_op == backward_caller) { + loop_body_func->emitOpError( + "Unexpected loop carried variable dependency on sc_"); + return signalPassFailure(); + } else if (llvm::isa(operand)) { + // pass + } else { + // This should never happen. + loop_body_func->emitOpError("Couldn't find mapping for return value "); + return signalPassFailure(); + } + } + + const int num_f_res = forward_caller->getNumResults(); + const int num_t_res = core_tpu_caller->getNumResults(); + + // At this point, we have separated the main while body ops into four + // functions: + // 1. SC forward pass ("forward_ops") + // 2. TC forward/backward pass ("core_tput_ops") + // 3. SC backward pass ("backward_ops") + // 4. Loop counter updates ("non_tpu_ops") + // + // Next, extract the original conditional function which we'll use to + // kick off the pre-loop pipelining steps. + // are just the operands passed to the original WhileOp. + func::FuncOp orig_cond_func = orig_while_op.cond_function(); + + std::vector loop_operands_0; + const int num_orig_loop_operands = orig_while_op->getNumOperands(); + loop_operands_0.reserve(num_orig_loop_operands); + Append(loop_operands_0, orig_while_op->getOperands()); + + // Evaluate the real conditional function before the new while loop. + builder.setInsertionPoint(orig_while_op); + Operation* cond_caller_0 = + MakeFuncCaller(builder, orig_while_op->getLoc(), orig_cond_func, + loop_operands_0, /*flag_for_inlining=*/false); + Value C_0 = cond_caller_0->getResults().front(); + + // Call the non_tpu function to update the loop counters. This is still + // part of the i=0 loop iteration. + builder.setInsertionPointAfter(cond_caller_0); + Operation* non_tpu_caller_0 = + LiftNonTpuFuncCaller(builder, non_tpu_caller, loop_operands_0); + // Save the results for later reference. + auto non_tpu_res_0 = ResultsAsVector(non_tpu_caller_0); + + // Start step 0. + // Now make the sc_fw + tc_fb call in the pre-loop. We assume (and assert) + // that we'll execute at least two steps. + builder.setInsertionPointAfter(non_tpu_caller_0); + TF::StatefulPartitionedCallOp start_step_0; + result = StartStep0(builder, loc, symbol_table, metadata_op, compilation_op, + C_0, orig_callers, loop_operands_0, start_step_0); + if (failed(result)) return signalPassFailure(); + + // Save the results of the forward_0 and core_tpu_0 calls by slicing them + // out of the results. + auto forward_res_0 = ResultsAsVector(start_step_0, 0, num_f_res); + auto core_tpu_res_0 = ResultsAsVector(start_step_0, num_f_res, num_t_res); + + // Update the loop operands with results of non_tpu() and core_tpu(). + std::vector loop_operands_1 = loop_operands_0; + for (auto p : loop_arg_update_map_non_tpu) + loop_operands_1[p.second] = non_tpu_res_0[p.first]; + for (auto p : loop_arg_update_map_core_tpu) + loop_operands_1[p.second] = core_tpu_res_0[p.first]; + + // The second conditional evaluation. + builder.setInsertionPointAfter(start_step_0); + Operation* cond_caller_1 = + MakeFuncCaller(builder, orig_while_op->getLoc(), orig_cond_func, + loop_operands_1, /*flag_for_inlining=*/false); + Value C_1 = cond_caller_1->getResults().front(); + + builder.setInsertionPointAfter(cond_caller_1); + Operation* non_tpu_caller_1 = + LiftNonTpuFuncCaller(builder, non_tpu_caller, loop_operands_1); + auto non_tpu_res_1 = ResultsAsVector(non_tpu_caller_1); + + // Start step 1. Again, assume. + builder.setInsertionPointAfter(non_tpu_caller_1); + TF::StatefulPartitionedCallOp start_step_1; + result = StartStep1(builder, loc, symbol_table, metadata_op, compilation_op, + C_1, orig_callers, loop_operands_1, start_step_1); + if (failed(result)) return signalPassFailure(); + + // Save the results of the forward_1 call. + auto forward_res_1 = ResultsAsVector(start_step_1); + + // Update the loop operands with any outputs from the non_tpu and core_tpu + // functions. Note, core_tpu isn't called again until the middle of the loop + // body. So, loop_operands_2 is only partially updated here. We'll finish + // updating this after core_tpu() is called in the new while body. + std::vector loop_operands_2 = loop_operands_1; + for (auto p : loop_arg_update_map_non_tpu) + loop_operands_2[p.second] = non_tpu_res_1[p.first]; + + // The second conditional evaluation. The assumption here is that the + // partially updated loop_operands_2 is sufficient for correct evaluation of + // the cond() function. + builder.setInsertionPointAfter(start_step_1); + Operation* cond_caller_2 = + MakeFuncCaller(builder, orig_while_op->getLoc(), orig_cond_func, + loop_operands_2, /*flag_for_inlining=*/false); + Value C_2 = cond_caller_2->getResults().front(); + + // The new while body: + // + // First, we need to construct the body and conditional functions. To do so, + // we need to create the initial operand list that we'll need. This will + // determine the type signature for the body and cond functions. + std::vector tmp_while_operands; + Append(tmp_while_operands, loop_operands_0); + Append(tmp_while_operands, loop_operands_1); + Append(tmp_while_operands, loop_operands_2); + Append(tmp_while_operands, forward_res_0); + Append(tmp_while_operands, forward_res_1); + Append(tmp_while_operands, core_tpu_res_0); + Append(tmp_while_operands, non_tpu_res_1); + Append(tmp_while_operands, {C_0, C_1, C_2}); + + // Dedupe the operands. We'll need a map to help translate. + llvm::SetVector new_while_operands; + llvm::MapVector loop_var_map; + for (auto operand : tmp_while_operands) { + if (new_while_operands.insert(operand)) { + // First time seeing this operand. Let's record the final resting place + // in the new_while_operands vector. + loop_var_map[operand] = new_while_operands.size() - 1; + } + } + // Save index mappings for canonical vectors. + auto BuildUnpackIndexes = + [&loop_var_map](std::vector& prototype_vals) { + std::vector indexes; + indexes.reserve(prototype_vals.size()); + for (auto prototype_val : prototype_vals) + indexes.push_back(loop_var_map[prototype_val]); + return indexes; + }; + auto loop_operands_indexes_im2 = BuildUnpackIndexes(loop_operands_0); + auto loop_operands_indexes_im1 = BuildUnpackIndexes(loop_operands_1); + auto loop_operands_indexes_i = BuildUnpackIndexes(loop_operands_2); + auto forward_res_indexes_im2 = BuildUnpackIndexes(forward_res_0); + auto forward_res_indexes_im1 = BuildUnpackIndexes(forward_res_1); + auto core_tpu_res_indexes_im2 = BuildUnpackIndexes(core_tpu_res_0); + auto non_tpu_res_indexes_im1 = BuildUnpackIndexes(non_tpu_res_1); + int C_index_im2 = loop_var_map[C_0]; + int C_index_im1 = loop_var_map[C_1]; + int C_index_i = loop_var_map[C_2]; + + // Get the operand types. + std::vector new_while_operand_types = GetValueTypes(new_while_operands); + + // Make cond and body functions for the new while op. + // Create the function based on input and result types and values. + // Note, for a while loop body function, the operand types and result types + // are identical. + auto body_func_type = mlir::FunctionType::get( + &getContext(), new_while_operand_types, new_while_operand_types); + auto cond_func_type = mlir::FunctionType::get( + &getContext(), new_while_operand_types, orig_cond_func.getResultTypes()); + func::FuncOp cond = + func::FuncOp::create(loc, "new_while_cond", cond_func_type); + func::FuncOp body = + func::FuncOp::create(loc, "new_while_body", body_func_type); + cond.setPrivate(); + body.setPrivate(); + symbol_table.insert(cond); + symbol_table.insert(body); + OpBuilder cond_builder = OpBuilder::atBlockBegin(cond.addEntryBlock()); + OpBuilder body_builder = OpBuilder::atBlockBegin(body.addEntryBlock()); + + //**************************************************************************** + // Build the internals of the new tf.While op's conditional function. + //**************************************************************************** + // Build the cond function body. All we need is a ReturnOp that returns C_i + // which is the last argument. + func::ReturnOp::create(cond_builder, loc, cond.getArgument(C_index_i)); + + //**************************************************************************** + // Build the internals of the new tf.While op's body function. + //**************************************************************************** + auto body_args = body.getArguments(); + // First, let's unpack all the body arguments. + auto UnpackArgs = [&body_args](std::vector& indexes) { + // This helper makes it easy to unpack "natural" vectors of values while + // still respecting the impact of deduping. + std::vector slice; + int num = indexes.size(); + slice.reserve(num); + for (auto i : indexes) slice.push_back(body_args[i]); + return slice; + }; + auto loop_operands_im2 = UnpackArgs(loop_operands_indexes_im2); + auto loop_operands_im1 = UnpackArgs(loop_operands_indexes_im1); + auto loop_operands_i = UnpackArgs(loop_operands_indexes_i); + auto forward_res_im2 = UnpackArgs(forward_res_indexes_im2); + auto forward_res_im1 = UnpackArgs(forward_res_indexes_im1); + auto core_tpu_res_im2 = UnpackArgs(core_tpu_res_indexes_im2); + auto non_tpu_res_im1 = UnpackArgs(non_tpu_res_indexes_im1); + auto C_im1 = body_args[C_index_im1]; + auto C_i = body_args[C_index_i]; + + // Now, construct the operand least for each op by unpacking values. + + // + // Finish step i-2 + // + // First, add all the inputs to sc_backward(). These all come from the block + // arguments, sc_forward() and core_tpu() and need to be pulled from the + // "i-2" (or "0") version of the inputs. + std::vector b_operands; + result = MakeBackwardOperands(forward_caller, core_tpu_caller, + backward_caller, loop_operands_im2, + forward_res_im2, core_tpu_res_im2, b_operands); + if (failed(result)) return signalPassFailure(); + auto backward_caller_im2 = body_builder.clone(*backward_caller); + backward_caller_im2->setOperands(b_operands); + + // + // Finish step i-1 + // + // Second, add all the inputs to core_tpu(). Thesse all come from the while + // loop opernads, sc_forward() or non_tpu() and need to be pulled from the + // "i-1" (or "1") version of the inputs. + std::vector t_operands; + result = MakeCoreTPUOperands(core_tpu_caller, non_tpu_caller, forward_caller, + loop_operands_im1, non_tpu_res_im1, + forward_res_im1, t_operands); + if (failed(result)) return signalPassFailure(); + auto core_tpu_caller_im1 = body_builder.clone(*core_tpu_caller); + core_tpu_caller_im1->setOperands(t_operands); + auto core_tpu_res_im1 = ResultsAsVector(core_tpu_caller_im1); + + // Update the loop operands with results of core_tpu(). + for (auto p : loop_arg_update_map_core_tpu) + loop_operands_i[p.second] = core_tpu_res_im1[p.first]; + + // + // Start step i + // + // Third, add all the inputs to non_tpu(). These all come from the while + // loop operands and need to be pulled from the "i" (or "2") version of the + // inputs. + std::vector n_operands; + result = MakeNonTPUOperands(non_tpu_caller, loop_operands_i, n_operands); + if (failed(result)) return signalPassFailure(); + auto non_tpu_caller_i = body_builder.clone(*non_tpu_caller); + non_tpu_caller_i->setOperands(n_operands); + auto non_tpu_res_i = ResultsAsVector(non_tpu_caller_i); + + // Fourth, add all the inputs to sc_forward(). These all come from the + // while loop operands or the non_tpu() call that's in the loop body. The + // loop operands need to be pulled from the "i" (or "2") version of the + // inputs. The inputs coming from non_tpu() are from the same loop iteration + // (non_tpu_res_i). + std::vector f_operands; + result = MakeForwardOperands(forward_caller, non_tpu_caller, loop_operands_i, + non_tpu_res_i, f_operands); + if (failed(result)) return signalPassFailure(); + auto forward_caller_i = body_builder.clone(*forward_caller); + forward_caller_i->setOperands(f_operands); + auto forward_res_i = ResultsAsVector(forward_caller_i); + + // Update the loop operands with results of non_tpu(). Results for + // core_tpu() are lagged. + std::vector loop_operands_ip1 = loop_operands_i; + for (auto p : loop_arg_update_map_non_tpu) + loop_operands_ip1[p.second] = non_tpu_res_i[p.first]; + + // Add the conditional evaluation for the next loop iteration. + Operation* cond_caller_ip1 = + MakeFuncCaller(body_builder, orig_while_op->getLoc(), orig_cond_func, + loop_operands_ip1, /*flag_for_inlining=*/false); + Value C_ip1 = cond_caller_ip1->getResults().front(); + + // Build the ReturnOp. This mirrors the construction of the operands with + // 'i' values incremented. + std::vector tmp_body_results; + Append(tmp_body_results, loop_operands_im1); + Append(tmp_body_results, loop_operands_i); + Append(tmp_body_results, loop_operands_ip1); + Append(tmp_body_results, forward_res_im1); + Append(tmp_body_results, forward_res_i); + Append(tmp_body_results, core_tpu_res_im1); + Append(tmp_body_results, non_tpu_res_i); + Append(tmp_body_results, {C_im1, C_i, C_ip1}); + + llvm::SetVector new_body_results; + // This should pack the same as deduping code above. + new_body_results.insert(tmp_body_results.begin(), tmp_body_results.end()); + auto new_body_return_types = GetValueTypes(new_body_results); + + body_builder.setInsertionPointAfter(cond_caller_ip1); + func::ReturnOp::create(body_builder, orig_while_op->getLoc(), + new_body_results.getArrayRef()); + + // Finally, create the new tf.WhileOp. + builder.setInsertionPoint(orig_while_op); + // Use the same parallel_iterations as the original WhileOp unless there's a + // flag override. + int parallel_iterations_flag = tensorflow::GetBuildXlaOpsPassFlags() + ->tf_xla_embedding_parallel_iterations; + int parallel_iterations = parallel_iterations_flag > 0 + ? parallel_iterations_flag + : orig_while_op.getParallelIterations(); + LOG(INFO) << "Setting parallel_iterations_flag to " + << parallel_iterations_flag; + auto new_while_op = TF::WhileOp::create( + builder, orig_while_op->getLoc(), new_body_return_types, + new_while_operands.getArrayRef(), cond.getSymName(), body.getSymName(), + /*parallel_iterations=*/parallel_iterations, + /*is_stateless=*/false, + /*shape_invariant=*/false); + SetBasicBlockAttributes(builder, new_while_op); + + // First, let's unpack all the body arguments. + auto UnpackResults = [&new_while_op](std::vector& indexes) { + int num = indexes.size(); + std::vector slice; + slice.reserve(num); + for (auto i : indexes) slice.push_back(new_while_op->getResult(i)); + return slice; + }; + auto loop_operands_nm2 = UnpackResults(loop_operands_indexes_im2); + auto loop_operands_nm1 = UnpackResults(loop_operands_indexes_im1); + auto loop_operands_n = UnpackResults(loop_operands_indexes_i); + auto forward_res_nm2 = UnpackResults(forward_res_indexes_im2); + auto forward_res_nm1 = UnpackResults(forward_res_indexes_im1); + auto core_tpu_res_nm2 = UnpackResults(core_tpu_res_indexes_im2); + auto non_tpu_res_nm1 = UnpackResults(non_tpu_res_indexes_im1); + auto C_nm2 = new_while_op->getResult(C_index_im2); + auto C_nm1 = new_while_op->getResult(C_index_im1); + + // Finish step n-2. + builder.setInsertionPointAfter(new_while_op); + TF::StatefulPartitionedCallOp finish_step_nm2; + result = FinishStepNm2(builder, loc, symbol_table, metadata_op, + compilation_op, C_nm2, orig_callers, loop_operands_nm2, + forward_res_nm2, core_tpu_res_nm2, finish_step_nm2); + if (failed(result)) return signalPassFailure(); + + // Finish step n-1. + builder.setInsertionPointAfter(finish_step_nm2); + TF::StatefulPartitionedCallOp finish_step_nm1; + result = FinishStepNm1(builder, loc, symbol_table, metadata_op, + compilation_op, C_nm1, orig_callers, loop_operands_nm1, + forward_res_nm1, finish_step_nm1); + if (failed(result)) return signalPassFailure(); + + // Save the results of the core_tpu_0 call and use it to finalize the + // loop_operands_n array. + auto core_tpu_res_nm1 = ResultsAsVector(finish_step_nm1, 0, num_t_res); + for (auto p : loop_arg_update_map_core_tpu) + loop_operands_n[p.second] = core_tpu_res_nm1[p.first]; + + // Replace the return values from the original WhileOp with the output of + // the pipelining. + for (auto p : llvm::zip(orig_while_op->getResults(), loop_operands_n)) + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + *orig_while_op->getParentRegion()); + + // Inline the new while body. + result = Inliner(builder, symbol_table).InlineCallsInFunc(body, false); + if (failed(result)) return signalPassFailure(); + + // Erase original while op and temporary functions. Note, we use the non_tpu + // function in the output graph. + symbol_table.lookup(orig_callers.forward.getF())->erase(); + symbol_table.lookup(orig_callers.core_tpu.getF())->erase(); + symbol_table.lookup(orig_callers.backward.getF())->erase(); + orig_while_op.body_function().erase(); + orig_while_op.erase(); + + LOG(INFO) << "EmbeddingPipeliningPass::runOnOperation done."; +} +} // namespace + +std::unique_ptr> CreateEmbeddingPipeliningPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_program_key.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_program_key.cc new file mode 100644 index 0000000000000000000000000000000000000000..f9b78f7300a6e81a34d974f0699617fd675d4476 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_program_key.cc @@ -0,0 +1,474 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +namespace mlir { +namespace TFDevice { +namespace { +constexpr char kDeviceAttr[] = "device"; +constexpr char kMiniBatchSplitsAttr[] = "mini_batch_splits"; +constexpr char kMiniBatchCsrAttr[] = "mini_batch_in_csr"; + +#define GEN_PASS_DEF_EMBEDDINGPROGRAMKEYPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h.inc" + +struct EmbeddingProgramKeyPass + : public impl::EmbeddingProgramKeyPassBase { + void runOnOperation() override; +}; + +// Checks if `op` is nested in `block`. +bool OpInBlock(Operation* op, Block* block) { + Block* op_block = op->getBlock(); + while (op_block) { + if (op_block == block) return true; + if (auto* parent_op = op_block->getParentOp()) { + op_block = parent_op->getBlock(); + } else { + break; + } + } + return false; +} + +// Find a TPUCompileMlirOp that's after the given `preprocess_op`, under `func`. +// Assumes the TPUCompileMlirOp is wrapped in a tf_device.launch. +Operation* FindCompileSuccessor(Operation* func_op, Operation* preprocess_op) { + bool in_launch = isa(preprocess_op->getParentOp()); + Operation* preprocess_or_launch = + in_launch ? preprocess_op->getParentOp() : preprocess_op; + + Operation* tpu_compile_successor = nullptr; + func_op->walk([&](TF::_TPUCompileMlirOp compile_op) { + if (compile_op->getParentOp() == nullptr || + !isa(compile_op->getParentOp())) + return WalkResult::advance(); + Operation* compile_launch_op = compile_op->getParentOp(); + + if (compile_launch_op->getBlock() == preprocess_or_launch->getBlock() && + preprocess_or_launch->isBeforeInBlock(compile_launch_op)) { + tpu_compile_successor = compile_op; + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + return tpu_compile_successor; +} + +// Get all the Blocks underneath `top` that are "in the scope" of `bottom`, +// i.e., are ancestors. +llvm::DenseSet GetAllBlocksBetween(Operation* bottom, Operation* top) { + llvm::DenseSet blocks; + Operation* op = bottom; + while (op && op != top) { + blocks.insert(op->getBlock()); + op = op->getParentOp(); + } + return blocks; +} + +// For a given value defined somewhere underneath `target_blocks`, get the +// result that's emitted through the op(s) that wrap it, in one of the +// `target_blocks`. +Value GetResultInBlock(Value value, llvm::DenseSet target_blocks) { + while (target_blocks.find(value.getParentBlock()) == target_blocks.end()) { + Value new_value = nullptr; + for (OpOperand& operand : value.getUses()) { + Operation* owner = operand.getOwner(); + if (!llvm::isa(owner)) continue; + Operation* parent = owner->getParentOp(); // op that owns the "return" + if (llvm::isa(parent)) { + new_value = parent->getResult(operand.getOperandNumber()); + } else if (auto replicate = + llvm::dyn_cast(parent)) { + int n = replicate.getN(); + new_value = parent->getResult(operand.getOperandNumber() * n); + } else { + parent->emitOpError("Unsupported wrapper op."); + return nullptr; + } + } + if (!new_value) { + return nullptr; + } else { + value = new_value; + } + } + return value; +} + +// Find a TPUCompileMlirOp that's before the given `preprocess_op`, under +// `func`. Allows both ops to be wrapped inside a tf_device.launch or a +// tf_device.replicate. +Operation* FindCompilePredecessor(Operation* func_op, + Operation* preprocess_op) { + llvm::DenseSet blocks = GetAllBlocksBetween(preprocess_op, func_op); + + llvm::DenseMap scope; + Operation* o = preprocess_op; + while (o && o != func_op) { + scope[o->getBlock()] = o; + o = o->getParentOp(); + } + + Operation* tpu_compile_predecessor = nullptr; + func_op->walk([&](TF::_TPUCompileMlirOp compile_op) { + // Check that the compile op is before the preprocess op. + Operation* o = compile_op; + while (o && o != func_op) { + auto it = scope.find(o->getBlock()); + if (it != scope.end()) { + // Once we have a found a compile op that's in the same + // (parent) block, we need to check that it's before (the + // (parent of) the preprocess_op. + if (o->isBeforeInBlock(it->second)) { + break; // valid compile predecessor + } else { + return WalkResult::advance(); + } + } + o = o->getParentOp(); + } + // Check that the the compile op actually passes its results to its parents. + if (GetResultInBlock(compile_op->getResult(1), blocks)) { + tpu_compile_predecessor = compile_op; + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + return tpu_compile_predecessor; +} + +// Get all of the successor ops of `root_op` in the same block. +llvm::SmallSetVector GetSuccessorOps(Operation* root_op) { + std::queue queue; + llvm::SmallSetVector ops_to_move; + + queue.push(root_op); + while (!queue.empty()) { + Operation* op = queue.front(); + queue.pop(); + if (llvm::isa(op)) continue; + ops_to_move.insert(op); + for (Operation* user : op->getUsers()) { + if (!ops_to_move.contains(user)) { + queue.push(user); + } + } + } + return ops_to_move; +} + +tf_device::LaunchOp CreateLaunchForBlock(OpBuilder* builder, + Operation* before_op, + Block* launch_block, + llvm::StringRef host_device) { + // Find results and result types of ops in block that needs to returned. + llvm::SmallVector launch_results; + llvm::SmallVector launch_result_types; + for (Operation& op : *launch_block) { + for (Value result : op.getResults()) { + bool has_external_uses = false; + for (Operation* user : result.getUsers()) { + if (OpInBlock(user, launch_block)) continue; + has_external_uses = true; + break; + } + if (has_external_uses) { + launch_results.push_back(result); + launch_result_types.push_back(result.getType()); + } + } + } + + builder->setInsertionPointAfter(before_op); + auto launch = tf_device::LaunchOp::create(*builder, before_op->getLoc(), + builder->getStringAttr(host_device), + launch_result_types); + launch.getBody().push_back(launch_block); + + builder->setInsertionPointToEnd(&launch.GetBody()); + tf_device::ReturnOp::create(*builder, before_op->getLoc(), launch_results); + + return launch; +} + +// Creates a new_launch after `before_op` with `ops_to_move` from +// `original_launch_op` on `device`. +void CreateNewLaunchOp(OpBuilder* builder, + tf_device::LaunchOp original_launch_op, + Operation* preprocess_op, + llvm::SmallSetVector& ops_to_move, + llvm::StringRef device, Operation* before_op) { + llvm::SmallDenseMap original_return_operand_map; + for (OpOperand& operand : + original_launch_op.GetBody().getTerminator()->getOpOperands()) { + original_return_operand_map.insert( + {operand.get(), operand.getOperandNumber()}); + } + Block* old_block = preprocess_op->getBlock(); + Block* launch_block = new Block; + for (Operation& op : llvm::make_early_inc_range(old_block->getOperations())) { + if (ops_to_move.contains(&op)) { + op.moveBefore(launch_block, launch_block->end()); + } + } + + // Create the new launch op after TPUCompileOp with the preprocessing ops. + tf_device::LaunchOp moved_launch_op = + CreateLaunchForBlock(builder, before_op, launch_block, device); + + // Replace usages of the original launch op with the new launch op for + // operations that were moved. + for (OpOperand& operand : + moved_launch_op.GetBody().getTerminator()->getOpOperands()) { + auto iter = original_return_operand_map.find(operand.get()); + if (iter != original_return_operand_map.end()) { + original_launch_op.getResult(iter->second) + .replaceAllUsesWith( + moved_launch_op.getResult(operand.getOperandNumber())); + } + } +} + +// Creates a smaller launch op on `device` from `original_launch_op` where some +// ops have been removed from `old_block`. +void CreateReducedLaunchOp(OpBuilder* builder, Block* old_block, + tf_device::LaunchOp original_launch_op, + llvm::StringRef device) { + Block* reduced_launch_block = new Block; + for (Operation& op : llvm::make_early_inc_range(old_block->getOperations())) { + if (!llvm::isa(op)) { + op.moveBefore(reduced_launch_block, reduced_launch_block->end()); + } + } + + tf_device::LaunchOp reduced_launch_op = CreateLaunchForBlock( + builder, original_launch_op, reduced_launch_block, device); + // Replace an still existing usages of original_launch_op with + // reduced_launch_op results. + llvm::SmallDenseMap reduced_return_operand_map; + for (OpOperand& operand : + reduced_launch_op.GetBody().getTerminator()->getOpOperands()) { + reduced_return_operand_map.insert( + {operand.get(), operand.getOperandNumber()}); + } + + for (int i = 0; i < original_launch_op->getNumResults(); i++) { + Value operand = original_launch_op.GetBody().getTerminator()->getOperand(i); + auto defining_op = operand.getDefiningOp(); + if (defining_op != nullptr && + defining_op->getParentOp() == reduced_launch_op.getOperation()) { + auto iter = reduced_return_operand_map.find(operand); + if (iter != reduced_return_operand_map.end()) { + original_launch_op->getResult(i).replaceAllUsesWith( + reduced_launch_op->getResult(iter->second)); + } + } + } + + // Replace the outside usage of ops still in reduced launch. + auto operand_not_in_launch = [&](OpOperand& operand) { + return !reduced_launch_op.getOperation()->isProperAncestor( + operand.getOwner()); + }; + + for (OpOperand& operand : + reduced_launch_op.GetBody().getTerminator()->getOpOperands()) { + operand.get().replaceUsesWithIf( + reduced_launch_op.getResult(operand.getOperandNumber()), + operand_not_in_launch); + } + + // Handle pass through block arguments. + for (OpOperand& operand : + original_launch_op.GetBody().getTerminator()->getOpOperands()) { + if (mlir::isa(operand.get())) { + original_launch_op.getResult(operand.getOperandNumber()) + .replaceAllUsesWith(operand.get()); + } + } +} + +// Move `preprocess_op` after _TPUCompileMlir op if there are no _TPUCompileMlir +// ops before `preprocess_op`. This actually creates a new launch op after +// _TPUCompileMlir and moves `preprocess_op` and its successors that are input +// to TPUExecute to it. +LogicalResult MovePreprocessingOpInLaunch(OpBuilder* builder, + func::FuncOp func_op, + Operation* preprocess_op) { + // If this is already a TPUCompile predecessor, no need to move the + // preprocessing ops. + if (FindCompilePredecessor(func_op, preprocess_op)) return success(); + + auto original_launch_op = + llvm::dyn_cast(preprocess_op->getParentOp()); + // Device of original launch looked up before moving the preprocessing ops + // around. + StringAttr device = + original_launch_op->getAttrOfType(kDeviceAttr); + + if (!device) { + return original_launch_op->emitOpError() + << "Launch op has an invalid device attribute."; + } + + // Find the TPUCompile successor. + Operation* tpu_compile_successor = + FindCompileSuccessor(func_op, preprocess_op); + + // Return early if can't find TPUCompile successor. + if (tpu_compile_successor == nullptr) return success(); + + // Get all of the successor ops of the preprocess_op that are in the same + // block. + llvm::SmallSetVector ops_to_move = + GetSuccessorOps(preprocess_op); + + Block* old_block = preprocess_op->getBlock(); + + // Move the successor ops of the preprocess op to a new launch after the + // successor TPUCompileOp. + CreateNewLaunchOp(builder, original_launch_op, preprocess_op, ops_to_move, + device, tpu_compile_successor->getParentOp()); + // Rewrite the original launch op with smaller set of returns. + CreateReducedLaunchOp(builder, old_block, original_launch_op, device); + + original_launch_op->erase(); + return success(); +} + +LogicalResult MoveStandalonePreprocessingOp(OpBuilder* builder, + func::FuncOp func_op, + Operation* preprocess_op) { + if (FindCompilePredecessor(func_op, preprocess_op)) return success(); + + // Find the TPUCompile successor we want to move upwards. We're moving the + // compile, not the preprocess_op, since it's easier to move (because it + // doesn't typically have any dependencies) + Operation* tpu_compile_successor = + FindCompileSuccessor(func_op, preprocess_op); + if (tpu_compile_successor == nullptr) return success(); + + Operation* compile_launch_op = tpu_compile_successor->getParentOp(); + + // If the launch isn't in the same block as the preprocess op, abort. + if (compile_launch_op->getBlock() != preprocess_op->getBlock()) + return success(); + + // Move the compile op launch right before our op. + compile_launch_op->moveBefore(preprocess_op); + + return success(); +} + +// Rewrites the program_key input of `preprocess_op` to use the output of +// _TPUCompileMlir. +void RewritePreprocessInputs(OpBuilder* builder, func::FuncOp func_op, + Operation* preprocess_op) { + if (preprocess_op->getParentOp() == nullptr) return; + + // Find predecessor TPUCompile Op and rewrite the program key. + Operation* tpu_compile_predecessor = + FindCompilePredecessor(func_op, preprocess_op); + if (tpu_compile_predecessor == nullptr) return; + + Value program = GetResultInBlock(tpu_compile_predecessor->getResult(1), + GetAllBlocksBetween(preprocess_op, func_op)); + if (!program) return; + preprocess_op->setOperand(0, program); +} + +LogicalResult VerifyAllProgramKeyOperandsReplaced(Operation* module) { + WalkResult result = module->walk([&](Operation* op) { + if (!op->hasAttr(kMiniBatchSplitsAttr) && !op->hasAttr(kMiniBatchCsrAttr)) + return WalkResult::advance(); + Operation* defining = op->getOperand(0).getDefiningOp(); + if (llvm::dyn_cast_or_null(defining)) { + op->emitError("Couldn't find a program key to insert into this op."); + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + return result.wasInterrupted() ? failure() : success(); +} + +void EmbeddingProgramKeyPass::runOnOperation() { + // Find all of the relevant post processing ops. + llvm::SmallVector preprocess_ops; + + // Handle ops with mini_batch_splits attribute first since all preprocessing + // ops may need to be moved. + getOperation().walk([&](Operation* op) { + if (op->hasAttr(kMiniBatchSplitsAttr)) preprocess_ops.push_back(op); + }); + + OpBuilder builder(&getContext()); + + for (Operation* op : preprocess_ops) { + if (isa(op->getParentOp())) { + if (failed(MovePreprocessingOpInLaunch(&builder, getOperation(), op))) + return signalPassFailure(); + } else { + if (failed(MoveStandalonePreprocessingOp(&builder, getOperation(), op))) + return signalPassFailure(); + } + RewritePreprocessInputs(&builder, getOperation(), op); + } + + // Handle ops with mini_batch_in_csr attribute. + preprocess_ops.clear(); + getOperation().walk([&](Operation* op) { + if (op->hasAttr(kMiniBatchCsrAttr)) { + preprocess_ops.push_back(op); + } + }); + + for (Operation* preprocess_op : preprocess_ops) { + RewritePreprocessInputs(&builder, getOperation(), preprocess_op); + } + + if (failed(VerifyAllProgramKeyOperandsReplaced(getOperation()))) { + signalPassFailure(); + } +} + +} // anonymous namespace + +std::unique_ptr> CreateEmbeddingProgramKeyPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_sequencing.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_sequencing.cc new file mode 100644 index 0000000000000000000000000000000000000000..411f44e983dd56d976f2846385492a1d049bd47c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/embedding_sequencing.cc @@ -0,0 +1,956 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass separates SparseCore, TensorCore, and non-TPU operations into +// separate functions for proper sequencing of TF2 TPU Embedding (see +// tpu_embedding_v3.py). This pass is a precursor for pipelining (see +// embedding_pipelining.cc) and DOES NOT permit parallel execution across SC and +// TC. This pass is a temporary fallback to use while developing full pipelining +// capabilities. +// +// Ops are broken up into: +// 1. SC forward pass +// 2. TC forward/backward pass +// 3. SC backward pass +// 4. non-TPU loop counter updates + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +#define GEN_PASS_DEF_EMBEDDINGSEQUENCINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h.inc" + +static constexpr char kEmbeddingPipelining[] = "_embedding_pipelining"; +static constexpr char kEmbeddingForward[] = "forward"; +static constexpr char kEmbeddingBackward[] = "backward"; +static constexpr char kEmbeddingForwardSequential[] = "forward_sequential"; +static constexpr char kEmbeddingBackwardSequential[] = "backward_sequential"; +static constexpr char kDevice[] = "device"; +static constexpr llvm::StringRef kTpuCompilationStatus = + "_tpu_compilation_status"; + +namespace mlir { +namespace TFDevice { +namespace { + +struct EmbeddingSequencingPass + : public ::impl::EmbeddingSequencingPassBase { + void getDependentDialects(mlir::DialectRegistry& registry) const override { + registry.insert(); + } + + void runOnOperation() override; +}; + +template +std::vector GetValueTypes(const InputContainer& input) { + // Convert a list of mlir::Value's into a list of mlir::Type's + std::vector types; + types.reserve(input.size()); + for (auto val : input) types.push_back(val.getType()); + return types; +} + +bool IsResourceType(Type val_type) { + if (auto tensor_type = mlir::dyn_cast(val_type)) { + if (mlir::isa(tensor_type.getElementType())) { + return true; + } + } + return false; +} + +bool IsTPUOp(mlir::Operation* op) { + return op->hasAttr(TF::kReplicationInfoAttr); +} + +StringAttr GetReplicationAttr(mlir::Operation* op) { + return op->getAttrOfType(TF::kReplicationInfoAttr); +} + +StringAttr GetReplicationAttr(TF::TPUCompilationResultOp op) { + // Special case for getting the replication region for + // TPUCompilationResultsOp. + return op->getAttrOfType(kTpuCompilationStatus); +} + +int64_t GetNumOps(func::FuncOp func) { + int64_t num_ops = 0; + for (auto it = func.begin(); it != func.end(); ++it) ++num_ops; + return num_ops; +} + +void GatherOpsForExtraction(mlir::SetVector* operations, + const mlir::SetVector& ops_to_avoid, + bool predecessors, bool successors) { + // Walk the input and output dependencies of the Ops in `operations` to form + // the closer of Ops needed to evaluate 'operations'. Input dependencies are + // walked if 'predecessors' is true and output dependencies are walked if + // 'successors' is true. In either case, if a discoverd Op is in the + // 'ops_to_avoid' set, then the dependency walking is terminated. + llvm::SetVector ops_to_process(*operations); + llvm::SetVector new_ops; + + while (!ops_to_process.empty()) { + for (Operation* op : ops_to_process) { + if (predecessors) { + for (Value operand : op->getOperands()) { + // Stop at the block boundary. + if (mlir::isa(operand)) continue; + + Operation* predecessor = operand.getDefiningOp(); + if (!operations->contains(predecessor) && + !ops_to_avoid.contains(predecessor)) { + new_ops.insert(operand.getDefiningOp()); + operations->insert(operand.getDefiningOp()); + } + } + } + if (successors) { + for (mlir::Operation* successor : op->getUsers()) { + // Don't include the return op + if (llvm::isa(successor)) continue; + + if (!operations->contains(successor) && + !ops_to_avoid.contains(successor)) { + new_ops.insert(successor); + operations->insert(successor); + } + } + } + } + ops_to_process.swap(new_ops); + new_ops.clear(); + } +} + +TF::StatefulPartitionedCallOp MakeFuncCaller( + mlir::OpBuilder& builder, const Location& loc, func::FuncOp func, + const llvm::SetVector& operands) { + // Constructs a tf.StatefulPartitionedCall to the function provided in 'func' + // using the operands in 'operands'. Assumes the insertion point on builder is + // already set. + auto symbol = + mlir::SymbolRefAttr::get(builder.getContext(), func.getSymName()); + auto result_types = func.getResultTypes(); + auto caller = TF::StatefulPartitionedCallOp::create( + builder, loc, result_types, operands.getArrayRef(), + /*arg_attrs=*/nullptr, + /*res_attrs=*/nullptr, symbol, + /*config=*/builder.getStringAttr(""), + /*config_proto=*/builder.getStringAttr(""), + /*executor_type=*/builder.getStringAttr("")); + caller.setFAttr(symbol); + return caller; +} + +func::FuncOp CreateFnWithSignature(ModuleOp module, + const llvm::SetVector& inputs, + const llvm::SetVector& outputs, + const std::string& name) { + // Creates an empty func.FuncOp with a signature compatible with 'inputs' + // (operands) and 'outputs' (results). + OpBuilder builder(module); + + std::vector input_types = GetValueTypes(inputs); + std::vector output_types = GetValueTypes(outputs); + builder.setInsertionPointToEnd(&module.getBodyRegion().back()); + func::FuncOp func_op = + func::FuncOp::create(builder, module.getLoc(), name, + builder.getFunctionType(input_types, output_types)); + func_op.setPrivate(); + + return func_op; +} + +TF::StatefulPartitionedCallOp EncapsulateOpsInFunc( + OpBuilder& builder, const llvm::SetVector& ops, + const llvm::SetVector& inputs, const llvm::SetVector& outputs, + func::FuncOp parent_func, ModuleOp module, const std::string& name) { + // Moves all of the Operations in 'ops' into a newly created func.FuncOp + // function named 'name' and replaces the original ops with a call to the + // newly created function using a tf.StatefulPartitionedCall. Here, + // 'parent_func' is the function that holds the original set of ops. + // Note, 'inputs' and 'outputs' are the predetermined set of values that + // should become the operands and return values, respectively. + auto insertion_point = builder.saveInsertionPoint(); + func::FuncOp new_func = CreateFnWithSignature(module, inputs, outputs, + absl::StrCat("_func_", name)); + + // This preserves the order of the ops that was in the original parent + // funtion. This is critical for preserving correctness in the presence of + // resource variables and stateful functions. + std::vector topological_order; + for (Operation& op : parent_func.getOps()) + if (ops.contains(&op)) topological_order.push_back(&op); + + // Create the partitioned call + builder.restoreInsertionPoint(insertion_point); + auto caller = MakeFuncCaller(builder, module.getLoc(), new_func, inputs); + + Block* block = new_func.addEntryBlock(); + + for (Operation* op : topological_order) op->moveBefore(block, block->end()); + + // Replace the 'inputs' values with the new function's arguments. + for (auto p : llvm::zip(inputs, new_func.getArguments())) + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + new_func.getBody()); + + builder.setInsertionPointToEnd(block); + func::ReturnOp::create(builder, parent_func.getLoc(), outputs.getArrayRef()); + + // Replace the original 'outputs' values with the result of the call to the + // new function. + for (auto p : llvm::zip(outputs, caller->getResults())) + replaceAllUsesInRegionWith(std::get<0>(p), std::get<1>(p), + parent_func.getBody()); + + return caller; +} + +void UpdateAndInsertTPUOps(TF::StatefulPartitionedCallOp caller, + TF::TPUReplicateMetadataOp metadata_op, + TF::TPUCompilationResultOp compilation_op, + StringAttr old_group) { + // Adds the TPUReplicateMetatdataOp and TPUCompilationResultOp ops to the + // function called by the provided 'caller'. + mlir::CallInterfaceCallable callable = caller.getCallableForCallee(); + mlir::SymbolRefAttr sym = callable.dyn_cast(); + auto func = llvm::dyn_cast( + mlir::SymbolTable::lookupNearestSymbolFrom(caller, sym)); + OpBuilder builder(func.getBody()); + + StringAttr new_group = builder.getStringAttr( + absl::StrCat(old_group.getValue().str(), caller.getF().str())); + + builder.insert(metadata_op.clone()); + for (Operation& op : func.getOps()) { + if (!IsTPUOp(&op)) continue; + op.setAttr(TF::kReplicationInfoAttr, new_group); + } + TF::TPUCompilationResultOp new_result = compilation_op.clone(); + new_result->setAttr(kTpuCompilationStatus, new_group); + builder.insert(new_result); +} + +template +LogicalResult FindAndExcludeOp(func::FuncOp func, + const StringAttr& replication_attr, + llvm::SetVector& merged_set, + OpType& found_op) { + // Find the TPUReplicationMetadata or TPUCompilationResult ops which will be + // cloned/inserted into each region. We add them to the merged_set so that + // they're ignored when extracting the four main functions. + found_op = nullptr; + for (OpType op : func.getOps()) { + if (found_op != nullptr) { + func.emitOpError() << "number of " << found_op.getOperationName() + << " in loop body is not 1"; + return LogicalResult::failure(); + } + if (GetReplicationAttr(op) != replication_attr) { + op.emitOpError() << "is not part of the replication region " + << replication_attr << " vs " << GetReplicationAttr(op); + return LogicalResult::failure(); + } + found_op = op; + merged_set.insert(found_op); + } + return LogicalResult::success(); +} + +LogicalResult FindOwningWhileOp(func::FuncOp body_func, ModuleOp module, + TF::WhileOp* while_op) { + // Given a while loop body function 'body_func', find the tf.While Op that + // uses it. + auto uses_optional = body_func.getSymbolUses(module); + if (!uses_optional.has_value()) { + body_func.emitOpError() << "no use of while loop body"; + return LogicalResult::failure(); + } + *while_op = nullptr; + for (auto& use : uses_optional.value()) { + if (llvm::isa(use.getUser())) { + if (*while_op != nullptr) { + use.getUser()->emitOpError() << "multiple users of function."; + return LogicalResult::failure(); + } else { + *while_op = llvm::cast(use.getUser()); + } + } else { + use.getUser()->emitOpError() << "non while use of function."; + return LogicalResult::failure(); + } + } + // TODO(bfontain): If the while op is not present we could just split things + // or we wait until the compiler supports multiple regions? + if (while_op == nullptr) { + body_func.emitOpError() << "unable to find while body user."; + return LogicalResult::failure(); + } + return LogicalResult::success(); +} + +LogicalResult FindForwardPassOps(OpBuilder& builder, + llvm::SetVector& forward_pass_ops, + llvm::SetVector& backward_pass_ops, + llvm::SetVector& merged_set, + func::FuncOp loop_body_func, + const int num_replicas) { + // Find all the ops that are to be included in the 'sc_forward' function which + // will be executed on the SparseCore. Note, 'forward_pass_ops' is initially + // seeded with ops from the input MLIR graph that have the + // _embedding_pipelining="forward" attribute which is set by the TF2 Embedding + // API. + // + // When outputs of the forward pass function are used outside of it, we'll + // need to insert a TPUReplicatedOutput Op and include that in the + // forward_pass_ops. And if that usage is also on the TPU (either TensorCore + // or SparseCore) we'll need to insert a matching TPUReplicatedInput. We do + // this before the Ops are removed from the original function/graph so that + // function operands and return values are handled automatically. + + // First, walk the op dependencies. + GatherOpsForExtraction(&forward_pass_ops, merged_set, /*predecessors=*/true, + /*successors=*/false); + + // Locate which variable inputs are part of the forwards pass. These will + // also be used in the backwards pass. We need to create a 'private' copy + // of the TpuReplicatedInput for for the fowards pass if there are users + // outside the pass. Note that in the case of the backwards pass existing + // this will be the case. + // This means that when we have put all out sections together some resource + // inputs will have multiple TPUReplicateInput nodes, so we will need a final + // pass to merge these together into the earliest copy. + llvm::SetVector forward_variable_inputs; + + // Validate that the only resource inputs that are read by ops in + // forward_pass_ops are dataset and variable ops. + int64_t resource_count = 0; + for (auto argument : loop_body_func.getArguments()) { + // Check that all resource arguments are either fed to iterator get next + // or a TPUReplicatedInput with is_packed. + + if (IsResourceType(argument.getType())) { + resource_count++; + bool is_variable = false; + bool is_non_variable = false; + bool use_in_forward = false; + bool use_in_not_forward = false; + for (auto user : argument.getUsers()) { + if (llvm::isa(user)) continue; + if (!forward_pass_ops.contains(user)) { + use_in_not_forward = true; + } else { + use_in_forward = true; + } + if (TF::TPUReplicatedInputOp input = + llvm::dyn_cast(user)) { + if (!input.getIsPacked()) { + input.emitOpError() << "unexpected variable input, not packed"; + return LogicalResult::failure(); + } + + if (is_variable) { + input.emitOpError() << "unexpected multiple TPUReplicatedInputOp " + << "for single argument"; + return LogicalResult::failure(); + } + is_variable = true; + } else { + is_non_variable = true; + } + } + if (use_in_forward && use_in_not_forward) { + loop_body_func.emitOpError() + << "resource input " << argument.getArgNumber() + << " is used both in the forwards and " + << "not forward passes dataset"; + return LogicalResult::failure(); + } + if (is_non_variable && is_variable) { + loop_body_func.emitOpError() + << "resource input " << argument.getArgNumber() + << " is used both as a varible and not a variable"; + return LogicalResult::failure(); + } + if (is_variable && use_in_forward) + forward_variable_inputs.insert(argument.getArgNumber()); + } + } + + VLOG(3) << "Found " << forward_variable_inputs.size() + << " variables used in forward pass of " << resource_count + << " total resource inputs"; + + // Clone the TPUReplicatedInputs. + int64_t cloned_inputs = 0; + for (int64_t index : forward_variable_inputs) { + Value argument = loop_body_func.getArgument(index); + // Uses of this argument should only be the return and the + // TPUReplicateInputOp. This is checked by the loop above. + Operation* input_ptr = nullptr; + for (Operation* user : argument.getUsers()) { + if (llvm::isa(user)) { + input_ptr = user; + break; + } + } + TF::TPUReplicatedInputOp input = + llvm::cast(input_ptr); + + // Validate that all users of the TPUReplicatedInput are ReadVariable + // or AssignVariable ops and check if any are outside the forwards pass. + bool duplicate_needed = false; + for (Operation* next_user : input.getOutput().getUsers()) { + if (!llvm::isa(next_user) && + !llvm::isa(next_user)) { + next_user->emitOpError() + << "unexpected user of output of TPUReplicatedInputOp"; + return LogicalResult::failure(); + } + if (!forward_pass_ops.contains(next_user)) duplicate_needed = true; + } + if (!duplicate_needed) continue; + + cloned_inputs++; + builder.setInsertionPointAfter(input); + forward_pass_ops.remove(input); + + TF::TPUReplicatedInputOp private_input = input.clone(); + builder.insert(private_input); + forward_pass_ops.insert(private_input); + for (OpOperand& next_use : input.getOutput().getUses()) { + if (!forward_pass_ops.contains(next_use.getOwner())) continue; + next_use.getOwner()->setOperand(next_use.getOperandNumber(), + private_input.getOutput()); + } + } + + VLOG(2) << "Cloned " << cloned_inputs << " TPUReplicatedInputOps"; + + // Add TPUReplicatedInput/TPUReplicatedOutput pairs along each edge. + llvm::SetVector new_forward_ops; + for (Operation* op : forward_pass_ops) { + // TODO(bfontain): Should validate that all the TPU ops are in the same + // replication region. + if (!IsTPUOp(op)) continue; + for (Value result : op->getResults()) { + std::vector> out_of_region_use; + for (OpOperand& use : result.getUses()) { + auto use_owner = use.getOwner(); + // TODO(bfontain): Error check here, if the use.getOwner() is not a TPU + // then this op must be a TPUReplicatedOutputOp. + if (IsTPUOp(use_owner) && !forward_pass_ops.contains(use_owner)) + out_of_region_use.push_back( + std::make_pair(use_owner, use.getOperandNumber())); + } + if (out_of_region_use.empty()) continue; + builder.setInsertionPointAfter(op); + std::vector types(num_replicas, result.getType()); + TF::TPUReplicatedOutputOp replicated_output = + TF::TPUReplicatedOutputOp::create(builder, op->getLoc(), + TypeRange(types), result); + new_forward_ops.insert(replicated_output); + // TODO(bfontain): Check for other attributes. + replicated_output->setAttr(kDevice, builder.getStringAttr("")); + TF::TPUReplicatedInputOp input = TF::TPUReplicatedInputOp::create( + builder, op->getLoc(), result.getType(), + replicated_output.getResults()); + input->setAttr(kDevice, builder.getStringAttr("")); + mlir::Value new_value = input.getOutput(); + + if (mlir::isa( + result.getDefiningOp())) { + TF::TPUAnnotateTensorsWithDynamicShapeOp annotate_op = + TF::TPUAnnotateTensorsWithDynamicShapeOp::create( + builder, op->getLoc(), result.getType(), new_value, + result.getDefiningOp()->getAttrs()); + for (auto [operation, index] : out_of_region_use) { + if (!backward_pass_ops.contains(operation)) { + operation->emitOpError() + << "expect all dynamic inputs consumed by backwards pass."; + return LogicalResult::failure(); + } + } + + backward_pass_ops.insert(annotate_op); + new_value = annotate_op->getResult(0); + } + for (auto [operation, index] : out_of_region_use) + operation->setOperand(index, new_value); + } + } + + VLOG(2) << "inserted " << new_forward_ops.size() << " TPU Input/Output ops"; + forward_pass_ops.insert(new_forward_ops.begin(), new_forward_ops.end()); + return LogicalResult::success(); +} + +LogicalResult FindBackwardPassOps( + OpBuilder& builder, llvm::SetVector& backward_pass_ops, + llvm::SetVector& merged_set, const int num_replicas) { + // Find all the ops that are to be included in the 'sc_backward' function + // which will be executed on the SparseCore. Note, 'backward_pass_ops' is + // initially seeded with ops from the input MLIR graph that have the + // _embedding_pipelining="backward" attribute which is set by the TF2 + // Embedding API. + // + // Since we're inserting a replication boundary around the backward pass + // function, we'll also need to make sure TPUReplicatedInputOp and + // TPUReplicatedOutputOp ops are inserted as necessary. + + // First, walk the Ops dependencies. + GatherOpsForExtraction(&backward_pass_ops, merged_set, /*predecessors=*/false, + /*successors=*/true); + + VLOG(3) << "found " << backward_pass_ops.size() << " backwards pass ops"; + + // If any inputs are to the backward_pass_ops region are direct + // TPUReplicatedInput ops, then include (if this is the only use) or + // clone the op. This will be the case for all Read/Assign variable ops. + + llvm::SetVector to_clone; + llvm::SetVector to_insert; + + for (Operation* op : backward_pass_ops) { + for (OpOperand& input_value : op->getOpOperands()) { + Operation* predecessor_op = input_value.get().getDefiningOp(); + if (TF::TPUReplicatedInputOp input = + llvm::dyn_cast(predecessor_op)) { + if (to_clone.contains(input) || to_insert.contains(input)) continue; + // Check if all uses in backwards pass. + bool all_in_backwards = true; + for (Operation* user : input->getUsers()) + if (!backward_pass_ops.contains(user)) all_in_backwards = false; + if (all_in_backwards) + to_insert.insert(input); + else + to_clone.insert(input); + } + } + } + backward_pass_ops.insert(to_insert.begin(), to_insert.end()); + for (TF::TPUReplicatedInputOp input : to_clone) { + builder.setInsertionPointAfter(input); + TF::TPUReplicatedInputOp private_input = input.clone(); + builder.insert(private_input); + backward_pass_ops.insert(private_input); + for (OpOperand& next_use : input.getOutput().getUses()) { + if (!backward_pass_ops.contains(next_use.getOwner())) continue; + next_use.getOwner()->setOperand(next_use.getOperandNumber(), + private_input.getOutput()); + } + } + + VLOG(2) << " cloned " << to_clone.size() << " and inserted " + << to_insert.size() << " TPUReplicatedInput ops"; + + // For all other inputs that go from TPU op to TPU op, insert the + // TPUOutput/Input pair. + + // Add TPUReplicatedInput/TPUReplicatedOutput pairs along each edge. + // TODO(bfontain): Should be merged with the above loop. + llvm::SetVector values_to_add_nodes; + + for (Operation* op : backward_pass_ops) { + // TODO(bfontain): Should validate that all the TPU ops are in the same + // replication region. + // If the op is already a replicated input, no need to to anything. + if (!IsTPUOp(op) || llvm::isa(op)) continue; + for (OpOperand& input_value : op->getOpOperands()) + // TODO(bfontain): Error check here, this line should never be false, + // since we skip the TF::TPUReplicatedInputOp case. + if (IsTPUOp(input_value.get().getDefiningOp()) && + !backward_pass_ops.contains(input_value.get().getDefiningOp())) + values_to_add_nodes.insert(input_value.get()); + } + + for (Value value : values_to_add_nodes) { + builder.setInsertionPointAfter(value.getDefiningOp()); + std::vector types(num_replicas, value.getType()); + Location loc = value.getDefiningOp()->getLoc(); + TF::TPUReplicatedOutputOp output = TF::TPUReplicatedOutputOp::create( + builder, loc, TypeRange(types), value); + // TODO(bfontain): Check for other attributes. + output->setAttr(kDevice, builder.getStringAttr("")); + TF::TPUReplicatedInputOp input = TF::TPUReplicatedInputOp::create( + builder, loc, value.getType(), output.getResults()); + input->setAttr(kDevice, builder.getStringAttr("")); + for (OpOperand& use : value.getUses()) + if (backward_pass_ops.contains(use.getOwner())) + use.getOwner()->setOperand(use.getOperandNumber(), input.getOutput()); + backward_pass_ops.insert(input); + } + + VLOG(2) << " inserted " << values_to_add_nodes.size() + << " TPUReplicatedInput/Output pairs"; + return LogicalResult::success(); +} + +LogicalResult FindCoreTPUOps( + llvm::SetVector& core_tpu_ops, + const llvm::SetVector& forward_pass_ops, + const llvm::SetVector& backward_pass_ops, + const llvm::SetVector& merged_set, + func::FuncOp loop_body_func) { + // Find all of the Ops that are part of the forward/backward pass but aren't + // targeting the SparseCore. Note that we need to include some non-TPU ops + // that flow out of the forward pass function. Otherwise, they would get + // absorbed into the non_tpu function which breaks the pipelining + // decomposition strategy. + // + // Find all the outputs of the forward pass that aren't fed into the backward + // pass. + for (Operation* op : forward_pass_ops) { + for (Value res : op->getResults()) { + for (auto user : res.getUsers()) { + if (!forward_pass_ops.contains(user) && + !backward_pass_ops.contains(user)) { + core_tpu_ops.insert(user); + } + } + } + } + + // Gather all TPU ops marked for compilation in this while loop body that also + // are not in one of the two other sets. + for (Operation& op : loop_body_func.getOps()) { + // Find all TPU ops that don't belong to the forward or backward pass. + if (merged_set.contains(&op) || llvm::isa(op) || + !IsTPUOp(&op) || op.hasAttr(kEmbeddingPipelining)) + continue; + // TODO(bfontain): only collect those ops in a fixed TPUReplica. + core_tpu_ops.insert(&op); + } + + GatherOpsForExtraction(&core_tpu_ops, merged_set, /*predecessors=*/true, + /*successors=*/true); + + // TODO(patn): Verify that all the ops here fall between the forward pass + // and backward pass ops (i.e., not before the forward pass or after the + // backward pass). + return LogicalResult::success(); +} + +LogicalResult FindNonTPUOps(llvm::SetVector& non_tpu_ops, + const llvm::SetVector& merged_set, + func::FuncOp loop_body_func) { + // Find all of the left over Ops after the sc_forward, sc_backward and + // core_tpu ops have been identified. What's left are just the ops necessary + // for updating loop counters etc. + llvm::SetVector non_tpu_args; + for (Operation& op : loop_body_func.getOps()) { + if (merged_set.contains(&op) || llvm::isa(op) || + op.hasAttr(kEmbeddingPipelining)) + continue; + // Note, there should be no TPU ops left at this point. If this trips, + // there's likely a bug in this pass. + if (IsTPUOp(&op)) { + loop_body_func.emitOpError() + << "Unexpcted TPU op found while identifying non-TPU ops."; + return LogicalResult::failure(); + } + non_tpu_ops.insert(&op); + } + + // Validate that remainder_ops takes and returns a subset of the loop carried + // args. This will basically be our set increment fn. + for (Operation* op : non_tpu_ops) + for (Value input : op->getOperands()) + if (BlockArgument arg = llvm::dyn_cast(input)) + // TODO(bfontain): Check that this is actually an argument to the loop + // body. + non_tpu_args.insert(arg.getArgNumber()); + + // All funcs have a return op so this should be safe. + func::ReturnOp return_op = *loop_body_func.getOps().begin(); + + for (OpOperand& operand : return_op->getOpOperands()) { + if (non_tpu_args.contains(operand.getOperandNumber())) { + if (BlockArgument argument = + llvm::dyn_cast(operand.get())) { + if (argument.getArgNumber() != operand.getOperandNumber()) { + return_op.emitOpError() + << "non TPU ops do not divide state into two pieces."; + return LogicalResult::failure(); + } + } else if (!non_tpu_ops.contains(operand.get().getDefiningOp())) { + return_op.emitOpError() + << "non TPU ops do not divide state into two pieces."; + return LogicalResult::failure(); + } + } + } + return LogicalResult::success(); +} + +LogicalResult ExtractOpsAsFunc( + OpBuilder& builder, ModuleOp module, llvm::SetVector& ops, + StringAttr replication_attr, TF::TPUReplicateMetadataOp metadata_op, + TF::TPUCompilationResultOp compilation_op, func::FuncOp parent_func, + const std::string& func_name, Operation** caller) { + // Move the given set of 'ops' into it's own function and replace them with a + // call to that function ('caller'). if 'metadata_op' and 'compilation_op' are + // non-null, also insert those (i.e., target the resulting function to the + // TPU). Here, 'parent_func' is the func.FuncOp that owns the ops in 'ops'. + // + // Returns in 'caller' a tf.StatefulPartitionedCallOp that calls the function + // that was extracted.. + + // Find the input edges to form the set of operands to the new function call. + llvm::SetVector inputs; + for (Operation* op : ops) { + for (Value operand : op->getOperands()) { + Operation* defining_op = operand.getDefiningOp(); + if (!ops.contains(defining_op)) inputs.insert(operand); + } + } + // Find the output edges to form the set of resutls of the new function call. + llvm::SetVector results; + for (Operation* op : ops) { + for (auto result : op->getResults()) { + for (const OpOperand& operand : result.getUsers()) { + if (!ops.contains(operand.getOwner())) { + results.insert(result); + break; + } + } + } + } + llvm::SetVector outputs; + for (auto output : results) outputs.insert(output); + auto tf_caller = EncapsulateOpsInFunc(builder, ops, inputs, outputs, + parent_func, module, func_name); + if (!ops.empty() && metadata_op != nullptr && compilation_op != nullptr) + UpdateAndInsertTPUOps(tf_caller, metadata_op, compilation_op, + replication_attr); + *caller = tf_caller; + return LogicalResult::success(); +} + +void EmbeddingSequencingPass::runOnOperation() { + LOG(INFO) << "EmbeddingSequencingPass::runOnOperation()"; + ModuleOp module = getOperation(); + + llvm::SetVector forward_pass_ops; + llvm::SetVector backward_pass_ops; + + // Find all ops that we know compose the embedding forward and backward pass. + // These ops are only tagged if one enables the + // `pipeline_execution_with_tensor_core` flag in the mid-level API. + WalkResult walk_result = module.walk([&](Operation* op) -> WalkResult { + if (op->hasAttr(kEmbeddingPipelining)) { + const std::string region = + op->getAttrOfType(kEmbeddingPipelining).getValue().str(); + if (region == kEmbeddingForward || + region == kEmbeddingForwardSequential) { + forward_pass_ops.insert(op); + } else if (region == kEmbeddingBackward || + region == kEmbeddingBackwardSequential) { + backward_pass_ops.insert(op); + } else { + return op->emitOpError() + << "embedding op has unknown " << kEmbeddingPipelining + << " attribute value " << region << "."; + } + op->removeAttr(kEmbeddingPipelining); + } + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return signalPassFailure(); + + // If there are no forward pass ops, there is no SC, so we end early. + if (forward_pass_ops.empty()) { + if (backward_pass_ops.empty()) { + LOG(INFO) << "No unprocessed embedding ops found - skipping embedding " + << "sequencing rewrite."; + return; + } else { + (*backward_pass_ops.begin())->emitOpError() + << "embedding backwards pass op with no forwards pass ops."; + return signalPassFailure(); + } + } + LOG(INFO) << "Embedding sequencing rewrite enabled."; + + // Ensure that all ops are in the same region, and have the same replication + // info. + // TODO(bfontain): Allow for multiple regions/loops in one module. + // TODO(patn): move this pass after cluster formation to remove the complexity + // with replication info and metadata, cluster checking and generalizing to + // multiple TPU clusters. + Region* region = (*forward_pass_ops.begin())->getParentRegion(); + StringAttr replication_attr = GetReplicationAttr(*forward_pass_ops.begin()); + llvm::SmallVector checkset(forward_pass_ops.getArrayRef()); + checkset.append(backward_pass_ops.begin(), backward_pass_ops.end()); + for (Operation* op : checkset) { + if (op->getParentRegion() != region) { + op->emitOpError() << "embedding ops in two different regions"; + return signalPassFailure(); + } + if (GetReplicationAttr(op) != replication_attr) { + op->emitOpError() << "embedding ops with different replication info " + << replication_attr << " vs " << GetReplicationAttr(op); + return signalPassFailure(); + } + } + + // TODO(bfontain): Check that the region here is the region + // of the loop body func. + // Find the FuncOp for the surrounding while loop body. + func::FuncOp loop_body_func = + (*forward_pass_ops.begin())->getParentOfType(); + + // merged_set will keep track of which ops are to be avoided when gather ops + // for inclusion into the four extracted functions. + llvm::SetVector merged_set; + + // Find the TPUReplicationMetadata and TPUCompilationResult ops and delete + // them. These will be cloned/inserted into each region. + TF::TPUReplicateMetadataOp metadata_op; + auto result = FindAndExcludeOp(loop_body_func, replication_attr, merged_set, + metadata_op); + if (failed(result)) return signalPassFailure(); + const int num_replicas = metadata_op.getNumReplicas(); + + TF::TPUCompilationResultOp compilation_op; + result = FindAndExcludeOp( + loop_body_func, replication_attr, merged_set, compilation_op); + if (failed(result)) return signalPassFailure(); + + TF::WhileOp while_op = nullptr; + result = FindOwningWhileOp(loop_body_func, module, &while_op); + if (failed(result)) { + LOG(INFO) << "WhileOp not found: assuming external loop."; + } else { + // Override the WhileOp parallel_iterations if requested by flag. + int parallel_iterations_flag = tensorflow::GetBuildXlaOpsPassFlags() + ->tf_xla_embedding_parallel_iterations; + if (parallel_iterations_flag > 0) { + LOG(INFO) << "Setting WhileOp parallel_iterations_flag to " + << parallel_iterations_flag; + while_op.setParallelIterations(parallel_iterations_flag); + } else { + LOG(INFO) << "Using original WhileOp parallel_iteration"; + } + } + + OpBuilder builder(module); + + result = FindForwardPassOps(builder, forward_pass_ops, backward_pass_ops, + merged_set, loop_body_func, num_replicas); + if (failed(result)) return signalPassFailure(); + merged_set.insert(forward_pass_ops.begin(), forward_pass_ops.end()); + + result = + FindBackwardPassOps(builder, backward_pass_ops, merged_set, num_replicas); + if (failed(result)) return signalPassFailure(); + merged_set.insert(backward_pass_ops.begin(), backward_pass_ops.end()); + + llvm::SetVector core_tpu_ops; + result = FindCoreTPUOps(core_tpu_ops, forward_pass_ops, backward_pass_ops, + merged_set, loop_body_func); + if (failed(result)) return signalPassFailure(); + merged_set.insert(core_tpu_ops.begin(), core_tpu_ops.end()); + + llvm::SetVector non_tpu_ops; + result = FindNonTPUOps(non_tpu_ops, merged_set, loop_body_func); + if (failed(result)) return signalPassFailure(); + merged_set.insert(non_tpu_ops.begin(), non_tpu_ops.end()); + + LOG(INFO) << "Forwards pass " << forward_pass_ops.size() + << " ops, backwards pass " << backward_pass_ops.size() + << " ops, core " << core_tpu_ops.size() + << " ops. Total = " << merged_set.size() << " of " + << GetNumOps(loop_body_func) << ".\n"; + + builder.setInsertionPointAfter(*non_tpu_ops.begin()); + Operation* non_tpu_caller = nullptr; + result = + ExtractOpsAsFunc(builder, module, non_tpu_ops, replication_attr, nullptr, + nullptr, loop_body_func, "non_tpu", &non_tpu_caller); + if (failed(result)) return signalPassFailure(); + + builder.setInsertionPointAfter(non_tpu_caller); + Operation* forward_caller = nullptr; + result = ExtractOpsAsFunc(builder, module, forward_pass_ops, replication_attr, + metadata_op, compilation_op, loop_body_func, + "sc_forward", &forward_caller); + if (failed(result)) return signalPassFailure(); + + // Create tpu_core function + builder.setInsertionPointAfter(forward_caller); + Operation* core_tpu_caller = nullptr; + result = ExtractOpsAsFunc(builder, module, core_tpu_ops, replication_attr, + metadata_op, compilation_op, loop_body_func, + "core_tpu", &core_tpu_caller); + if (failed(result)) return signalPassFailure(); + + builder.setInsertionPointAfter(core_tpu_caller); + Operation* backwards_pass_caller = nullptr; + result = ExtractOpsAsFunc( + builder, module, backward_pass_ops, replication_attr, metadata_op, + compilation_op, loop_body_func, "sc_backward", &backwards_pass_caller); + if (failed(result)) return signalPassFailure(); + + metadata_op->erase(); + compilation_op->erase(); + + LOG(INFO) << "EmbeddingSequencingPass::runOnOperation done."; +} + +} // namespace + +std::unique_ptr> CreateEmbeddingSequencingPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..52109c202646ac96a5bec4c1a8385696bc501551 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h @@ -0,0 +1,50 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SPARSECORE_SPARSECORE_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SPARSECORE_SPARSECORE_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace TFDevice { + +// For architectures that support accelerated embedding lookups, this pass will +// rewrite the graph to use pipelining for better device utilization. +std::unique_ptr> CreateEmbeddingSequencingPass(); + +// This is a strictly sequential and formally correct fallback option for the +// embedding pipelining pass intended for debugging during pipelining +// development. +std::unique_ptr> CreateEmbeddingPipeliningPass(); + +// Passes in the program key to embedding ops, by moving the embedding ops +// after the _TPUCompileMlir op. +std::unique_ptr> CreateEmbeddingProgramKeyPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_EMBEDDINGSEQUENCINGPASS +#define GEN_PASS_DECL_EMBEDDINGPIPELININGPASS +#define GEN_PASS_DECL_EMBEDDINGPROGRAMKEYPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h.inc" + +} // namespace TFDevice +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_SPARSECORE_SPARSECORE_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..8bd68dae8c6a572a539453ce3be4c2dfb80391b7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.td @@ -0,0 +1,83 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +def EmbeddingPipeliningPass : Pass<"tf-embedding-pipelining", "mlir::ModuleOp"> { + let summary = "Rewrite graph for embedding pipelining"; + let constructor = "TFDevice::CreateEmbeddingPipeliningPass()"; + let description = [{ + For architectures that support accelerated embedding lookups, this pass will + rewrite the graph to use pipelining for better device utilization. + }]; +} + +def EmbeddingSequencingPass : Pass<"tf-embedding-sequencing", "mlir::ModuleOp"> { + let summary = "Rewrite graph for sequential execution of embeddings"; + let constructor = "TFDevice::CreateEmbeddingSequencingPass()"; + let description = [{ + This is a strictly sequential and formally correct fallback option for the + embedding pipelining pass intended for debugging during pipelining + development. + }]; +} + +def EmbeddingProgramKeyPass : Pass<"tf-embedding-program-key", "mlir::func::FuncOp"> { + let summary = "Sets the program key for embedding ops."; + let constructor = "TFDevice::CreateEmbeddingProgramKeyPass()"; + let description = [{ + Passes in the program key to embedding ops. Will move the embedding ops + after a _TPUCompileMlir op if there is no predecessor _TPUCompileMlir op. + Both the embedding op and compile op are assumed to be wrapped in separate + tf_device.launch() ops. This is because the embedding op is head outside + compiled and the compile op is wrapped in launch to execute on host + during TPURewritePass. + + For example, the tf.OpA with the `mini_batch_splits` attribute will be + moved after _TPUCompileMlir and the first input will use the + _TPUCompileMlir program output: + + ```mlir + "tf_device.launch"() ({ + %cst_0 = "tf.Const"() {value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + "tf.OpA"(%cst_0) { mini_batch_splits = ""} : (tensor<1x!tf_type.string>) -> () + tf_device.return + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> () + %0:2 = "tf_device.launch"() ({ + %compilation_status, %program = "tf._TPUCompileMlir"() { metadata = "...", mlir_module = "..." } : () -> (tensor, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program : tensor, tensor<3x!tf_type.string> + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>) + ``` + + becomes: + + ```mlir + %0:2 = "tf_device.launch"() ({ + %compilation_status, %program = "tf._TPUCompileMlir"() {metadata = "...", mlir_module = "..."} : () -> (tensor, tensor<3x!tf_type.string>) + tf_device.return %compilation_status, %program : tensor, tensor<3x!tf_type.string> + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> (tensor, tensor<3x!tf_type.string>) + "tf_device.launch"() ({ + %cst = "tf.Const"() {value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + "tf.OpA"(%0#1) {mini_batch_splits = ""} : (tensor<3x!tf_type.string>) -> () + tf_device.return + }) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> () + ``` + }]; + + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/stack_ops_decomposition.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/stack_ops_decomposition.cc new file mode 100644 index 0000000000000000000000000000000000000000..f136ea492517762f7a22c079528aae20be720c48 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/stack_ops_decomposition.cc @@ -0,0 +1,576 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/types.h" + +namespace mlir { + +namespace { + +namespace cutil = TF::collection_ops_util; + +#define GEN_PASS_DEF_STACKOPSDECOMPOSITIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct StackOpsDecompositionPass + : public impl::StackOpsDecompositionPassBase { + void runOnOperation() final; +}; + +// Returns the type of the local variable for the stack size. +Type GetSizeVarType(OpBuilder builder) { + auto size_type = cutil::GetSizeType(builder); + return RankedTensorType::get( + {}, TF::ResourceType::get(ArrayRef{size_type}, + builder.getContext())); +} + +// Returns the aliasing argument number of a fucntion return value if it simply +// forwards the argument. Otherwise, returns -1. +int64_t FindAliasedInput(func::FuncOp func, int64_t return_index) { + Value return_val = func.front().getTerminator()->getOperand(return_index); + auto maybe_arg = mlir::dyn_cast(return_val); + if (!maybe_arg) return -1; + return maybe_arg.getArgNumber(); +} + +// Changes the function signature that has stacks in the arguments. A stack +// argument will be turned into a variable type if arg_to_stack_type returns +// such a type, and a new argument will be added to the end of the argument +// list for the size variable. +// +// If stack_var_to_size_var is not nullptr, it will be used to store the +// mapping from the stack-variable argument to the size-variable argument. +// +// If handle_new_size_vars is provided, it will be invoked on the list of new +// size variables before finally changing the function type. +void ModifyFunctionSignature( + func::FuncOp func, llvm::SmallDenseMap* stack_var_to_size_var, + llvm::function_ref(int64_t)> arg_to_stack_type, + llvm::function_ref)> handle_new_size_vars = + nullptr) { + auto new_input_types = llvm::to_vector<8>(func.getFunctionType().getInputs()); + auto size_var_type = GetSizeVarType(OpBuilder(func)); + int64_t original_arg_count = new_input_types.size(); + for (int64_t i = 0; i < original_arg_count; ++i) { + auto stack_type = arg_to_stack_type(i); + if (!stack_type.has_value()) continue; + func.getArgument(i).setType(*stack_type); + new_input_types[i] = *stack_type; + auto size_arg = func.front().addArgument(size_var_type, func.getLoc()); + new_input_types.push_back(size_arg.getType()); + if (stack_var_to_size_var) { + (*stack_var_to_size_var)[func.getArgument(i)] = size_arg; + } + } + if (handle_new_size_vars) { + handle_new_size_vars(func.getArguments().drop_front(original_arg_count)); + } + func.setType( + FunctionType::get(func.getContext(), new_input_types, + func.front().getTerminator()->getOperandTypes())); +} + +// Contains cached information for decomposed callee functions for (stateful) +// partitioned call ops. +struct PartitionedCallStackOpsInfo { + bool signature_change; + func::FuncOp decomposed_callee; + llvm::SmallDenseMap stack_var_arg_to_size_arg; +}; + +LogicalResult DecomposeStackOpsInternal( + Block*, ModuleOp, llvm::SmallDenseMap*, + llvm::StringMap*); + +// Handles stack usage by a tf.While. It will convert the body and conditional +// function signatures, and performs stack ops decomposition on them. +LogicalResult HandleWhileOp( + TF::WhileOp while_op, ModuleOp module, + const llvm::SmallDenseMap& data_var_to_size_var, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto body = while_op.body_function(); + llvm::SmallDenseMap body_map; + auto find_arg_stack_type = [&](int64_t index) -> std::optional { + auto it = data_var_to_size_var.find(while_op.getOperand(index)); + if (it == data_var_to_size_var.end()) return std::nullopt; + return it->getFirst().getType(); + }; + auto add_size_vars_to_return = [&](ArrayRef new_args) { + if (new_args.empty()) return; + auto body_ret = body.front().getTerminator(); + auto new_body_returns = llvm::to_vector<8>(body_ret->getOperands()); + for (auto arg : new_args) new_body_returns.push_back(arg); + OpBuilder builder(body_ret); + func::ReturnOp::create(builder, body_ret->getLoc(), new_body_returns); + body_ret->erase(); + }; + // Handle body. + ModifyFunctionSignature(body, &body_map, find_arg_stack_type, + add_size_vars_to_return); + const bool signature_change = !body_map.empty(); + if (failed(DecomposeStackOpsInternal(&body.front(), module, &body_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + // Cond should not change stacks in the arguments, so use an empty map. + auto cond = while_op.cond_function(); + ModifyFunctionSignature(cond, nullptr, find_arg_stack_type); + llvm::SmallDenseMap empty_map; + if (failed(DecomposeStackOpsInternal(&cond.front(), module, &empty_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (!signature_change) return success(); + // Create the new while op. + auto new_while_operands = llvm::to_vector<8>(while_op.getOperands()); + OpBuilder builder(while_op); + assert(while_op.getNumOperands() == while_op.getNumResults()); + for (int64_t i = 0; i < while_op.getNumResults(); ++i) { + auto it = data_var_to_size_var.find(while_op.getOperand(i)); + if (it == data_var_to_size_var.end()) continue; + new_while_operands.push_back(it->getSecond()); + } + auto new_while = TF::WhileOp::create( + builder, while_op.getLoc(), body.getFunctionType().getInputs(), + new_while_operands, while_op->getAttrs()); + for (int64_t i = 0; i < while_op.getNumResults(); ++i) { + if (!mlir::isa( + getElementTypeOrSelf(while_op.getOperand(i).getType()))) { + continue; + } + int64_t aliased_input = FindAliasedInput(body, i); + if (aliased_input == i) { + // Replace aliased stack output uses with input. + while_op.getResult(i).replaceAllUsesWith(while_op.getOperand(i)); + } + } + while_op.replaceAllUsesWith( + new_while.getResults().take_front(while_op.getNumResults())); + while_op.erase(); + return success(); +} + +// Handles stack usage by a tf.If. It will convert the branch function +// signatures, and performs stack ops decomposition on them. +LogicalResult HandleIfOp( + TF::IfOp if_op, ModuleOp module, + const llvm::SmallDenseMap& data_var_to_size_var, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto then_func = if_op.then_function(); + auto else_func = if_op.else_function(); + llvm::SmallDenseMap then_map; + llvm::SmallDenseMap else_map; + + auto find_arg_stack_type = [&](int64_t index) -> std::optional { + auto it = data_var_to_size_var.find(if_op.getOperand(index + 1)); + if (it == data_var_to_size_var.end()) return std::nullopt; + return it->getFirst().getType(); + }; + ModifyFunctionSignature(then_func, &then_map, find_arg_stack_type); + ModifyFunctionSignature(else_func, &else_map, find_arg_stack_type); + const bool signature_change = !then_map.empty() || !else_map.empty(); + if (failed(DecomposeStackOpsInternal(&then_func.front(), module, &then_map, + decomposed_partitioned_call_callees)) || + failed(DecomposeStackOpsInternal(&else_func.front(), module, &else_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (!signature_change) return success(); + auto new_if_operands = llvm::to_vector<8>(if_op.getOperands()); + for (auto operand : if_op.getOperands()) { + auto it = data_var_to_size_var.find(operand); + if (it == data_var_to_size_var.end()) continue; + new_if_operands.push_back(it->getSecond()); + } + OpBuilder builder(if_op); + auto new_if = TF::IfOp::create(builder, if_op.getLoc(), + then_func.getFunctionType().getResults(), + new_if_operands, if_op->getAttrs()); + for (auto result : if_op.getResults()) { + if (!mlir::isa(getElementTypeOrSelf(result.getType()))) { + continue; + } + int64_t then_aliased_input = + FindAliasedInput(then_func, result.getResultNumber()); + int64_t else_aliased_input = + FindAliasedInput(else_func, result.getResultNumber()); + if (then_aliased_input >= 0 && then_aliased_input == else_aliased_input) { + // Replace aliased stack output uses with input. + result.replaceAllUsesWith(if_op.getOperand(then_aliased_input + 1)); + } + } + if_op.replaceAllUsesWith(new_if); + if_op.erase(); + return success(); +} + +// Handles stack usage by a tf.StatefulPartitionedCall or a tf.PartitionedCall. +// It will first check if the callee was previously handled, and try to reuse +// that result if so. Otherwise, it will clone and convert the callee function, +// and performs stack ops decomposition on it. +template +LogicalResult HandlePartitionedCallOp( + CallOp call, func::FuncOp callee, ModuleOp module, + const llvm::SmallDenseMap& data_var_to_size_var, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto emplace_res = decomposed_partitioned_call_callees->try_emplace( + callee.getName(), PartitionedCallStackOpsInfo()); + auto& info = emplace_res.first->second; + // Recreate the call op with info. + auto recreate_caller = [&] { + auto new_operands = llvm::to_vector<8>(call.getOperands()); + for (int64_t i = 0; i < call.getNumOperands(); ++i) { + auto arg_it = info.stack_var_arg_to_size_arg.find(i); + if (arg_it == info.stack_var_arg_to_size_arg.end()) continue; + auto it = data_var_to_size_var.find(call.getOperand(i)); + if (it == data_var_to_size_var.end()) { + call.emitOpError("unknown stack"); + return failure(); + } + assert(arg_it->second == new_operands.size()); + new_operands.push_back(it->getSecond()); + } + OpBuilder builder(call); + auto new_call = + CallOp::create(builder, call.getLoc(), + info.decomposed_callee.getFunctionType().getResults(), + new_operands, call->getAttrs()); + new_call->setAttr( + "f", SymbolRefAttr::get( + builder.getContext(), + const_cast(info.decomposed_callee).getName())); + for (int64_t i = 0; i < call.getNumResults(); ++i) { + auto result = call.getResult(i); + if (!mlir::isa( + getElementTypeOrSelf(result.getType()))) { + continue; + } + int64_t aliased_input = FindAliasedInput(info.decomposed_callee, i); + if (aliased_input >= 0) { + // Replace aliased stack output uses with input. + result.replaceAllUsesWith(call.getOperand(aliased_input)); + } + } + call.replaceAllUsesWith(new_call); + call.erase(); + return success(); + }; + if (!emplace_res.second) { + // This callee was handled before. + if (!info.signature_change) return success(); + return recreate_caller(); + } + llvm::SmallDenseMap callee_map; + func::FuncOp lowered_callee = callee; + if (!callee.isPrivate()) { + // Clone non-private callee in case of signature change. + lowered_callee = callee.clone(); + lowered_callee.setPrivate(); + } + auto find_arg_stack_type = [&](int64_t index) -> std::optional { + auto it = data_var_to_size_var.find(call.getOperand(index)); + if (it == data_var_to_size_var.end()) return std::nullopt; + return it->getFirst().getType(); + }; + ModifyFunctionSignature(lowered_callee, &callee_map, find_arg_stack_type); + info.signature_change = !callee_map.empty(); + if (!info.signature_change) { + // Signature is not modified. We do not need the clone. + if (lowered_callee != callee) { + lowered_callee.erase(); + } + } else { + info.decomposed_callee = lowered_callee; + for (auto& entry : callee_map) { + info.stack_var_arg_to_size_arg[mlir::cast(entry.getFirst()) + .getArgNumber()] = + mlir::cast(entry.getSecond()).getArgNumber(); + } + if (lowered_callee != callee) { + // Add the clone with a new name. + lowered_callee.setName(StringAttr::get( + callee->getContext(), + llvm::formatv("{0}_stack_decomposed", callee.getName()).str())); + SymbolTable(module).insert(lowered_callee); + callee = lowered_callee; + } + } + if (failed(DecomposeStackOpsInternal(&callee.front(), module, &callee_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (info.signature_change) return recreate_caller(); + return success(); +} + +LogicalResult HandleStackV2Op( + TF::StackV2Op stack, ModuleOp module, + llvm::SmallDenseMap* data_var_to_size_var) { + // Create a buffer variable and a size variable to replace the stack. + auto elem_type = cutil::GetElementTypeFromAccess( + stack.getHandle(), module, [](Operation* user) -> std::optional { + auto push = llvm::dyn_cast(user); + if (!push) return std::nullopt; + return push.getElem().getType(); + }); + if (!elem_type.has_value()) { + return stack.emitOpError("cannot infer element shape of stack"); + } + OpBuilder builder(stack); + Value buffer; + if (failed(cutil::CreateInitBufferValue( + elem_type->getShape(), stack.getMaxSize(), stack, + elem_type->getElementType(), builder, &buffer))) { + return failure(); + } + auto size_var_type = GetSizeVarType(builder); + auto var_type = RankedTensorType::get( + {}, TF::ResourceType::get( + ArrayRef{mlir::cast(buffer.getType())}, + stack.getContext())); + auto local_var = TF::MlirLocalVarOp::create( + builder, stack.getLoc(), ArrayRef{var_type}, ArrayRef{}); + auto local_size_var = TF::MlirLocalVarOp::create( + builder, stack.getLoc(), ArrayRef{size_var_type}, + ArrayRef{}); + // Zero-initialize the local vars. + cutil::WriteLocalVariable(local_size_var, + cutil::GetR1Const({0LL}, builder, stack.getLoc()), + builder, stack.getLoc()); + cutil::WriteLocalVariable(local_var, buffer, builder, stack.getLoc()); + stack.getHandle().replaceAllUsesWith(local_var); + (*data_var_to_size_var)[local_var] = local_size_var; + stack.erase(); + return success(); +} + +LogicalResult HandleStackPushV2Op( + TF::StackPushV2Op push, + llvm::SmallDenseMap* data_var_to_size_var) { + auto it = data_var_to_size_var->find(push.getHandle()); + if (it == data_var_to_size_var->end()) { + return push.emitOpError("unknown stack"); + } + // Push output simply forward the input element. + push.replaceAllUsesWith(push.getElem()); + OpBuilder builder(push); + // Read the current buffer and size. + auto stack_val = + cutil::ReadLocalVariable(push.getHandle(), builder, push.getLoc()); + auto index = + cutil::ReadLocalVariable(it->getSecond(), builder, push.getLoc()); + stack_val = cutil::SetElement(index, stack_val, push.getElem(), builder, + push.getLoc()); + // Assign the new buffer and size. + cutil::WriteLocalVariable(push.getHandle(), stack_val, builder, + push.getLoc()); + index = TF::AddV2Op::create( + builder, push.getLoc(), ArrayRef{index.getType()}, + ArrayRef{index, cutil::GetR1Const({1}, builder, push.getLoc())}); + cutil::WriteLocalVariable(it->getSecond(), index, builder, push.getLoc()); + push.erase(); + return success(); +} + +LogicalResult HandleStackPopV2Op( + TF::StackPopV2Op pop, + llvm::SmallDenseMap* data_var_to_size_var) { + auto it = data_var_to_size_var->find(pop.getHandle()); + if (it == data_var_to_size_var->end()) { + return pop.emitOpError("unknown stack"); + } + OpBuilder builder(pop); + // Read the current buffer and size. + auto stack_val = + cutil::ReadLocalVariable(pop.getHandle(), builder, pop.getLoc()); + auto size = cutil::ReadLocalVariable(it->getSecond(), builder, pop.getLoc()); + auto new_size = TF::SubOp::create( + builder, pop.getLoc(), ArrayRef{size.getType()}, + ArrayRef{size, cutil::GetR1Const({1}, builder, pop.getLoc())}); + auto pop_val = cutil::GetElement(new_size, stack_val, builder, pop.getLoc()); + pop.replaceAllUsesWith(pop_val); + // Update the size. + cutil::WriteLocalVariable(it->getSecond(), new_size, builder, pop.getLoc()); + pop.erase(); + return success(); +} + +LogicalResult HandleRegionControlFlowOps( + Operation& op, ModuleOp module, + llvm::SmallDenseMap* data_var_to_size_var, + llvm::StringMap* + decomposed_partitioned_call_callees) { + for (OpOperand& operand : op.getOpOperands()) { + if (mlir::isa( + getElementTypeOrSelf(operand.get().getType()))) { + return op.emitOpError() + << "found unexpected type " << operand.get().getType() + << " of operand #" << operand.getOperandNumber() + << ", resource type operands are expected to have been " + "canonicalized away for region based control flow ops"; + } + } + for (OpResult result : op.getResults()) { + if (mlir::isa(getElementTypeOrSelf(result.getType()))) { + return op.emitOpError() + << "found unexpected type " << result.getType() << " of result #" + << result.getResultNumber() + << ", resource type results are expected to have been " + "canonicalized away for region based control flow ops"; + } + } + for (Region& region : op.getRegions()) { + if (failed(DecomposeStackOpsInternal(®ion.front(), module, + data_var_to_size_var, + decomposed_partitioned_call_callees))) + return failure(); + } + return success(); +} + +// Decomposes stack ops on a region and recursively decomposes called functions. +// data_var_to_size_var: a mapping from stacks' buffer local variables to size +// local variables. +// decomposed_partitioned_call_callees: cache for partitioned call ops' callee +// function handling. +LogicalResult DecomposeStackOpsInternal( + Block* block, ModuleOp module, + llvm::SmallDenseMap* data_var_to_size_var, + llvm::StringMap* + decomposed_partitioned_call_callees) { + for (auto& op : llvm::make_early_inc_range(block->getOperations())) { + if (llvm::isa(&op)) { + // Removes identity nodes in the block. The device computation does not + // need such nodes to carry information. + op.replaceAllUsesWith(op.getOperands()); + op.erase(); + } else if (auto stack = llvm::dyn_cast(&op)) { + if (failed(HandleStackV2Op(stack, module, data_var_to_size_var))) { + return failure(); + } + } else if (auto push = llvm::dyn_cast(&op)) { + if (failed(HandleStackPushV2Op(push, data_var_to_size_var))) { + return failure(); + } + } else if (auto pop = llvm::dyn_cast(&op)) { + if (failed(HandleStackPopV2Op(pop, data_var_to_size_var))) { + return failure(); + } + } else if (auto close = llvm::dyn_cast(&op)) { + data_var_to_size_var->erase(close.getHandle()); + close.erase(); + } else if (auto while_op = llvm::dyn_cast(&op)) { + if (failed(HandleWhileOp(while_op, module, *data_var_to_size_var, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto if_op = llvm::dyn_cast(&op)) { + if (failed(HandleIfOp(if_op, module, *data_var_to_size_var, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (llvm::isa(op) || + llvm::isa(op) || + llvm::isa(op)) { + if (failed( + HandleRegionControlFlowOps(op, module, data_var_to_size_var, + decomposed_partitioned_call_callees))) + return failure(); + } else if (auto pcall = llvm::dyn_cast(&op)) { + if (!pcall.func()) { + return pcall.emitOpError( + "stack decomposition does not support call with nested references"); + } + if (failed(HandlePartitionedCallOp( + pcall, pcall.func(), module, *data_var_to_size_var, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto spcall = + llvm::dyn_cast(&op)) { + if (failed(HandlePartitionedCallOp( + spcall, spcall.func(), module, *data_var_to_size_var, + decomposed_partitioned_call_callees))) { + return failure(); + } + } + } + return success(); +} + +LogicalResult DecomposeStackOps(Block* block, ModuleOp module) { + llvm::SmallDenseMap data_var_to_size_var; + llvm::StringMap + decomposed_partitioned_call_callees; + return DecomposeStackOpsInternal(block, module, &data_var_to_size_var, + &decomposed_partitioned_call_callees); +} + +void StackOpsDecompositionPass::runOnOperation() { + auto module = getOperation(); + auto main = module.lookupSymbol("main"); + if (!main) return; + if (failed(DecomposeStackOps(&main.front(), module))) { + signalPassFailure(); + } +} + +} // namespace + +namespace TF { +std::unique_ptr> CreateStackOpsDecompositionPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_noinline_attribute.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_noinline_attribute.cc new file mode 100644 index 0000000000000000000000000000000000000000..add8ce4d668815c74ef6a7ae1b964f1cd669d2c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_noinline_attribute.cc @@ -0,0 +1,47 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_STRIPNOINLINEATTRIBUTEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// This tranformation pass strips any "_noinline" attributes from the module. +struct StripNoinlineAttributePass + : public impl::StripNoinlineAttributePassBase { + public: + // void runOnOperation() override; + void runOnOperation() override { + // Strip the "tf._noinline" attribute from top-level functions. + for (auto func_op : getOperation().getOps()) + func_op->removeAttr("tf._noinline"); + } +}; + +} // namespace + +std::unique_ptr> CreateStripNoinlineAttributePass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_saved_module_metadata.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_saved_module_metadata.cc new file mode 100644 index 0000000000000000000000000000000000000000..446646630daf579a807dc78426104167b9096f7d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_saved_module_metadata.cc @@ -0,0 +1,100 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +#define GEN_PASS_DEF_STRIPSAVEDMODULEMETADATAPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +struct StripSavedModuleMetadataPass + : public impl::StripSavedModuleMetadataPassBase< + StripSavedModuleMetadataPass> { + void runOnOperation() override; +}; + +bool ShouldStripAttr(NamedAttribute &namedAttr) { + auto name = namedAttr.getName().strref(); + return name.starts_with("tf_saved_model."); +} + +void StripModule(Operation *module) { + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + module->getAttrs(), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + module->removeAttr(namedAttr.getName()); + } +} + +void StripFunction(func::FuncOp func) { + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + func->getAttrs(), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func->removeAttr(namedAttr.getName()); + } + + for (int i = 0; i < func.getNumArguments(); ++i) { + llvm::ArrayRef attrs = + mlir::function_interface_impl::getArgAttrs(func, i); + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + attrs, + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func.removeArgAttr(i, namedAttr.getName()); + } + } + + for (int i = 0; i < func.getNumResults(); ++i) { + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + func.getResultAttrs(i), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func.removeResultAttr(i, namedAttr.getName()); + } + } +} + +void StripSavedModuleMetadataPass::runOnOperation() { + auto module = getOperation(); + StripModule(module); + module.walk(StripFunction); +} + +} // namespace + +std::unique_ptr> CreateStripSavedModuleMetadataPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_tf_attributes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_tf_attributes.cc new file mode 100644 index 0000000000000000000000000000000000000000..b0fc2662648822235bcdfaeb40244d1f34b735c9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/strip_tf_attributes.cc @@ -0,0 +1,95 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_STRIPTFATTRIBUTESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct StripTfAttributesPass + : public impl::StripTfAttributesPassBase { + void runOnOperation() override; +}; + +bool ShouldStripAttr(NamedAttribute &namedAttr) { + StringRef name = namedAttr.getName().strref(); + if (name.starts_with("tf.") || name.starts_with("tf_")) return true; + StringRef value = namedAttr.getValue().getDialect().getNamespace(); + return value == "tf" || value.starts_with("tf_"); +} + +void StripFunction(func::FuncOp func) { + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + func->getAttrs(), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func->removeAttr(namedAttr.getName()); + } + + for (int i = 0; i < func.getNumArguments(); ++i) { + llvm::ArrayRef attrs = + mlir::function_interface_impl::getArgAttrs(func, i); + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + attrs, + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func.removeArgAttr(i, namedAttr.getName()); + } + } + + for (int i = 0; i < func.getNumResults(); ++i) { + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + func.getResultAttrs(i), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + func.removeResultAttr(i, namedAttr.getName()); + } + } +} + +void StripTfAttributesPass::runOnOperation() { + ModuleOp module = getOperation(); + + // strip module itself + auto stripAttrs = llvm::to_vector<4>(llvm::make_filter_range( + module->getAttrs(), + [](NamedAttribute namedAttr) { return ShouldStripAttr(namedAttr); })); + for (auto namedAttr : stripAttrs) { + module->removeAttr(namedAttr.getName()); + } + + // strip functions in module + module.walk([&](func::FuncOp func) { StripFunction(func); }); +} + +} // namespace + +std::unique_ptr> CreateStripTfAttributesPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_array_ops_decomposition.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_array_ops_decomposition.cc new file mode 100644 index 0000000000000000000000000000000000000000..0463b90c3cb87315da28a410836cea7d2fd72dd3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_array_ops_decomposition.cc @@ -0,0 +1,968 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/types.h" + +namespace mlir { +namespace { + +namespace cutil = TF::collection_ops_util; + +using std::string; + +#define GEN_PASS_DEF_TENSORARRAYOPSDECOMPOSITIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass that converts tensor array operations to tensor operations and +// read/assign ops on local variables. A later resource lifting pass can further +// remove the local variables. +// +// This pass requires that the full shape of the tensor array can be inferred: +// 1) the size needs to be a constant, 2) it specifies the full element shape, +// or that can be inferred from a later write, and 3) all elements have the same +// shape. +// +struct TensorArrayOpsDecompositionPass + : public impl::TensorArrayOpsDecompositionPassBase< + TensorArrayOpsDecompositionPass> { + void runOnOperation() override; +}; + +// Infers the element type and count for a TensorArraySplitV3Op. Requires +// constant lengths and static shape on the input value. +LogicalResult GetSplitElementTypeAndCount(TF::TensorArraySplitV3Op split, + RankedTensorType* elem_type, + int64_t* count) { + auto lengths_const = + llvm::dyn_cast_or_null(split.getLengths().getDefiningOp()); + if (!lengths_const) return split.emitOpError("non-constant split lengths"); + *count = lengths_const.getValue().getNumElements(); + if (*count <= 0) return split.emitOpError("non-positive split count"); + auto buffer_type = + llvm::dyn_cast(split.getValue().getType()); + if (!buffer_type || !buffer_type.hasStaticShape() || + buffer_type.getRank() < 1) { + return split.emitOpError("unknown or invalid split tensor shape"); + } + int64_t length = buffer_type.getDimSize(0) / *count; + for (const auto& len : lengths_const.getValue().getValues()) { + if (length == len.getSExtValue()) continue; + return split.emitOpError("different split lengths are not supported"); + } + llvm::SmallVector elem_shape; + elem_shape.push_back(length); + for (int64_t dim : buffer_type.getShape().drop_front()) { + elem_shape.push_back(dim); + } + *elem_type = RankedTensorType::get(elem_shape, buffer_type.getElementType()); + return success(); +} + +// Tries to infer the tensor array element shape. +std::optional> GetTensorArrayElementShape( + TF::TensorArrayV3Op ta, ModuleOp module) { + auto element_shape = llvm::cast(ta.getElementShapeAttr()); + if (element_shape.hasStaticShape()) { + auto shape = element_shape.getShape(); + // Convert int64 to int64_t. + llvm::SmallVector dims(shape.begin(), shape.end()); + return dims; + } + + bool has_failure = false; + auto elem_type = cutil::GetElementTypeFromAccess( + ta.getHandle(), module, [&](Operation* user) -> std::optional { + if (has_failure) return std::nullopt; + if (auto write = llvm::dyn_cast(user)) { + return write.getValue().getType(); + } else if (auto split = + llvm::dyn_cast(user)) { + if (!split.getLengths().getDefiningOp() || + !llvm::isa(split.getLengths().getDefiningOp())) { + return std::nullopt; + } + RankedTensorType t; + int64_t count; + if (failed(GetSplitElementTypeAndCount(split, &t, &count))) { + has_failure = true; + return std::nullopt; + } + return t; + } else if (auto scatter = + llvm::dyn_cast(user)) { + // TensorArrayScatter writes vector of tensors to TensorArray. We can + // deduce the shape of TensorArray by dropping the 0th dim of + // TensorArrayScatter `value`. + auto t = + llvm::dyn_cast(scatter.getValue().getType()); + if (!t || t.getShape().empty()) return std::nullopt; + return RankedTensorType::get(t.getShape().drop_front(), + t.getElementType()); + } else if (auto gather = + llvm::dyn_cast(user)) { + // Try to infer from result type of gather. + auto t = + llvm::dyn_cast(gather.getValue().getType()); + if (t && !t.getShape().empty()) + return RankedTensorType::get(t.getShape().drop_front(), + t.getElementType()); + // Try to infer from `element_shape` attribute of gather. + auto element_shape = llvm::dyn_cast_if_present( + gather.getElementShapeAttr()); + if (element_shape && element_shape.hasStaticShape()) { + return RankedTensorType::get(element_shape.getShape(), + gather.getDtype()); + } + } + return std::nullopt; + }); + if (!elem_type) return std::nullopt; + return llvm::to_vector<8>(elem_type->getShape()); +} + +void ReplaceAllUsesWithCast(Value old_val, Value new_val) { + if (old_val.use_empty()) return; + auto builder = OpBuilder(old_val.getDefiningOp()); + auto cast_op = tensor::CastOp::create(builder, old_val.getLoc(), + old_val.getType(), new_val); + old_val.replaceAllUsesWith(cast_op); +} + +void ReplaceAllUsesExceptTerminator(Value old_val, Value new_val) { + if (old_val.getType() == new_val.getType()) { + old_val.replaceAllUsesWith(new_val); + return; + } + Operation* old_op = old_val.getDefiningOp(); + Operation* terminator_op = + old_op->getParentOfType().front().getTerminator(); + llvm::SmallPtrSet exceptions = {terminator_op}; + old_val.replaceAllUsesExcept(new_val, exceptions); +} + +struct TensorArrayStats { + // Whether a write op should accumulate with the old value. Set to true if + // this is a gradient. + bool accumulate_on_write; + // Maps from a gradient source string to the local variable to the gradient. + llvm::StringMap grads; +}; + +LogicalResult HandleTensorArrayV3Op( + TF::TensorArrayV3Op ta, ModuleOp module, + llvm::SmallDenseMap* stats) { + auto elem_shape = GetTensorArrayElementShape(ta, module); + if (!elem_shape) return ta.emitOpError("unknown element shape"); + if (ta.getDynamicSize()) { + return ta.emitOpError("dynamic tensor array size is unsupported"); + } + Value buffer; + OpBuilder builder(ta); + if (failed(cutil::CreateInitBufferValue(*elem_shape, ta.getSize(), ta, + ta.getDtype(), builder, &buffer))) { + return failure(); + } + auto var_type = RankedTensorType::get( + {}, TF::ResourceType::get( + ArrayRef{llvm::cast(buffer.getType())}, + ta.getContext())); + auto local_var = TF::MlirLocalVarOp::create( + builder, ta.getLoc(), ArrayRef{var_type}, ArrayRef{}); + cutil::WriteLocalVariable(local_var, buffer, builder, ta.getLoc()); + ta.getHandle().replaceAllUsesWith(local_var); + // The flow output is just a way for the front end to enforce ordering among + // tensor array ops, but in the MLIR TF dialect they have sequential ordering. + // Just create a constant to replace its uses. + tensorflow::Tensor scalar_tensor(tensorflow::DT_FLOAT, {}); + scalar_tensor.scalar()() = 0.0f; + auto flow = TF::ConstOp::create( + builder, ta.getLoc(), + tensorflow::ConvertTensor(scalar_tensor, &builder).value()); + ta.getFlow().replaceAllUsesWith(flow); + ta.erase(); + (*stats)[local_var].accumulate_on_write = false; + return success(); +} + +LogicalResult HandleTensorArrayReadV3Op( + TF::TensorArrayReadV3Op read, + const llvm::SmallDenseMap& stats) { + auto local_var = read.getHandle(); + if (stats.count(local_var) == 0) { + return read.emitOpError("unknown tensor array"); + } + OpBuilder builder(read); + auto buffer = cutil::ReadLocalVariable(local_var, builder, read.getLoc()); + auto index_reshape = + cutil::ReshapeScalarToSizeType(builder, read.getIndex(), read.getLoc()); + auto elem = cutil::GetElement(index_reshape, buffer, builder, read.getLoc()); + ReplaceAllUsesExceptTerminator(read.getValue(), elem); + ReplaceAllUsesWithCast(read.getValue(), elem); + read.erase(); + // The clear_after_read attribute does not mean setting the tensor to 0 after + // read; instead it does not allow a second read before the next write. We + // follow the old bridge's implementation not to do anything here. + return success(); +} + +LogicalResult HandleTensorArrayWriteV3Op( + TF::TensorArrayWriteV3Op write, + const llvm::SmallDenseMap& stats) { + auto local_var = write.getHandle(); + auto stat_it = stats.find(local_var); + if (stat_it == stats.end()) return write.emitOpError("unknown tensor array"); + OpBuilder builder(write); + auto buffer = cutil::ReadLocalVariable(local_var, builder, write.getLoc()); + auto index_reshape = + cutil::ReshapeScalarToSizeType(builder, write.getIndex(), write.getLoc()); + Value elem = write.getValue(); + if (stat_it->getSecond().accumulate_on_write) { + // Get the old slice, and accumulate with it. We set keep_slice_shape + // (keeping the leading size-1 dimension) because it avoids reshape back and + // forth. + auto original_elem = + cutil::GetElement(index_reshape, buffer, builder, write.getLoc(), + /*keep_slice_shape=*/true); + // Add a size-1 leading dimension to elem. + auto slice_type = llvm::cast(original_elem.getType()); + elem = TF::ReshapeOp::create( + builder, write.getLoc(), ArrayRef{slice_type}, + ArrayRef{elem, cutil::GetR1Const(slice_type.getShape(), builder, + write.getLoc())}); + elem = + cutil::AccumulateBuffers(elem, original_elem, builder, write.getLoc()); + } + buffer = + cutil::SetElement(index_reshape, buffer, elem, builder, write.getLoc()); + cutil::WriteLocalVariable(local_var, buffer, builder, write.getLoc()); + write.getFlowOut().replaceAllUsesWith(write.getFlowIn()); + write.erase(); + return success(); +} + +LogicalResult HandleTensorArrayConcatV3Op( + TF::TensorArrayConcatV3Op concat, + const llvm::SmallDenseMap& stats) { + auto local_var = concat.getHandle(); + if (stats.count(local_var) == 0) { + return concat.emitOpError("unknown tensor array"); + } + OpBuilder builder(concat); + auto buffer = cutil::ReadLocalVariable(local_var, builder, concat.getLoc()); + auto buffer_type = llvm::cast(buffer.getType()); + if (buffer_type.getShape().size() <= 1) { + return concat.emitOpError("cannot concat on scalar-element tensor array"); + } + // Merget he first two dimensions. + auto shape = llvm::to_vector<8>(buffer_type.getShape().drop_front()); + shape[0] *= buffer_type.getDimSize(0); + buffer = TF::ReshapeOp::create( + builder, concat.getLoc(), + ArrayRef{ + RankedTensorType::get(shape, buffer_type.getElementType())}, + ArrayRef{buffer, + cutil::GetR1Const(shape, builder, concat.getLoc())}); + ReplaceAllUsesExceptTerminator(concat.getValue(), buffer); + ReplaceAllUsesWithCast(concat.getValue(), buffer); + + // Create the lengths as a list of the same value (element size). + tensorflow::Tensor lengths_tensor(tensorflow::DT_INT64, + {buffer_type.getDimSize(0)}); + for (int64_t i = 0; i < buffer_type.getDimSize(0); ++i) { + lengths_tensor.vec()(i) = buffer_type.getDimSize(1); + } + concat.getLengths().replaceAllUsesWith(TF::ConstOp::create( + builder, concat.getLoc(), + tensorflow::ConvertTensor(lengths_tensor, &builder).value())); + concat.erase(); + return success(); +} + +LogicalResult HandleTensorArraySplitV3Op( + TF::TensorArraySplitV3Op split, + const llvm::SmallDenseMap& stats) { + auto local_var = split.getHandle(); + if (stats.count(local_var) == 0) { + return split.emitOpError("unknown tensor array"); + } + OpBuilder builder(split); + int64_t count; + RankedTensorType elem_type; + if (failed(GetSplitElementTypeAndCount(split, &elem_type, &count))) { + return failure(); + } + llvm::SmallVector buffer_shape; + buffer_shape.push_back(count); + for (int64_t dim : elem_type.getShape()) buffer_shape.push_back(dim); + // Reshape the input to match the buffer of the tensor array. + Value buffer = TF::ReshapeOp::create( + builder, split.getLoc(), + ArrayRef{RankedTensorType::get( + buffer_shape, elem_type.getElementType())}, + ArrayRef{split.getValue(), + cutil::GetR1Const(buffer_shape, builder, + split.getLoc())}) + .getOutput(); + // Accumulate with the old buffer. + auto old_buffer = + cutil::ReadLocalVariable(local_var, builder, split.getLoc()); + buffer = + cutil::AccumulateBuffers(old_buffer, buffer, builder, split.getLoc()); + cutil::WriteLocalVariable(local_var, buffer, builder, split.getLoc()); + split.getFlowOut().replaceAllUsesWith(split.getFlowIn()); + split.erase(); + return success(); +} + +LogicalResult HandleTensorArraySizeV3Op( + TF::TensorArraySizeV3Op size, + const llvm::SmallDenseMap& stats) { + auto local_var = size.getHandle(); + if (stats.count(local_var) == 0) { + return size.emitOpError("unknown tensor array"); + } + auto buffer_type = llvm::cast( + llvm::cast(getElementTypeOrSelf(local_var.getType())) + .getSubtypes()[0]); + OpBuilder builder(size); + auto result = cutil::CreateScalarConst(buffer_type.getDimSize(0), builder, + size.getLoc()); + size.getSize().replaceAllUsesWith(result); + size.erase(); + return success(); +} + +LogicalResult CreateAndInitializeGradVariable(Type local_var_type, + Operation* op, Value* var) { + OpBuilder builder(op); + *var = TF::MlirLocalVarOp::create( + builder, op->getLoc(), ArrayRef{local_var_type}, ArrayRef{}); + Value buffer; + auto buffer_type = llvm::cast( + llvm::cast(getElementTypeOrSelf(local_var_type)) + .getSubtypes()[0]); + if (failed(cutil::CreateInitBufferValue( + buffer_type.getShape().drop_front(), buffer_type.getDimSize(0), op, + buffer_type.getElementType(), builder, &buffer))) { + return failure(); + } + cutil::WriteLocalVariable(*var, buffer, builder, op->getLoc()); + return success(); +} + +LogicalResult HandleTensorArrayGradV3Op( + TF::TensorArrayGradV3Op grad, + llvm::SmallDenseMap* stats) { + auto local_var = grad.getHandle(); + OpBuilder builder(grad); + Value grad_var; + auto sit = stats->find(local_var); + if (sit == stats->end()) return grad.emitOpError("unknown tensor array"); + auto emplace_res = + sit->getSecond().grads.try_emplace(grad.getSource().str(), Value()); + if (!emplace_res.second) { + // If the source has been assigned a grad, use it. + grad_var = emplace_res.first->second; + } else { + if (failed(CreateAndInitializeGradVariable(local_var.getType(), grad, + &grad_var))) { + return failure(); + } + emplace_res.first->second = grad_var; + // Write to a grad accumulates with previous writes. + (*stats)[grad_var].accumulate_on_write = true; + } + grad.getFlowOut().replaceAllUsesWith(grad.getFlowIn()); + grad.getGradHandle().replaceAllUsesWith(grad_var); + grad.erase(); + return success(); +} + +LogicalResult HandleTensorArrayGatherV3Op( + TF::TensorArrayGatherV3Op gather, + const llvm::SmallDenseMap& stats) { + auto local_var = gather.getHandle(); + if (stats.count(local_var) == 0) { + return gather.emitOpError("unknown tensor array"); + } + OpBuilder builder(gather); + auto buffer = cutil::ReadLocalVariable(local_var, builder, gather.getLoc()); + auto result = cutil::GatherElements(gather.getIndices(), buffer, builder, + gather.getLoc()); + ReplaceAllUsesExceptTerminator(gather.getValue(), result); + ReplaceAllUsesWithCast(gather.getValue(), result); + gather.erase(); + return success(); +} + +LogicalResult HandleTensorArrayScatterV3Op( + TF::TensorArrayScatterV3Op scatter, + const llvm::SmallDenseMap& stats) { + auto local_var = scatter.getHandle(); + if (stats.count(local_var) == 0) { + return scatter.emitOpError("unknown tensor array"); + } + OpBuilder builder(scatter); + auto buffer = cutil::ReadLocalVariable(local_var, builder, scatter.getLoc()); + buffer = + cutil::ScatterAccumulateElements(scatter.getIndices(), scatter.getValue(), + buffer, builder, scatter.getLoc()); + cutil::WriteLocalVariable(local_var, buffer, builder, scatter.getLoc()); + scatter.getFlowOut().replaceAllUsesWith(scatter.getFlowIn()); + scatter.erase(); + return success(); +} + +// Updates func's type according to its current arguments and return values. +void UpdateFuncType(func::FuncOp func) { + llvm::SmallVector arg_types; + for (auto arg : func.getArguments()) arg_types.push_back(arg.getType()); + func.setType( + FunctionType::get(func.getContext(), arg_types, + func.front().getTerminator()->getOperandTypes())); +} + +// Finds the accessed gradient sources for each tensor array argument. +llvm::SmallDenseMap> AccessedGradients( + ArrayRef funcs, ModuleOp module) { + llvm::SmallDenseMap> result; + llvm::SmallDenseMap> result_sets; + auto insert = [&](Value v, const string& source, const Block& func_block) { + auto arg = dyn_cast(v); + if (!arg || arg.getOwner() != &func_block) return; + auto insert_res = result_sets[arg.getArgNumber()].insert(source); + if (!insert_res.second) return; + result[arg.getArgNumber()].push_back(source); + }; + for (func::FuncOp func : funcs) { + const Block& func_block = func.front(); + // Walk all operations and nested regions to find accessed gradient sources + // for function arguments. + func.walk([&](Operation* op) { + if (llvm::isa(op)) { + op->replaceAllUsesWith(op->getOperands()); + return; + } + if (auto grad = llvm::dyn_cast(op)) { + insert(grad.getHandle(), grad.getSource().str(), func_block); + } else if (auto while_op = llvm::dyn_cast(op)) { + for (const auto& entry : AccessedGradients( + {while_op.body_function(), while_op.cond_function()}, module)) + for (const string& source : entry.getSecond()) + insert(while_op.getOperand(entry.getFirst()), source, func_block); + } else if (auto if_op = llvm::dyn_cast(op)) { + for (const auto& entry : AccessedGradients( + {if_op.then_function(), if_op.else_function()}, module)) + for (const string& source : entry.getSecond()) + insert(if_op.getOperand(entry.getFirst() + 1), source, func_block); + } else if (auto call = llvm::dyn_cast(op)) { + auto callee = dyn_cast(call.resolveCallable()); + for (const auto& entry : AccessedGradients({callee}, module)) + for (const string& source : entry.getSecond()) + insert(call.getArgOperands()[entry.getFirst()], source, func_block); + } + }); + } + return result; +} + +// Contains cached information for decomposed callee functions for (stateful) +// partitioned call ops. +struct PartitionedCallTensorArrayOpsInfo { + bool signature_change; + func::FuncOp decomposed_callee; + llvm::SmallVector>, 4> + arg_grads; + llvm::SmallVector, 4> ret_forward_input; +}; + +// Updates a called function's input signature by adjusting resource types, and +// adding required gradient arguments. +void ChangeFunctionInputSignature( + func::FuncOp func, + const llvm::SmallDenseMap>& grads, + llvm::function_ref ta_arg_buffer_type, + llvm::function_ref ta_accumulate_on_write, + llvm::SmallDenseMap* stats) { + int64_t original_args = func.getNumArguments(); + for (int64_t argnum = 0; argnum < original_args; ++argnum) { + auto arg = func.getArgument(argnum); + Type t = ta_arg_buffer_type(argnum); + if (!t) continue; + arg.setType(t); + auto grad_it = grads.find(argnum); + if (grad_it == grads.end()) continue; + llvm::StringMap grads_map; + for (const string& source : grad_it->getSecond()) { + auto g = func.front().addArgument(t, func.getLoc()); + (*stats)[g].accumulate_on_write = true; + grads_map[source] = g; + } + auto& stat = (*stats)[arg]; + stat.accumulate_on_write = ta_accumulate_on_write(argnum); + stat.grads = std::move(grads_map); + } + UpdateFuncType(func); +} + +LogicalResult DecomposeTensorArrayOps( + Block*, ModuleOp, llvm::SmallDenseMap*, + llvm::StringMap*); + +LogicalResult HandleWhileOp(TF::WhileOp while_op, ModuleOp module, + llvm::SmallDenseMap* stats, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto body = while_op.body_function(); + auto cond = while_op.cond_function(); + auto grads = AccessedGradients({body, cond}, module); + auto ta_arg_buffer_type = [&](int64_t index) -> Type { + auto it = stats->find(while_op.getOperand(index)); + if (it == stats->end()) return nullptr; + return it->getFirst().getType(); + }; + auto ta_accumulate_on_write = [&](int64_t index) { + auto it = stats->find(while_op.getOperand(index)); + if (it == stats->end()) return false; + return it->getSecond().accumulate_on_write; + }; + llvm::SmallDenseMap body_stats; + ChangeFunctionInputSignature(body, grads, ta_arg_buffer_type, + ta_accumulate_on_write, &body_stats); + llvm::SmallDenseMap cond_stats; + ChangeFunctionInputSignature(cond, grads, ta_arg_buffer_type, + ta_accumulate_on_write, &cond_stats); + if (failed(DecomposeTensorArrayOps(&body.front(), module, &body_stats, + decomposed_partitioned_call_callees)) || + failed(DecomposeTensorArrayOps(&cond.front(), module, &cond_stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (body_stats.empty() && cond_stats.empty()) return success(); + auto old_body_ret = body.front().getTerminator(); + auto new_retvals = llvm::to_vector<8>(old_body_ret->getOperands()); + for (int64_t i = 0; i < while_op.getNumResults(); ++i) { + if (!ta_arg_buffer_type(i)) continue; + auto retval = old_body_ret->getOperand(i); + auto arg = dyn_cast(retval); + if (!arg) { + return while_op.emitOpError( + "output tensor array does not alias input in a while loop"); + } + for (const string& source : grads[i]) { + new_retvals.push_back(body_stats[arg].grads[source]); + } + } + auto ret_op_builder = OpBuilder(old_body_ret); + func::ReturnOp::create(ret_op_builder, old_body_ret->getLoc(), new_retvals); + old_body_ret->erase(); + UpdateFuncType(body); + // Recreate the while op. + auto operands = llvm::to_vector<8>(while_op.getOperands()); + for (int64_t i = 0; i < while_op.getNumOperands(); ++i) { + auto grad_it = grads.find(i); + auto& stat = (*stats)[operands[i]]; + if (grad_it == grads.end()) continue; + for (const string& source : grad_it->getSecond()) { + auto it = stat.grads.find(source); + if (it != stat.grads.end()) { + operands.push_back(it->second); + } else { + Value grad_var; + if (failed(CreateAndInitializeGradVariable(operands[i].getType(), + while_op, &grad_var))) { + return failure(); + } + stat.grads[source] = grad_var; + operands.push_back(grad_var); + (*stats)[grad_var].accumulate_on_write = true; + } + } + } + OpBuilder while_op_builder(while_op); + auto new_while = TF::WhileOp::create(while_op_builder, while_op.getLoc(), + body.getFunctionType().getInputs(), + operands, while_op->getAttrs()); + for (int64_t i = 0; i < while_op.getNumOperands(); ++i) { + if (ta_arg_buffer_type(i)) { + while_op.getResult(i).replaceAllUsesWith(while_op.getOperand(i)); + } else { + while_op.getResult(i).replaceAllUsesWith(new_while.getResult(i)); + } + } + while_op.erase(); + return success(); +} + +LogicalResult HandleIfOp(TF::IfOp if_op, ModuleOp module, + llvm::SmallDenseMap* stats, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto then_branch = if_op.then_function(); + auto else_branch = if_op.else_function(); + auto grads = AccessedGradients({then_branch, else_branch}, module); + auto ta_arg_buffer_type = [&](int64_t index) -> Type { + auto it = stats->find(if_op.getOperand(index + 1)); + if (it == stats->end()) return nullptr; + return it->getFirst().getType(); + }; + auto ta_accumulate_on_write = [&](int64_t index) { + auto it = stats->find(if_op.getOperand(index + 1)); + if (it == stats->end()) return false; + return it->getSecond().accumulate_on_write; + }; + llvm::SmallDenseMap then_stats; + ChangeFunctionInputSignature(then_branch, grads, ta_arg_buffer_type, + ta_accumulate_on_write, &then_stats); + llvm::SmallDenseMap else_stats; + ChangeFunctionInputSignature(else_branch, grads, ta_arg_buffer_type, + ta_accumulate_on_write, &else_stats); + if (failed(DecomposeTensorArrayOps(&then_branch.front(), module, &then_stats, + decomposed_partitioned_call_callees)) || + failed(DecomposeTensorArrayOps(&else_branch.front(), module, &else_stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (then_stats.empty() && else_stats.empty()) return success(); + // Recreate the if op. + auto operands = llvm::to_vector<8>(if_op.getOperands()); + for (int64_t i = 0; i < if_op.getNumOperands() - 1; ++i) { + auto grad_it = grads.find(i); + auto& stat = (*stats)[operands[i + 1]]; + if (grad_it == grads.end()) continue; + for (const string& source : grad_it->getSecond()) { + auto it = stat.grads.find(source); + if (it != stat.grads.end()) { + operands.push_back(it->second); + } else { + Value grad_var; + if (failed(CreateAndInitializeGradVariable(operands[i + 1].getType(), + if_op, &grad_var))) { + return failure(); + } + stat.grads[source] = grad_var; + operands.push_back(grad_var); + (*stats)[grad_var].accumulate_on_write = true; + } + } + } + OpBuilder builder(if_op); + auto new_if = TF::IfOp::create(builder, if_op.getLoc(), + then_branch.getFunctionType().getResults(), + operands, if_op->getAttrs()); + auto ret_forwards_input = [](func::FuncOp f, int64_t ret_ind) -> int64_t { + auto retval = f.front().getTerminator()->getOperand(ret_ind); + auto arg = dyn_cast(retval); + if (!arg) return -1; + return arg.getArgNumber(); + }; + for (int64_t i = 0; i < if_op.getNumResults(); ++i) { + if (!isa( + getElementTypeOrSelf(if_op.getResult(i).getType()))) { + if_op.getResult(i).replaceAllUsesWith(new_if.getResult(i)); + continue; + } + int64_t then_forward_input = ret_forwards_input(then_branch, i); + int64_t else_foward_input = ret_forwards_input(else_branch, i); + if (then_forward_input != else_foward_input || then_forward_input < 0) { + return if_op.emitOpError( + "branches do not forward the same input resource"); + } + if_op.getResult(i).replaceAllUsesWith( + if_op.getOperand(then_forward_input + 1)); + } + if_op.erase(); + return success(); +} + +template +LogicalResult HandlePartitionedCallOp( + CallOp call, func::FuncOp callee, ModuleOp module, + llvm::SmallDenseMap* stats, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto emplace_res = decomposed_partitioned_call_callees->try_emplace( + callee.getName(), PartitionedCallTensorArrayOpsInfo()); + auto& info = emplace_res.first->second; + // Recreates the call op with info. + auto recreate_caller = [&]() -> LogicalResult { + auto new_operands = llvm::to_vector<8>(call.getOperands()); + for (const auto& entry : info.arg_grads) { + auto it = stats->find(call.getOperand(entry.first)); + if (it == stats->end()) return call.emitOpError("unknown tensor array"); + for (const string& source : entry.second) { + auto grad_it = it->getSecond().grads.find(source); + if (grad_it != it->getSecond().grads.end()) { + new_operands.push_back(grad_it->second); + } else { + Value grad_var; + if (failed(CreateAndInitializeGradVariable(it->getFirst().getType(), + call, &grad_var))) { + return failure(); + } + it->getSecond().grads[source] = grad_var; + new_operands.push_back(grad_var); + } + } + } + OpBuilder builder(call); + auto new_call = + CallOp::create(builder, call.getLoc(), + info.decomposed_callee.getFunctionType().getResults(), + new_operands, call->getAttrs()); + new_call->setAttr( + "f", SymbolRefAttr::get( + builder.getContext(), + const_cast(info.decomposed_callee).getName())); + for (const auto& entry : info.ret_forward_input) { + call.getResult(entry.first) + .replaceAllUsesWith(call.getOperand(entry.second)); + } + call.replaceAllUsesWith(new_call); + call.erase(); + return success(); + }; + if (!emplace_res.second) { + // This callee was handled before. + if (!info.signature_change) return success(); + return recreate_caller(); + } + // Rewrite the callee. + info.signature_change = false; + auto ta_arg_buffer_type = [&](int64_t index) -> Type { + auto it = stats->find(call.getOperand(index)); + if (it == stats->end()) return nullptr; + info.signature_change = true; + return it->getFirst().getType(); + }; + auto ta_accumulate_on_write = [&](int64_t index) { + auto it = stats->find(call.getOperand(index)); + if (it == stats->end()) return false; + return it->getSecond().accumulate_on_write; + }; + func::FuncOp lowered_callee = callee; + if (!callee.isPrivate()) { + // Clone non-private callee in case of signature change. + lowered_callee = callee.clone(); + lowered_callee.setPrivate(); + } + auto grads = AccessedGradients({lowered_callee}, module); + for (int64_t i = 0; i < lowered_callee.getNumArguments(); ++i) { + auto it = grads.find(i); + if (it == grads.end()) continue; + info.arg_grads.emplace_back(i, it->getSecond()); + } + llvm::SmallDenseMap callee_stats; + ChangeFunctionInputSignature(lowered_callee, grads, ta_arg_buffer_type, + ta_accumulate_on_write, &callee_stats); + if (failed(DecomposeTensorArrayOps(&lowered_callee.front(), module, + &callee_stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + for (int64_t i = 0; i < call.getNumResults(); ++i) { + auto ret = lowered_callee.front().getTerminator()->getOperand(i); + if (!isa(getElementTypeOrSelf(ret.getType()))) continue; + auto arg = dyn_cast(ret); + if (!arg) continue; + info.ret_forward_input.emplace_back(i, arg.getArgNumber()); + } + + info.decomposed_callee = lowered_callee; + if (lowered_callee != callee) { + if (!info.signature_change) { + // Signature is not modified. We do not need to keep two copies. + lowered_callee.setName( + StringAttr::get(callee->getContext(), callee.getName())); + callee.erase(); + } else { + // Add the clone with a new name. + lowered_callee.setName(StringAttr::get( + callee->getContext(), + llvm::formatv("{0}_tensorarray_decomposed", callee.getName()).str())); + } + SymbolTable(module).insert(lowered_callee); + } + if (info.signature_change) return recreate_caller(); + return success(); +} + +LogicalResult HandleRegionControlFlowOps( + Operation& op, ModuleOp module, + llvm::SmallDenseMap* stats, + llvm::StringMap* + decomposed_partitioned_call_callees) { + for (OpOperand& operand : op.getOpOperands()) { + if (isa(getElementTypeOrSelf(operand.get().getType()))) { + return op.emitOpError() + << "found unexpected type " << operand.get().getType() + << " of operand #" << operand.getOperandNumber() + << ", resource type operands are expected to have been " + "canonicalized away for region based control flow ops"; + } + } + for (OpResult result : op.getResults()) { + if (isa(getElementTypeOrSelf(result.getType()))) { + return op.emitOpError() + << "found unexpected type " << result.getType() << " of result #" + << result.getResultNumber() + << ", resource type results are expected to have been " + "canonicalized away for region based control flow ops"; + } + } + + for (Region& region : op.getRegions()) { + if (failed(DecomposeTensorArrayOps(®ion.front(), module, stats, + decomposed_partitioned_call_callees))) + return failure(); + } + return success(); +} + +LogicalResult DecomposeTensorArrayOps( + Block* block, ModuleOp module, + llvm::SmallDenseMap* stats, + llvm::StringMap* + decomposed_partitioned_call_callees) { + for (auto& op : llvm::make_early_inc_range(block->getOperations())) { + if (llvm::isa(&op)) { + op.replaceAllUsesWith(op.getOperands()); + op.erase(); + } else if (auto ta = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayV3Op(ta, module, stats))) { + return failure(); + } + } else if (auto read = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayReadV3Op(read, *stats))) return failure(); + } else if (auto write = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayWriteV3Op(write, *stats))) return failure(); + } else if (auto concat = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayConcatV3Op(concat, *stats))) return failure(); + } else if (auto split = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArraySplitV3Op(split, *stats))) return failure(); + } else if (auto size = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArraySizeV3Op(size, *stats))) return failure(); + } else if (auto grad = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayGradV3Op(grad, stats))) return failure(); + } else if (auto gather = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayGatherV3Op(gather, *stats))) return failure(); + } else if (auto scatter = llvm::dyn_cast(&op)) { + if (failed(HandleTensorArrayScatterV3Op(scatter, *stats))) { + return failure(); + } + } else if (auto close = llvm::dyn_cast(&op)) { + close.erase(); + } else if (auto while_op = llvm::dyn_cast(&op)) { + if (failed(HandleWhileOp(while_op, module, stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto if_op = llvm::dyn_cast(&op)) { + if (failed(HandleIfOp(if_op, module, stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (llvm::isa(op) || + llvm::isa(op) || + llvm::isa(op)) { + if (failed(HandleRegionControlFlowOps( + op, module, stats, decomposed_partitioned_call_callees))) + return failure(); + } else if (auto pcall = llvm::dyn_cast(&op)) { + auto callee = pcall.func(); + if (!callee) + return pcall.emitOpError( + "TensorArray decomposition does not support call with nested " + "references."); + + if (failed( + HandlePartitionedCallOp(pcall, callee, module, stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto spcall = + llvm::dyn_cast(&op)) { + if (failed( + HandlePartitionedCallOp(spcall, spcall.func(), module, stats, + decomposed_partitioned_call_callees))) { + return failure(); + } + } + } + return success(); +} + +void TensorArrayOpsDecompositionPass::runOnOperation() { + auto module = getOperation(); + auto main = module.lookupSymbol("main"); + if (!main) return; + llvm::SmallDenseMap stats; + llvm::StringMap + decomposed_partitioned_call_callees; + if (failed(DecomposeTensorArrayOps(&main.front(), module, &stats, + &decomposed_partitioned_call_callees))) { + signalPassFailure(); + } +} + +} // namespace + +namespace TF { + +std::unique_ptr> +CreateTensorArrayOpsDecompositionPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_device_copy_conversion.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_device_copy_conversion.cc new file mode 100644 index 0000000000000000000000000000000000000000..cbaa9eb142d0f0869315638b91de605f76ff37fd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_device_copy_conversion.cc @@ -0,0 +1,122 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass folds the tf.Identity op if the operation has the same device as +// its operand. + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/Transforms/Passes.h" +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Pass/PassOptions.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" + +namespace mlir { +namespace TF { + +namespace { + +constexpr const char *kDeviceAttr = "device"; +constexpr const char *kTFDeviceAttr = "tf.device"; + +#define GEN_PASS_DEF_TENSORDEVICECOPYCONVERSIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TensorDeviceCopyConversionPass + : public impl::TensorDeviceCopyConversionPassBase< + TensorDeviceCopyConversionPass> { + void runOnOperation() override; +}; + +// Folds tf.IdentityOp and tf.IdentityNOp if op device and the argument devices +// from the defining ops match. +void TensorDeviceCopyConversionPass::runOnOperation() { + func::FuncOp func_op = getOperation(); + + auto should_fold_op_func = [&func_op](const Value &arg, + const StringAttr &op_device) { + // In TFRT TPU, tensor transfer is handled specifically by D2H and + // H2D transfer kernels. So fold the tf.Identity op if: + // * the identity op is placed on TPU, and + // * the arg to the identity op is produced by a TPUExecuteOp. + if (op_device && op_device.getValue().contains("TPU")) { + return true; + } + + Operation *def_op = arg.getDefiningOp(); + // If the arg to this identity op is the arg of a function, there's no + // defining op. + if (def_op != nullptr && + (isa(def_op))) { + return true; + } + if (BlockArgument block_arg = mlir::dyn_cast(arg)) { + // Skip the folding logic if the block argument is not from the function + // arguments. This can happen when the argument is from a while loop. + if (block_arg.getParentRegion() != &func_op.getRegion()) { + return false; + } + if (StringAttr attr = func_op.getArgAttrOfType( + block_arg.getArgNumber(), kTFDeviceAttr)) { + return op_device == attr; + } + } else if (StringAttr attr = arg.getDefiningOp()->getAttrOfType( + kDeviceAttr)) { + return op_device == attr; + } + // Fold tf.Identity when arg device is not defined. + return true; + }; + + func_op.walk([&should_fold_op_func](TF::IdentityOp op) { + StringAttr op_device = op->getAttrOfType(kDeviceAttr); + if (should_fold_op_func(op.getOperand(), op_device)) { + op.replaceAllUsesWith(op.getOperand()); + op.erase(); + } + return WalkResult::advance(); + }); + + func_op.walk([&should_fold_op_func](TF::IdentityNOp op) { + StringAttr op_device = op->getAttrOfType(kDeviceAttr); + bool should_fold = llvm::all_of( + op.getOperands(), [&op_device, &should_fold_op_func](const Value &arg) { + return should_fold_op_func(arg, op_device); + }); + if (should_fold) { + op.replaceAllUsesWith(op.getOperands()); + op.erase(); + } + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> +CreateTensorDeviceCopyConversionPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_list_ops_decomposition.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_list_ops_decomposition.cc new file mode 100644 index 0000000000000000000000000000000000000000..21396d6b6d61e1906d7dc57808d16f45b1637eed --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tensor_list_ops_decomposition.cc @@ -0,0 +1,980 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/collection_ops_util.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/types.h" + +namespace mlir { + +namespace { + +namespace cutil = TF::collection_ops_util; + +#define GEN_PASS_DEF_TENSORLISTOPSDECOMPOSITIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TensorListOpsDecompositionPass + : public impl::TensorListOpsDecompositionPassBase< + TensorListOpsDecompositionPass> { + void runOnOperation() override; +}; + +// Updates func's type according to its current arguments and return values. +void UpdateFuncType(func::FuncOp func) { + llvm::SmallVector arg_types; + for (auto arg : func.getArguments()) arg_types.push_back(arg.getType()); + func.setType( + FunctionType::get(func.getContext(), arg_types, + func.front().getTerminator()->getOperandTypes())); +} + +// Holds the size value of a tensor list and whether the size is statically +// known (fixed). +struct SizeInfo { + Value size; + bool fixed; +}; + +// Modifies a function's signature to rewrite tensor list arguments to buffers +// and sizes. +void ModifyFunctionSignature( + func::FuncOp func, Type size_type, + llvm::SmallDenseMap* buffer_to_size, + llvm::function_ref(int64_t)> arg_to_buffer_type, + llvm::function_ref arg_buffer_size_is_fixed) { + auto new_input_types = llvm::to_vector<8>(func.getFunctionType().getInputs()); + int64_t original_arg_count = new_input_types.size(); + Location loc = func.getLoc(); + for (int64_t i = 0; i < original_arg_count; ++i) { + auto buffer_type = arg_to_buffer_type(i); + if (!buffer_type.has_value()) continue; + func.getArgument(i).setType(*buffer_type); + new_input_types[i] = *buffer_type; + auto size_arg = func.front().addArgument(size_type, loc); + new_input_types.push_back(size_arg.getType()); + if (buffer_to_size) { + (*buffer_to_size)[func.getArgument(i)] = {size_arg, + arg_buffer_size_is_fixed(i)}; + } + } + UpdateFuncType(func); +} + +// Holds information about a decomposed callee function for +// PartitionedCall/StatefulPartitionedCall. +struct PartitionedCallDecompositionInfo { + bool signature_change; + func::FuncOp decomposed_callee; + llvm::SmallDenseMap buffer_arg_to_size_arg; + // Each element is a tuple of (buffer_return_index, size_return_index, + // fixed_size). + llvm::SmallVector, 8> + buffer_ret_to_size_ret; +}; + +LogicalResult DecomposeTensorListOpsInternal( + Block*, ModuleOp, llvm::SmallDenseMap*, + llvm::StringMap*); + +// Adds the corresponding sizes of tensor list buffers in block's terminator +// to the list of return values. Returns the mapping from the buffer +// indices to the added size indices, which is a list of tuples +// (buffer_return_index, size_return_index, fixed_size). +template +llvm::SmallVector, 8> +AddTensorListSizesToTerminator( + Block& block, const llvm::SmallDenseMap& buffer_to_size) { + auto old_terminator = block.getTerminator(); + auto new_outputs = llvm::to_vector<8>(old_terminator->getOperands()); + llvm::SmallVector, 8> + output_buffer_to_size; + for (auto retval : llvm::enumerate(old_terminator->getOperands())) { + auto it = buffer_to_size.find(retval.value()); + if (it == buffer_to_size.end()) continue; + output_buffer_to_size.emplace_back(retval.index(), new_outputs.size(), + it->getSecond().fixed); + new_outputs.push_back(it->getSecond().size); + } + OpBuilder builder(old_terminator); + TerminatorOp::create(builder, old_terminator->getLoc(), new_outputs); + old_terminator->erase(); + return output_buffer_to_size; +} + +// Adds the corresponding sizes of tensor list buffers in func's return values +// to the list of return values. Returns the mapping from the buffer indices to +// the added size indices, which is a list of tuples (buffer_return_index, +// size_return_index, fixed_size). +llvm::SmallVector, 8> ModifyFunctionReturn( + func::FuncOp func, + const llvm::SmallDenseMap& buffer_to_size) { + auto output_buffer_to_size = AddTensorListSizesToTerminator( + func.front(), buffer_to_size); + UpdateFuncType(func); + return output_buffer_to_size; +} + +LogicalResult HandleWhileOp( + TF::WhileOp while_op, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + // Rewrite body. + auto body = while_op.body_function(); + llvm::SmallDenseMap body_map; + auto find_arg_tensor_list_type = [&](int64_t index) -> std::optional { + auto it = buffer_to_size->find(while_op.getOperand(index)); + if (it == buffer_to_size->end()) return std::nullopt; + return it->getFirst().getType(); + }; + auto arg_buffer_size_is_fixed = [&](int64_t index) { + return (*buffer_to_size)[while_op.getOperand(index)].fixed; + }; + OpBuilder builder(while_op); + ModifyFunctionSignature(body, cutil::GetSizeType(builder), &body_map, + find_arg_tensor_list_type, arg_buffer_size_is_fixed); + if (failed(DecomposeTensorListOpsInternal( + &body.front(), module, &body_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + auto output_buffer_to_size = ModifyFunctionReturn(body, body_map); + + // Rewrite cond. + auto cond = while_op.cond_function(); + llvm::SmallDenseMap cond_map; + ModifyFunctionSignature(cond, cutil::GetSizeType(builder), &cond_map, + find_arg_tensor_list_type, arg_buffer_size_is_fixed); + if (failed(DecomposeTensorListOpsInternal( + &cond.front(), module, &cond_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + if (output_buffer_to_size.empty()) { + return success(); + } + // Create the new while op. + auto new_while_operands = llvm::to_vector<8>(while_op.getOperands()); + for (int64_t i = 0; i < while_op.getNumResults(); ++i) { + auto it = buffer_to_size->find(while_op.getOperand(i)); + if (it == buffer_to_size->end()) continue; + new_while_operands.push_back(it->getSecond().size); + } + auto new_while = TF::WhileOp::create( + builder, while_op.getLoc(), body.getFunctionType().getInputs(), + new_while_operands, while_op->getAttrs()); + for (const auto& entry : output_buffer_to_size) { + (*buffer_to_size)[new_while.getResult(std::get<0>(entry))] = { + new_while.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + while_op.replaceAllUsesWith( + new_while.getResults().take_front(while_op.getNumResults())); + while_op.erase(); + return success(); +} + +template +LogicalResult HandleCaseOrIfOp( + CaseOrIfOp op, ArrayRef branches, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + // Rewrite the branches. + SmallVector, 2> branch_maps; + branch_maps.resize(branches.size()); + + auto find_arg_buffer_type = [&](int64_t index) -> std::optional { + auto it = buffer_to_size->find(op.getOperand(index + 1)); + if (it == buffer_to_size->end()) return std::nullopt; + return it->getFirst().getType(); + }; + auto arg_buffer_size_is_fixed = [&](int64_t index) { + return (*buffer_to_size)[op.getOperand(index + 1)].fixed; + }; + OpBuilder builder(op); + for (const auto& pair : llvm::zip(branches, branch_maps)) { + func::FuncOp branch = std::get<0>(pair); + llvm::SmallDenseMap& branch_map = std::get<1>(pair); + ModifyFunctionSignature(branch, cutil::GetSizeType(builder), &branch_map, + find_arg_buffer_type, arg_buffer_size_is_fixed); + + if (failed(DecomposeTensorListOpsInternal( + &branch.front(), module, &branch_map, + decomposed_partitioned_call_callees))) + return failure(); + } + + const bool arg_no_changed = branch_maps.front().empty(); + auto output_buffer_to_size = + ModifyFunctionReturn(branches.front(), branch_maps.front()); + for (const auto& pair : llvm::drop_begin(llvm::zip(branches, branch_maps), 1)) + ModifyFunctionReturn(std::get<0>(pair), std::get<1>(pair)); + + if (output_buffer_to_size.empty() && arg_no_changed) return success(); + + // Recreate the op. + auto new_operands = llvm::to_vector<8>(op.getOperands()); + for (int64_t i = 1; i < op.getNumOperands(); ++i) { + auto it = buffer_to_size->find(op.getOperand(i)); + if (it == buffer_to_size->end()) continue; + new_operands.push_back(it->getSecond().size); + } + func::FuncOp first_branch = branches.front(); + builder.setInsertionPoint(op); + auto new_op = CaseOrIfOp::create(builder, op.getLoc(), + first_branch.getFunctionType().getResults(), + new_operands, op->getAttrs()); + for (const auto& entry : output_buffer_to_size) { + (*buffer_to_size)[new_op.getResult(std::get<0>(entry))] = { + new_op.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + op.replaceAllUsesWith(new_op.getResults().take_front(op.getNumResults())); + op.erase(); + return success(); +} + +LogicalResult HandleWhileRegionOp( + TF::WhileRegionOp while_op, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + OpBuilder builder(while_op); + auto modify_region_arguments = [&](Region& region) { + int64_t original_arg_count = region.getNumArguments(); + for (int64_t i = 0; i < original_arg_count; ++i) { + auto operand = while_op.getOperand(i); + auto it = buffer_to_size->find(operand); + if (it == buffer_to_size->end()) continue; + auto buffer_type = it->getFirst().getType(); + region.getArgument(i).setType(buffer_type); + auto size_arg = + region.addArgument(cutil::GetSizeType(builder), region.getLoc()); + (*buffer_to_size)[region.getArgument(i)] = {size_arg, + it->getSecond().fixed}; + } + }; + + // Rewrite body. + Region& body_region = while_op.getBody(); + modify_region_arguments(body_region); + if (failed(DecomposeTensorListOpsInternal( + &body_region.front(), module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + auto output_buffer_to_size = AddTensorListSizesToTerminator( + body_region.front(), *buffer_to_size); + + // Rewrite cond. + Region& cond_region = while_op.getCond(); + modify_region_arguments(cond_region); + if (failed(DecomposeTensorListOpsInternal( + &cond_region.front(), module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + + if (output_buffer_to_size.empty()) return success(); + + // Create the new while op. + auto new_while_operands = llvm::to_vector<8>(while_op.getOperands()); + for (int64_t i = 0; i < while_op.getNumResults(); ++i) { + auto it = buffer_to_size->find(while_op.getOperand(i)); + if (it == buffer_to_size->end()) continue; + new_while_operands.push_back(it->getSecond().size); + } + auto new_while = TF::WhileRegionOp::create( + builder, while_op.getLoc(), + body_region.front().getTerminator()->getOperandTypes(), + new_while_operands, while_op->getAttrs()); + new_while.getBody().takeBody(body_region); + new_while.getCond().takeBody(cond_region); + for (const auto& entry : output_buffer_to_size) { + (*buffer_to_size)[new_while.getResult(std::get<0>(entry))] = { + new_while.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + while_op.replaceAllUsesWith( + new_while.getResults().take_front(while_op.getNumResults())); + while_op.erase(); + return success(); +} + +LogicalResult HandleIfRegionOp( + TF::IfRegionOp if_op, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + // Rewrite the branches. + Region& then_branch = if_op.getThenBranch(); + Region& else_branch = if_op.getElseBranch(); + if (failed(DecomposeTensorListOpsInternal( + &then_branch.front(), module, buffer_to_size, + decomposed_partitioned_call_callees))) + return failure(); + if (failed(DecomposeTensorListOpsInternal( + &else_branch.front(), module, buffer_to_size, + decomposed_partitioned_call_callees))) + return failure(); + + auto output_buffer_to_size = AddTensorListSizesToTerminator( + then_branch.front(), *buffer_to_size); + AddTensorListSizesToTerminator(else_branch.front(), + *buffer_to_size); + + if (output_buffer_to_size.empty()) return success(); + + // Recreate the op. + OpBuilder builder(if_op); + auto new_op = TF::IfRegionOp::create( + builder, if_op.getLoc(), + then_branch.front().getTerminator()->getOperandTypes(), + if_op.getOperand(), if_op->getAttrs()); + for (const auto& entry : output_buffer_to_size) { + (*buffer_to_size)[new_op.getResult(std::get<0>(entry))] = { + new_op.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + + new_op.getThenBranch().takeBody(if_op.getThenBranch()); + new_op.getElseBranch().takeBody(if_op.getElseBranch()); + + if_op.replaceAllUsesWith( + new_op.getResults().take_front(if_op.getNumResults())); + if_op.erase(); + return success(); +} + +LogicalResult HandleCaseRegionOp( + TF::CaseRegionOp case_op, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + // Rewrite the branches. + RegionRange branches = case_op.getRegions(); + + for (Region* branch : branches) { + if (failed(DecomposeTensorListOpsInternal( + &branch->front(), module, buffer_to_size, + decomposed_partitioned_call_callees))) + return failure(); + } + + // Get the output buffer index to size index mapping one of the branches. It + // should be same for all the branches so we only get it for the first branch. + Region* first_branch = branches.front(); + auto output_buffer_to_size = AddTensorListSizesToTerminator( + first_branch->front(), *buffer_to_size); + for (Region* branch : branches.drop_front()) { + AddTensorListSizesToTerminator(branch->front(), + *buffer_to_size); + } + + if (output_buffer_to_size.empty()) return success(); + + // Recreate the op. + OpBuilder builder(case_op); + auto new_op = TF::CaseRegionOp::create( + builder, case_op.getLoc(), + first_branch->front().getTerminator()->getOperandTypes(), + case_op.getOperand(), case_op->getAttrs(), case_op.getNumRegions()); + for (const auto& entry : output_buffer_to_size) { + (*buffer_to_size)[new_op.getResult(std::get<0>(entry))] = { + new_op.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + + for (auto pair : llvm::zip(new_op.getRegions(), case_op.getRegions())) { + std::get<0>(pair)->takeBody(*std::get<1>(pair)); + } + case_op.replaceAllUsesWith( + new_op.getResults().take_front(case_op.getNumResults())); + case_op.erase(); + return success(); +} + +template +LogicalResult HandlePartitionedCallOp( + CallOp call, func::FuncOp callee, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + auto emplace_res = decomposed_partitioned_call_callees->try_emplace( + callee.getName(), PartitionedCallDecompositionInfo()); + auto& info = emplace_res.first->second; + // Recreates the call op with info. + auto recreate_caller = [&] { + auto new_operands = llvm::to_vector<8>(call.getOperands()); + for (int64_t i = 0; i < call.getNumOperands(); ++i) { + auto arg_it = info.buffer_arg_to_size_arg.find(i); + if (arg_it == info.buffer_arg_to_size_arg.end()) continue; + auto it = buffer_to_size->find(call.getOperand(i)); + if (it == buffer_to_size->end()) { + call.emitOpError("unknown tensor list."); + return failure(); + } + assert(arg_it->second == new_operands.size()); + new_operands.push_back(it->getSecond().size); + } + OpBuilder builder(call); + auto new_call = + CallOp::create(builder, call.getLoc(), + info.decomposed_callee.getFunctionType().getResults(), + new_operands, call->getAttrs()); + new_call->setAttr( + "f", SymbolRefAttr::get( + builder.getContext(), + const_cast(info.decomposed_callee).getName())); + for (const auto& entry : info.buffer_ret_to_size_ret) { + (*buffer_to_size)[new_call.getResult(std::get<0>(entry))] = { + new_call.getResult(std::get<1>(entry)), std::get<2>(entry)}; + } + call.replaceAllUsesWith( + new_call.getResults().take_front(call.getNumResults())); + call.erase(); + return success(); + }; + if (!emplace_res.second) { + // This callee was handled before. + if (!info.signature_change) return success(); + return recreate_caller(); + } + // Rewrite the callee. + llvm::SmallDenseMap callee_map; + func::FuncOp lowered_callee = callee; + if (!callee.isPrivate()) { + // Clone non-private callee in case of signature change. + lowered_callee = callee.clone(); + lowered_callee.setPrivate(); + } + auto find_arg_buffer_type = [&](int64_t index) -> std::optional { + auto it = buffer_to_size->find(call.getOperand(index)); + if (it == buffer_to_size->end()) return std::nullopt; + return it->getFirst().getType(); + }; + auto arg_buffer_size_is_fixed = [&](int64_t index) { + return (*buffer_to_size)[call.getOperand(index)].fixed; + }; + ModifyFunctionSignature(lowered_callee, cutil::GetSizeType(OpBuilder(call)), + &callee_map, find_arg_buffer_type, + arg_buffer_size_is_fixed); + const bool args_no_changed = callee_map.empty(); + if (failed(DecomposeTensorListOpsInternal( + &lowered_callee.front(), module, &callee_map, + decomposed_partitioned_call_callees))) { + return failure(); + } + info.buffer_ret_to_size_ret = + ModifyFunctionReturn(lowered_callee, callee_map); + info.decomposed_callee = lowered_callee; + if (args_no_changed && info.buffer_ret_to_size_ret.empty()) { + // Signature is not modified. We do not need to keep two copies. + info.signature_change = false; + if (lowered_callee != callee) { + lowered_callee.setName( + StringAttr::get(callee->getContext(), callee.getName())); + callee.erase(); + SymbolTable(module).insert(lowered_callee); + } + } else { + info.signature_change = true; + for (auto& entry : callee_map) { + auto buffer_arg = mlir::dyn_cast(entry.getFirst()); + if (!buffer_arg) continue; + info.buffer_arg_to_size_arg[buffer_arg.getArgNumber()] = + mlir::cast(entry.getSecond().size).getArgNumber(); + } + if (lowered_callee != callee) { + // Add the clone with a new name. + lowered_callee.setName(StringAttr::get( + callee->getContext(), + llvm::formatv("{0}_tensorlist_decomposed", callee.getName()).str())); + SymbolTable(module).insert(lowered_callee); + callee = lowered_callee; + } + } + if (info.signature_change) return recreate_caller(); + return success(); +} + +// Parses an R1 value to `shape` if it is a TF::ConstOp output. Otherwise, +// returns an error. +LogicalResult GetConstShapeValue(Value shape_value, + llvm::SmallVector* shape) { + auto shape_op = shape_value.getDefiningOp(); + if (!shape_op) return failure(); + auto shape_const_op = llvm::dyn_cast(shape_op); + if (!shape_const_op) return failure(); + for (const auto& v : shape_const_op.getValue().getValues()) { + int64_t dim_size = v.getSExtValue(); + if (dim_size == tensorflow::kTFDynamicSize) return failure(); + shape->push_back(dim_size); + } + return success(); +} + +// Checks the result Variant type to infer the element shape if fully defined. +// If the Variant type has multiple subtypes or does not have static shape, +// return error. +LogicalResult GetElementShapeFromResultType( + Type type, llvm::SmallVector* shape) { + auto variant_type = + mlir::dyn_cast(getElementTypeOrSelf(type)); + if (!variant_type || variant_type.getSubtypes().size() != 1) return failure(); + TensorType tensor_type = variant_type.getSubtypes().front(); + if (!tensor_type.hasStaticShape()) return failure(); + for (auto d : tensor_type.getShape()) shape->push_back(d); + return success(); +} + +LogicalResult HandleEmptyTensorListOp( + TF::EmptyTensorListOp list, + llvm::SmallDenseMap* buffer_to_size) { + Value buffer; + OpBuilder builder(list); + llvm::SmallVector element_shape; + // Infer TensorList element shape from the return type first, and then from + // the const element shape operand. We first check the return type because + // shape inference might have successfully inferred the element shape from + // write operations on the TensorList. + if (failed(GetElementShapeFromResultType(list.getType(), &element_shape))) { + if (failed(GetConstShapeValue(list.getElementShape(), &element_shape))) { + return list.emitOpError("unknown tensor list element shape"); + } + } + if (failed(cutil::CreateInitBufferValue( + element_shape, list.getMaxNumElements(), list, list.getElementDtype(), + builder, &buffer))) { + return failure(); + } + Value size = cutil::GetR1Const({0LL}, builder, list.getLoc()); + list.getHandle().replaceAllUsesWith(buffer); + (*buffer_to_size)[buffer] = {size, /*fixed=*/false}; + list.erase(); + return success(); +} + +LogicalResult HandleTensorListReserveOp( + TF::TensorListReserveOp list, + llvm::SmallDenseMap* buffer_to_size) { + Value buffer; + OpBuilder builder(list); + llvm::SmallVector element_shape; + // Infer TensorList element shape from the return type first, and then from + // the const element shape operand. We first check the return type because + // shape inference might have successfully inferred the element shape from + // write operations on the TensorList. + if (failed(GetElementShapeFromResultType(list.getType(), &element_shape))) { + if (failed(GetConstShapeValue(list.getElementShape(), &element_shape))) { + return list.emitOpError("unknown tensor list element shape"); + } + } + if (failed(cutil::CreateInitBufferValue(element_shape, list.getNumElements(), + list, list.getElementDtype(), builder, + &buffer))) { + return failure(); + } + Value size = cutil::ReshapeScalarToSizeType(builder, list.getNumElements(), + list.getLoc()); + (*buffer_to_size)[buffer] = {size, /*fixed=*/true}; + list.getHandle().replaceAllUsesWith(buffer); + list.erase(); + return success(); +} + +LogicalResult HandleTensorListFromTensorOp( + TF::TensorListFromTensorOp list, + llvm::SmallDenseMap* buffer_to_size) { + OpBuilder builder(list); + Value buffer = TF::IdentityOp::create( + builder, list.getLoc(), ArrayRef{list.getTensor().getType()}, + ArrayRef{list.getTensor()}); + auto type = mlir::cast(buffer.getType()); + if (!type.hasStaticShape()) { + return list.emitOpError("TensorListFromTensorOp input has unknown shape."); + } + Value size = cutil::GetR1Const({type.getShape()[0]}, builder, list.getLoc()); + (*buffer_to_size)[buffer] = {size, /*fixed=*/true}; + list.getOutputHandle().replaceAllUsesWith(buffer); + list.erase(); + return success(); +} + +LogicalResult HandleTensorListPushBackOp( + TF::TensorListPushBackOp push, + llvm::SmallDenseMap* buffer_to_size) { + auto buffer = push.getInputHandle(); + auto it = buffer_to_size->find(buffer); + if (it == buffer_to_size->end()) { + return push.emitOpError( + "found tf.TensorListPushBack on unknown TensorList."); + } + if (it->getSecond().fixed) { + return push.emitError("cannot push on a fixed-size tensor list"); + } + auto size = it->getSecond().size; + OpBuilder builder(push); + auto new_buffer = + cutil::SetElement(size, buffer, push.getTensor(), builder, push.getLoc()); + auto new_size = TF::AddV2Op::create( + builder, push.getLoc(), ArrayRef{size.getType()}, + ArrayRef{size, cutil::GetR1Const({1LL}, builder, push.getLoc())}); + push.getOutputHandle().replaceAllUsesWith(new_buffer); + (*buffer_to_size)[new_buffer] = {new_size, /*fixed=*/false}; + push.erase(); + return success(); +} + +LogicalResult HandleTensorListPopBackOp( + TF::TensorListPopBackOp pop, + llvm::SmallDenseMap* buffer_to_size) { + auto buffer = pop.getInputHandle(); + auto it = buffer_to_size->find(buffer); + if (it == buffer_to_size->end()) { + pop.emitOpError("found tf.TensorListPopBack on unknown TensorList."); + return failure(); + } + if (it->getSecond().fixed) { + return pop.emitError("cannot pop on a fixed-size tensor list"); + } + auto size = it->getSecond().size; + OpBuilder builder(pop); + auto new_buffer = TF::IdentityOp::create(builder, pop.getLoc(), + ArrayRef{buffer.getType()}, + ArrayRef{buffer}); + auto new_size = TF::SubOp::create( + builder, pop.getLoc(), ArrayRef{size.getType()}, + ArrayRef{size, cutil::GetR1Const({1LL}, builder, pop.getLoc())}); + auto element = cutil::GetElement(new_size, new_buffer, builder, pop.getLoc()); + pop.getOutputHandle().replaceAllUsesWith(new_buffer); + pop.getTensor().replaceAllUsesWith(element); + pop.erase(); + (*buffer_to_size)[new_buffer] = {new_size, /*fixed=*/false}; + return success(); +} + +LogicalResult HandleTensorListGetItemOp( + TF::TensorListGetItemOp get_item, + const llvm::SmallDenseMap& buffer_to_size) { + auto buffer = get_item.getInputHandle(); + auto it = buffer_to_size.find(buffer); + if (it == buffer_to_size.end()) { + get_item.emitOpError("found tf.TensorListGetItemOp on unknown TensorList."); + return failure(); + } + OpBuilder builder(get_item); + auto index = cutil::ReshapeScalarToSizeType(builder, get_item.getIndex(), + get_item.getLoc()); + auto element = + cutil::GetElement(index, buffer, OpBuilder(get_item), get_item.getLoc()); + get_item.getItem().replaceAllUsesWith(element); + get_item.erase(); + return success(); +} + +LogicalResult HandleTensorListSetItemOp( + TF::TensorListSetItemOp set_item, + llvm::SmallDenseMap* buffer_to_size) { + auto buffer = set_item.getInputHandle(); + auto it = buffer_to_size->find(buffer); + if (it == buffer_to_size->end()) { + set_item.emitOpError("found tf.TensorListSetItemOp on unknown TensorList."); + return failure(); + } + OpBuilder builder(set_item); + auto index = cutil::ReshapeScalarToSizeType(builder, set_item.getIndex(), + set_item.getLoc()); + auto new_buffer = cutil::SetElement(index, buffer, set_item.getItem(), + builder, set_item.getLoc()); + set_item.getOutputHandle().replaceAllUsesWith(new_buffer); + auto size = it->getSecond(); + (*buffer_to_size)[new_buffer] = size; + set_item.erase(); + return success(); +} + +LogicalResult HandleTensorListLengthOp( + TF::TensorListLengthOp length, + const llvm::SmallDenseMap& buffer_to_size) { + auto it = buffer_to_size.find(length.getInputHandle()); + if (it == buffer_to_size.end()) { + length.emitOpError("found tf.TensorListLength on unknown TensorList."); + return failure(); + } + OpBuilder builder(length); + if (it->getSecond().fixed) { + auto dim = cutil::CreateScalarConst( + mlir::cast(length.getInputHandle().getType()) + .getDimSize(0), + builder, length.getLoc()); + length.getLength().replaceAllUsesWith(dim); + } else { + auto current_size = it->getSecond().size; + // Reshapes the R1 length to a scalar. + auto reshape = TF::ReshapeOp::create( + builder, length.getLoc(), + ArrayRef{RankedTensorType::get( + {}, getElementTypeOrSelf(current_size.getType()))}, + ArrayRef{current_size, + cutil::GetR1Const({}, builder, length.getLoc())}); + length.getLength().replaceAllUsesWith(reshape); + } + length.erase(); + return success(); +} + +LogicalResult HandleTensorListElementShapeOp( + TF::TensorListElementShapeOp elem_shape, + const llvm::SmallDenseMap& buffer_to_size) { + if (buffer_to_size.count(elem_shape.getInputHandle()) == 0) { + return elem_shape.emitOpError("unknown tensor list"); + } + auto buffer = elem_shape.getInputHandle(); + auto result = cutil::GetR1Const( + mlir::cast(buffer.getType()).getShape().drop_front(), + OpBuilder(elem_shape), elem_shape.getLoc(), + elem_shape.getShapeType().getIntOrFloatBitWidth()); + elem_shape.getElementShape().replaceAllUsesWith(result); + elem_shape.erase(); + return success(); +} + +LogicalResult HandleTensorListGatherOp( + TF::TensorListGatherOp gather, + const llvm::SmallDenseMap& buffer_to_size) { + auto it = buffer_to_size.find(gather.getInputHandle()); + if (it == buffer_to_size.end()) { + return gather.emitOpError("unknown tensor list"); + } + auto buffer = gather.getInputHandle(); + auto result = cutil::GatherElements(gather.getIndices(), buffer, + OpBuilder(gather), gather.getLoc()); + gather.getValues().replaceAllUsesWith(result); + gather.erase(); + return success(); +} + +LogicalResult HandleTensorListScatterIntoExistingListOp( + TF::TensorListScatterIntoExistingListOp scatter, + llvm::SmallDenseMap* buffer_to_size) { + auto it = buffer_to_size->find(scatter.getInputHandle()); + if (it == buffer_to_size->end()) { + return scatter.emitOpError("unknown tensor list"); + } + auto buffer = scatter.getInputHandle(); + OpBuilder builder(scatter); + auto indices_type = + mlir::cast(scatter.getIndices().getType()); + if (!indices_type) return scatter.emitOpError("unranked indices shape"); + auto shape_type = RankedTensorType::get({2}, builder.getIntegerType(32)); + auto shape = TF::ConstOp::create( + builder, scatter.getLoc(), + DenseElementsAttr::get( + shape_type, {static_cast(indices_type.getDimSize(0)), 1})); + auto indices = TF::ReshapeOp::create(builder, scatter.getLoc(), + scatter.getIndices(), shape); + Value tensor_scatter_update = TF::TensorScatterUpdateOp::create( + builder, scatter.getLoc(), buffer, indices, scatter.getTensor()); + scatter.getOutputHandle().replaceAllUsesWith(tensor_scatter_update); + scatter.erase(); + auto size = it->getSecond(); + (*buffer_to_size)[tensor_scatter_update] = size; + return success(); +} + +LogicalResult DecomposeTensorListOpsInternal( + Block* block, ModuleOp module, + llvm::SmallDenseMap* buffer_to_size, + llvm::StringMap* + decomposed_partitioned_call_callees) { + for (auto& op : llvm::make_early_inc_range(block->getOperations())) { + // TODO(yuanzx): Add a pass to remove identities in device computation. + if (llvm::isa(&op)) { + op.replaceAllUsesWith(op.getOperands()); + op.erase(); + } else if (auto list = llvm::dyn_cast(&op)) { + if (failed(HandleEmptyTensorListOp(list, buffer_to_size))) { + return failure(); + } + } else if (auto list = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListReserveOp(list, buffer_to_size))) { + return failure(); + } + } else if (auto list = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListFromTensorOp(list, buffer_to_size))) { + return failure(); + } + } else if (auto push = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListPushBackOp(push, buffer_to_size))) { + return failure(); + } + } else if (auto pop = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListPopBackOp(pop, buffer_to_size))) { + return failure(); + } + } else if (auto get_item = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListGetItemOp(get_item, *buffer_to_size))) { + return failure(); + } + } else if (auto set_item = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListSetItemOp(set_item, buffer_to_size))) { + return failure(); + } + } else if (auto length = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListLengthOp(length, *buffer_to_size))) { + return failure(); + } + } else if (auto stack = llvm::dyn_cast(&op)) { + stack.getTensor().replaceAllUsesWith(stack.getInputHandle()); + stack.erase(); + } else if (auto elem_shape = + llvm::dyn_cast(&op)) { + if (failed(HandleTensorListElementShapeOp(elem_shape, *buffer_to_size))) { + return failure(); + } + } else if (auto gather = llvm::dyn_cast(&op)) { + if (failed(HandleTensorListGatherOp(gather, *buffer_to_size))) { + return failure(); + } + } else if (auto scatter = + llvm::dyn_cast( + &op)) { + if (failed(HandleTensorListScatterIntoExistingListOp(scatter, + buffer_to_size))) { + return failure(); + } + } else if (auto addn = llvm::dyn_cast(&op)) { + auto it = buffer_to_size->find(addn.getOperand(0)); + if (it != buffer_to_size->end()) { + addn.getSum().setType( + mlir::cast(addn.getOperand(0).getType())); + auto size = it->getSecond(); + (*buffer_to_size)[addn.getSum()] = size; + } + } else if (auto zeros = llvm::dyn_cast(&op)) { + if (buffer_to_size->count(zeros.getX()) > 0) { + zeros.getY().setType(zeros.getX().getType()); + auto size = (*buffer_to_size)[zeros.getX()]; + (*buffer_to_size)[zeros.getY()] = size; + } + } else if (auto while_op = llvm::dyn_cast(&op)) { + if (failed(HandleWhileOp(while_op, module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto if_op = llvm::dyn_cast(&op)) { + if (failed(HandleCaseOrIfOp( + if_op, {if_op.then_function(), if_op.else_function()}, module, + buffer_to_size, decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto case_op = llvm::dyn_cast(&op)) { + SmallVector branches; + case_op.get_branch_functions(branches); + if (failed(HandleCaseOrIfOp(case_op, branches, module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto pcall = llvm::dyn_cast(&op)) { + if (!pcall.func()) + return pcall.emitOpError( + "TensorList decomposition does not support call with nested " + "references."); + + if (failed(HandlePartitionedCallOp( + pcall, pcall.func(), module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto spcall = + llvm::dyn_cast(&op)) { + if (failed(HandlePartitionedCallOp( + spcall, spcall.func(), module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto while_op = llvm::dyn_cast(&op)) { + if (failed(HandleWhileRegionOp(while_op, module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto if_op = llvm::dyn_cast(&op)) { + if (failed(HandleIfRegionOp(if_op, module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } else if (auto case_op = llvm::dyn_cast(&op)) { + if (failed(HandleCaseRegionOp(case_op, module, buffer_to_size, + decomposed_partitioned_call_callees))) { + return failure(); + } + } + } + return success(); +} + +LogicalResult DecomposeTensorListOps(Block* block, ModuleOp module) { + llvm::SmallDenseMap buffer_to_size; + llvm::StringMap + decomposed_partitioned_call_callees; + return DecomposeTensorListOpsInternal(block, module, &buffer_to_size, + &decomposed_partitioned_call_callees); +} + +void TensorListOpsDecompositionPass::runOnOperation() { + auto module = getOperation(); + auto main = module.lookupSymbol("main"); + if (!main) return; + if (failed(DecomposeTensorListOps(&main.front(), module))) { + signalPassFailure(); + } +} + +} // namespace + +namespace TF { +std::unique_ptr> +CreateTensorListOpsDecompositionPass() { + return std::make_unique(); +} +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_cluster_ops_by_policy.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_cluster_ops_by_policy.cc new file mode 100644 index 0000000000000000000000000000000000000000..9ba284d923d332c0343fee8915da3a068410d6d1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_cluster_ops_by_policy.cc @@ -0,0 +1,91 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h" + +namespace mlir { +namespace tf_test { +namespace { + +using mlir::TFDevice::ClusteringPolicy; +using mlir::TFDevice::ClusteringPolicySet; +using mlir::TFDevice::ValueConstraint; +using mlir::TFDevice::ValuesConstraintSet; + +#define GEN_PASS_DEF_TESTCLUSTERINGPOLICYPASS +#define GEN_PASS_DECL_TESTCLUSTERINGPOLICYPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +struct TestClusteringPolicyPass + : public impl::TestClusteringPolicyPassBase { + void runOnOperation() override; +}; + +// Clustering policy for `test.OpA` and `test.OpB` operations; +class TestOpsClusteringPolicy : public ClusteringPolicy { + LogicalResult MatchAndUpdateConstraints( + Operation* op, const ValuesConstraintSet& results, + ValuesConstraintSet& operands) const final { + // Check if operation is `test.OpA` or `test.OpB`. + bool is_op_a = op->getName().getStringRef() == "test.OpA"; + bool is_op_b = op->getName().getStringRef() == "test.OpB"; + if (!is_op_a && !is_op_b) return failure(); + + if (auto result_constraint = results.GetConstraint(op->getResult(0))) { + // `test.OpA` converts shape constraint to rank constraint. + if (is_op_a && *result_constraint == ValueConstraint::kShape) + operands.Insert(op->getOperand(0), ValueConstraint::kRank); + + // `test.OpB` converts value constraint to shape constraint. + if (*result_constraint == ValueConstraint::kValue) + operands.Insert(op->getOperand(0), ValueConstraint::kShape); + } + + return success(); + } +}; + +void TestClusteringPolicyPass::runOnOperation() { + func::FuncOp func = getOperation(); + ValuesConstraintSet constraints; + + ClusteringPolicySet policies; + policies.Add(); + + // Initialize constraints based on the return type attributes. + if (failed(InferFunctionBodyValuesConstraints(func, constraints))) + return signalPassFailure(); + + // Propagate constraints though the function body. + auto result = + PropagateValuesConstraints(func.getBody(), policies, constraints, + /*resolve=*/false, /*emit_remarks=*/true); + (void)result; + + // Emit remarks for all operations that use constrained values. + EmitValueConstraintsRemarks(constraints); +} + +} // anonymous namespace + +std::unique_ptr> CreateTestClusteringPolicyPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..49dad58c325576b9dd9fc609afdf59f29f2dbecd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h @@ -0,0 +1,79 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TEST_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TEST_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace tf_test { + +// Returns test pass for variable freezing. +std::unique_ptr> CreateFreezeVariableTestPass(); + +// Test pass for applying TF->TF lowering patterns. +std::unique_ptr> CreateTestTFLowerTFPass(); + +// Test passes for visitor util. +std::unique_ptr> CreateTestVisitorUtilPass(); +std::unique_ptr> +CreateTestVisitorUtilInterruptPass(); + +// Test operation clustering based on user defined policy. +std::unique_ptr> CreateTestClusteringPolicyPass(); + +// Test pass for analyzing side-effect analysis result. +std::unique_ptr> CreateTestSideEffectAnalysisPass(); + +std::unique_ptr> CreateTestResourceAliasAnalysisPass(); + +std::unique_ptr> CreateInitTextFileToImportTestPass(); +std::unique_ptr> +CreateInitTextFileToImportSavedModelTestPass(); + +// Variable Lifting test passes: only useful for lit testing. +std::unique_ptr> CreateLiftVariablesTestPass(); +std::unique_ptr> +CreateLiftVariablesInvalidSessionTestPass(); + +// Create a test pass for the above with a "fake" session, for lit testing. +std::unique_ptr> +CreateInitializeVariablesInSessionInitializerTestPass(); + +// Create a test pass that emits remarks for each analysis result for resources. +// This pass is only used for lit testing. +std::unique_ptr> CreateResourceAnalyzerTestPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_FREEZEVARIABLESTESTPASS +#define GEN_PASS_DECL_INITTEXTFILETOIMPORTSAVEDMODELTESTPASS +#define GEN_PASS_DECL_INITTEXTFILETOIMPORTTESTPASS +#define GEN_PASS_DECL_INITIALIZEVARIABLESINSESSIONINITIALIZERPASS +#define GEN_PASS_DECL_LIFTVARIABLESINVALIDSESSIONTESTPASS +#define GEN_PASS_DECL_LIFTVARIABLESTESTPASS +#define GEN_PASS_DECL_RESOURCEANALYZERTESTPASS +#define GEN_PASS_DECL_TESTCLUSTERINGPOLICYPASS +#define GEN_PASS_DECL_TESTRESOURCEALIASANALYSIS +#define GEN_PASS_DECL_TESTSIDEEFFECTANALYSISPASS +#define GEN_PASS_DECL_TESTTENSORFLOWLOWERTFPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +} // namespace tf_test +} // namespace mlir +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TEST_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc new file mode 100644 index 0000000000000000000000000000000000000000..372dc1313330dc6c10f7562b62882bb961e2ba82 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc @@ -0,0 +1,123 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace tf_test { +namespace { + +// A pass that annotates each operation with a resource type result with the +// aliasing values for each such result. Each value is assigned a unique ID, and +// that ID is used to annotate the operations. +struct TestResourceAliasAnalysis + : public TF::PerFunctionAggregateAnalysisConsumerPass< + TestResourceAliasAnalysis, TF::ResourceAliasAnalysis> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestResourceAliasAnalysis) + + StringRef getArgument() const final { + return "tf-test-resource-alias-analysis"; + } + + StringRef getDescription() const final { + return "Add remarks based on resource alias analysis result, for testing " + "purpose."; + } + + void runOnFunction(func::FuncOp func, + const TF::ResourceAliasAnalysis::Info& analysis) { + int64_t next_id = 0; + llvm::SmallDenseMap ids; + + auto assign_id = [&](Value value) { + if (ids.find(value) == ids.end()) ids.insert({value, next_id++}); + }; + + auto get_id = [&](Value value) -> int64_t { + auto it = ids.find(value); + assert(it != ids.end()); + return it->second; + }; + + auto print_aliases = [&](InFlightDiagnostic& diag, Value value) { + diag << ", ID " << get_id(value) << " : "; + if (analysis.IsUnknownResource(value)) { + diag << "Unknown"; + } else { + auto aliases = llvm::to_vector<4>(analysis.GetResourceAliases(value)); + llvm::sort(aliases, + [&](Value v1, Value v2) { return get_id(v1) < get_id(v2); }); + llvm::interleaveComma(aliases, diag, + [&](Value v) { diag << get_id(v); }); + } + }; + + // Assign a unique ID to each value seen in this function. + func.walk([&](Operation* op) { + // For all attached regions, assign ID to the region arguments. + for (Region& region : op->getRegions()) { + for (auto region_arg : TF::filter_resources(region.getArguments())) + assign_id(region_arg); + } + + // Assign ID for all results. + for (auto result : TF::filter_resources(op->getResults())) + assign_id(result); + }); + + // Now walk each operation, and annotate it wil remarks for aliases for + // each resource type result + func.walk([&](Operation* op) { + // For all attached regions, assign ID to the region arguments. + for (Region& region : op->getRegions()) { + for (auto region_arg : TF::filter_resources(region.getArguments())) { + InFlightDiagnostic diag = op->emitRemark("Region #") + << region.getRegionNumber() << ", Arg #" + << region_arg.getArgNumber(); + print_aliases(diag, region_arg); + } + } + + for (auto result : TF::filter_resources(op->getResults())) { + InFlightDiagnostic diag = op->emitRemark("Result #") + << result.getResultNumber(); + print_aliases(diag, result); + } + }); + } +}; + +} // anonymous namespace + +std::unique_ptr> CreateTestResourceAliasAnalysisPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_side_effect_analysis.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_side_effect_analysis.cc new file mode 100644 index 0000000000000000000000000000000000000000..652421a9c5586518ebeef056f55736ce0ce14afb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/test_side_effect_analysis.cc @@ -0,0 +1,88 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +namespace mlir { +namespace tf_test { + +namespace { + +struct TestSideEffectAnalysisPass + : public TF::PerFunctionAggregateAnalysisConsumerPass< + TestSideEffectAnalysisPass, TF::SideEffectAnalysis> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSideEffectAnalysisPass) + + void runOnFunction(func::FuncOp func, + const TF::SideEffectAnalysis::Info& analysis) { + int64_t next_id = 0; + llvm::SmallDenseMap ids; + func.walk([&](Operation* op) { + ids[op] = next_id++; + op->emitRemark("ID: ") << ids[op]; + }); + auto join_ids = [&](const llvm::ArrayRef ops) { + llvm::SmallVector id_vec; + id_vec.reserve(ops.size()); + for (auto op : ops) id_vec.push_back(std::to_string(ids[op])); + return llvm::join(id_vec, ","); + }; + func.walk([&](Operation* op) { + if (!analysis.DirectControlPredecessors(op).empty()) { + op->emitRemark("Predecessors: ") + << "{" << join_ids(analysis.DirectControlPredecessors(op)) << "}"; + } + if (!analysis.DirectControlSuccessors(op).empty()) { + op->emitRemark("Successors: ") + << "{" << join_ids(analysis.DirectControlSuccessors(op)) << "}"; + } + if (llvm::isa(op)) { + op->emitRemark("Sinks: ") + << "{" << join_ids(analysis.ControlSinks()) << "}"; + } + }); + } + + StringRef getArgument() const final { return "tf-test-side-effect-analysis"; } + StringRef getDescription() const final { + return "Test pass for analyzing side-effect analysis result"; + } +}; + +} // anonymous namespace + +std::unique_ptr> CreateTestSideEffectAnalysisPass() { + return std::make_unique(); +} + +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.cc new file mode 100644 index 0000000000000000000000000000000000000000..d08240775c06f108ffd280070de62d3e4de39c5b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.cc @@ -0,0 +1,70 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.h" + +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TF { + +namespace { + +struct FuseParallelMapAndBatch : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BatchDatasetV2Op op, + PatternRewriter &rewriter) const override { + auto batchInputDataset = op.getInputDataset(); + + ParallelMapDatasetOp batchInputOp = dyn_cast_or_null( + batchInputDataset.getDefiningOp()); + if (!batchInputOp) return failure(); + + // The type of the `num_parallel_calls` argument in ParallelMapDataset + // and MapAndBatchDataset is different (int32 and int64 respectively) + auto num_parallel_calls_op = CastOp::create( + rewriter, op.getLoc(), + UnrankedTensorType::get(rewriter.getIntegerType(64)), + batchInputOp.getNumParallelCalls(), rewriter.getBoolAttr(false)); + + if (op.getMetadata() != batchInputOp.getMetadata()) { + return failure(); + } + + auto fused_op = MapAndBatchDatasetOp::create( + rewriter, op.getLoc(), op.getType(), batchInputOp.getInputDataset(), + batchInputOp.getOtherArguments(), op.getBatchSize(), + num_parallel_calls_op.getY(), op.getDropRemainder(), + batchInputOp.getF(), op.getOutputTypes(), op.getOutputShapes(), + batchInputOp.getPreserveCardinality(), op.getMetadata()); + rewriter.replaceOp(op, {fused_op.getHandle()}); + return failure(); + } +}; + +#include "tensorflow/compiler/mlir/tensorflow/transforms/generated_tf_data_optimization.inc" +} // namespace + +void PopulateTFDataOptimizationPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + patterns->add(context); + populateWithGenerated(*patterns); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.h new file mode 100644 index 0000000000000000000000000000000000000000..d0b3d528c00629dfc688403ad41e69b84cf1796f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.h @@ -0,0 +1,32 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_DATA_OPTIMIZATION_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_DATA_OPTIMIZATION_H_ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Populates patterns to perform optimizations specific to tf.data operations. +void PopulateTFDataOptimizationPatterns(MLIRContext *context, + RewritePatternSet *patterns); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_DATA_OPTIMIZATION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.td new file mode 100644 index 0000000000000000000000000000000000000000..7d14ff6ccfe152a7f4bdc4f38d68ffa91521ca26 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.td @@ -0,0 +1,36 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/IR/OpBase.td" +include "mlir/IR/PatternBase.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" + +// TODO(jpienaar): Move this somewhere general. +class GetI64ScalarElementsAttr : + NativeCodeCall<"DenseElementsAttr::get(RankedTensorType::get({}, $_builder.getIntegerType(64)), " # value # ")">; + +def FuseMapAndBatch : Pat< + (TF_BatchDatasetV2Op + (TF_MapDatasetOp $input_dataset, $other_arguments, $f, $output_types, + $output_shapes, $use_inter_op_parallelism, $preserve_cardinality, + $force_synchronous, $map_dataset_metadata), + $batch_size, $drop_remainder, $parallel_copy, $batch_output_types, + $batch_output_shapes, $unused_batch_dataset_metadata), + (TF_MapAndBatchDatasetOp $input_dataset, $other_arguments, $batch_size, + (TF_ConstOp (GetI64ScalarElementsAttr<1>)), $drop_remainder, $f, + $batch_output_types, $batch_output_shapes, $preserve_cardinality, + // TODO(kramm): Should we merge batch_dataset_metadata and + // map_dataset_metadata? + $map_dataset_metadata)>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..adc0856c2cbdb13be48e3539f4432c1cd7a43517 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization_pass.cc @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_data_optimization.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_TFDATAOPTIMIZATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Perform tf.data optimizations. +struct TFDataOptimization + : public impl::TFDataOptimizationPassBase { + void runOnOperation() override { + RewritePatternSet patterns(&getContext()); + mlir::TF::PopulateTFDataOptimizationPatterns(&getContext(), &patterns); + + (void)applyPatternsGreedily(getOperation(), std::move(patterns)); + } +}; + +} // namespace + +std::unique_ptr> CreateTFDataOptimizationPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_assignment.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_assignment.cc new file mode 100644 index 0000000000000000000000000000000000000000..811f8d577e6d502efed35150be8901988b76e972 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_assignment.cc @@ -0,0 +1,117 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements device assignment in TF dialect. +#include +#include + +#include "mlir/IR/Builders.h" +#include "mlir/Pass/Pass.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TF { +namespace { + +constexpr char kDeviceAttr[] = "device"; +constexpr char kTFDeviceAttr[] = "tf.device"; + +#define GEN_PASS_DEF_SIMPLETFDEVICEASSIGNMENTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class SimpleTFDeviceAssignmentPass + : public impl::SimpleTFDeviceAssignmentPassBase< + SimpleTFDeviceAssignmentPass> { + public: + SimpleTFDeviceAssignmentPass() = default; + SimpleTFDeviceAssignmentPass(const SimpleTFDeviceAssignmentPass&) {} + explicit SimpleTFDeviceAssignmentPass(llvm::StringRef default_device) { + default_device_ = std::string(default_device); + } + + void runOnOperation() override { + Builder builder(&getContext()); + Dialect* tf = getContext().getLoadedDialect(); + getOperation().walk([&](Operation* op) { + if (auto device_attr = op->getAttrOfType(kDeviceAttr)) { + // We assign default device to ops with device attribute that is empty. + if (device_attr.getValue().empty()) { + op->setAttr(kDeviceAttr, builder.getStringAttr(default_device_)); + } + } else if (op->getDialect() == tf) { + // Assign default device to all ops in Tensorflow dialect that do not + // have device attribute. + op->setAttr(kDeviceAttr, builder.getStringAttr(default_device_)); + } + }); + } +}; + +#define GEN_PASS_DEF_TFDEVICEASSIGNMENTBYFUNCATTRPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass to perform device assignment for TF dialect ops that do not +// have device assignment, by using the device attribute of the function. +// If device attribute is not found from the function, nothing is done. +class TFDeviceAssignmentByFuncAttrPass + : public impl::TFDeviceAssignmentByFuncAttrPassBase< + TFDeviceAssignmentByFuncAttrPass> { + public: + TFDeviceAssignmentByFuncAttrPass() = default; + TFDeviceAssignmentByFuncAttrPass(const TFDeviceAssignmentByFuncAttrPass&) {} + + void runOnOperation() override { + func::FuncOp func = getOperation(); + auto func_device_attr = func->getAttrOfType(kTFDeviceAttr); + + // Skip device assignment if there is no device specified in the function + // attribute. + if (!func_device_attr || func_device_attr.getValue().empty()) { + return; + } + + Builder builder(&getContext()); + Dialect* tf = getContext().getLoadedDialect(); + getOperation().walk([&](Operation* op) { + if (auto device_attr = op->getAttrOfType(kDeviceAttr)) { + // Assign device to ops with device attribute that is empty. + if (device_attr.getValue().empty()) { + op->setAttr(kDeviceAttr, + builder.getStringAttr(func_device_attr.getValue())); + } + } else if (op->getDialect() == tf) { + // Assign device to all ops in Tensorflow dialect that do not have + // device attribute. + op->setAttr(kDeviceAttr, + builder.getStringAttr(func_device_attr.getValue())); + } + }); + } +}; +} // namespace + +std::unique_ptr> CreateSimpleTFDeviceAssignmentPass( + llvm::StringRef default_device) { + return std::make_unique(default_device); +} + +std::unique_ptr> +CreateTFDeviceAssignmentByFuncAttrPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..66eb77ef069133621ef41eea446efed5c82f9741 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.td @@ -0,0 +1,332 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +// TF device dialect passes. + +def ResourceOpLiftingPass : Pass<"tf-resource-op-lifting", "ModuleOp"> { + let summary = "Lifting resource operations out of device computation"; + let description = [{ + This pass lifts resource variable operations outside of device computation. + This is useful because a lot of accelerator devices can not interact with + resource variables directly.. + + Here is a simple example in TensorFlow where a device doubles the value of a + TensorFlow resource variable and returns new value: + + ```mlir + %resource_handle = "tf.VarHandleOp"() + %1 = "tf_device.cluster"() ( { + %init_value = "tf.ReadVariableOp"(%resource_handle) + "tf.AssignAddVariableOp"(%resource_handle, %init_value) + %new_value = "tf.ReadVariableOp"(%resource_handle) + tf_device.return %new_value + }) + ``` + + After this pass, the computation would become: + + ```mlir + %resource_handle = "tf.VarHandleOp"() + %init_value = "tf.ReadVariableOp"(%resource_handle) + %1:2 = "tf_device.cluster"() ( { + %new_value = "tf.AddV2"(%init_value, %init_value) + tf_device.return %new_value, %new_value + }) + "tf.AssignVariableOp"(%resource_handle, %1#1) + ``` + + You can see that there are a few main changes applied: + 1) All the resource variable reads and writes are now outside of + tf_device.cluster op. + 2) Instead of taking resource handles as input, this device computation now + takes snapshotted values of that device. + 3) Some resource load operations are eliminated with store-load forwarding. + 4) Updated values to resource are appended to `tf_device.return` and used by + external resource store operations so that resources are still updated + after the computation. + + If the cluster body contains functional control flow, the pass first lifts + the loads/stores in the body/cond/branch functions to the cluster body, then + performs the above lifting. E.g., + + ```mlir + func @cluster_with_loop() -> () { + %0 = "tf.VarHandleOp"() ... + "tf_device.cluster"() ( { + %1 = "tf.While"(%0) {body = @while_body, cond = @while_cond} + tf_device.return + }) + return + } + func @while_body(%arg0: tensor<*x!tf_type.resource>>) { + %constant = "tf.Const"() ... + "tf.AssignVariableOp"(%arg0, %constant) + return %arg0 + } + func @while_cond(%arg0: tensor<*x!tf_type.resource>>) { + %read = "tf.ReadVariableOp"(%arg0) + return %read + } + ``` + + will be transformed to: + + ```mlir + func @cluster_with_loop() { + %0 = "tf.VarHandleOp"() ... + %1 = "tf.ReadVariableOp"(%0) + %2 = "tf_device.cluster"() ( { + %3 = "tf.While"(%1) {body = @while_body, cond = @while_cond} + tf_device.return %3 : tensor + }) : () -> tensor + "tf.AssignVariableOp"(%0, %2) + return + } + func @while_body(%arg0: tensor) { + %0 = "tf.Const"() ... + return %0 : tensor + } + func @while_cond(%arg0: tensor) { + return %arg0 + } + ``` + }]; + + let constructor = "TFDevice::CreateResourceOpLiftingPass()"; +} + +def ResourceOpLiftingForMainFunctionPass : + Pass<"tf-resource-op-lifting-for-main-function", "ModuleOp"> { + let summary = "Lifting resource operations out of control flow statements " + "for the main function"; + let constructor = "TFDevice::CreateResourceOpLiftingForMainFunctionPass()"; +} + +def AnnotateParameterReplicationPass : + Pass<"tf-annotate-parameter-replication", "ModuleOp"> { + let summary = "Annotate whether a ClusterFuncOp's parameters have the same data across replicas."; + let constructor = "TFDevice::CreateAnnotateParameterReplicationPass()"; +} + +def DeviceAttributeToLaunchPass : Pass<"tf-device-attribute-to-launch", "mlir::func::FuncOp"> { + let summary = "Wraps each TF op which has a non-empty device attribute in a tf_device.launch."; + + let description = [{ + This pass wraps TF ops which have a non-empty device attribute in a tf_device.lauch with + the same device attribute. + + For example, the following: + + ```mlir + func @single_op_launch() { + %a = "tf.opA"() {device = "CPU:0"} : () -> tensor + return %a + } + ``` + + will be transformed into: + + ```mlir + func @single_op_launch() { + %1 = tf_device.launch() ( { + %a = "tf.opA"() : () -> tensor + tf_device.return %a + }) {device = "CPU:0"} : () -> tensor + return %1 + } + ``` + }]; + + let constructor = "TFDevice::CreateDeviceAttributeToLaunchPass()"; +} + +def VerifyNoOutsideCompilationMarkersPass : Pass<"verify-no-outside-compilation-markers", "mlir::func::FuncOp"> { + let summary = "Verifies that after Outside Compilation passes complete, there are no more _xla_outside_compilation attributes and no tf_device.launch ops."; + + let description = [{ + This pass takes iterates over ops and ensures that _xla_outside_compilation + and tf_device.launch ops no longer exist and have been removed. This happens + after Outside Compilation passes have completed. + }]; + + let constructor = "TFDevice::CreateVerifyNoOutsideCompilationMarkersPass()"; +} + +def DecomposeResourceOpsPass : Pass<"tf-device-decompose-resource-ops", "mlir::func::FuncOp"> { + let summary = "Decompose composite resource variable operations into primitive Read/AssignVariableOp and raw computation."; + + let description = [{ + A pass that decomposes composite resource operations into primitive ones like + ReadVariableOp, AssignVariableOp and other computations to facilitate + transformations like resource op lifting. + + For example: + + ```mlir + tf.AssignAddVariableOp(%res, %0) + ``` + + Becomes + + ```mlir + %res_val = tf.ReadVariableOp(%res) + %1 = tf.AddV2(%res_val, %0) + tf.AssignVariableOp(%res, %1) + ``` + + NOTE: This pass does not support `use_locking=true` for a lot of resource + operations. So decomposition may not be correct outside of backends like XLA, + which automatically locks all resource variables. + }]; + + let constructor = "TFDevice::CreateDecomposeResourceOpsPass()"; +} + +def DecomposeResourceOpsInClusterPass : Pass<"tf-device-decompose-resource-ops-in-cluster", "ModuleOp"> { + let summary = "Decompose composite resource variable operations into primitive Read/AssignVariableOp and raw computation within device cluster and reachable functions."; + + let description = [{ + This is a variation of DecomposeResourceOps pass. It operates on a module and + only decomposes ops within a device cluster (tf_device.cluster op) and any + functions reachable from the cluster. + }]; + + let constructor = "TFDevice::CreateDecomposeResourceOpsInClusterPass()"; +} + +def LaunchToDeviceAttributePass : Pass<"tf-launch-to-device-attribute", "mlir::func::FuncOp"> { + let summary = "Hoists and annotates device launch inner ops with associated device attribute."; + + let description = [{ + This pass hoists a `tf_device.launch` body and assigns a `device` attribute + to each TensorFlow dialect op in the body based on the `device` attribute on + the `tf_device.launch`. If a TensorFlow dialect op already has a device + attribute, that attribute will be overwritten with the `tf_device.launch` + device. + + For example: + + ```mlir + %island:5 = tf_executor.island { + %a = "tf.opA"() : () -> tensor + %launch:2 = "tf_device.launch"() ( { + %b = "tf.opB"() : () -> tensor + %c = "tf.opC"() : () -> tensor + tf_device.return %c, %b : tensor, tensor + }) {device = "CPU:0"} : () -> (tensor, tensor) + %d = "tf.opD"() : () -> tensor + tf_executor.yield %a, %launch#0, %launch#1, %d : + tensor, tensor, tensor, tensor + } + ``` + + Will be transformed into: + + ```mlir + %island:5 = tf_executor.island { + %a = "tf.opA"() : () -> tensor + %b = "tf.opB"() {device = "CPU:0"} : () -> tensor + %c = "tf.opC"() {device = "CPU:0"} : () -> tensor + %d = "tf.opD"() : () -> tensor + tf_executor.yield %a, %c, %b, %d : + tensor, tensor, tensor, tensor + } + ``` + }]; + + let options = [ + Option<"legacy_graph_export_", "legacy-graph-export", "bool", + /*default=*/"true", + "Determines whether or not this pass should execute logic that is " + "reserved for the legacy graph export pipeline to maintain expected " + "invariants. In the case of this pass, that means manually propagating " + "controls to lifted parallel execute regions to the graph fetch to " + "ensure the ops execute, as well as determining whether or not the " + "islands created by this pass should be split after the replicated " + "ops have been lifted."> + ]; + + let constructor = "TFDevice::CreateLaunchToDeviceAttributePass()"; +} + +def XlaInlineDeviceOpsPass : Pass<"tf-xla-inline-device-ops", "ModuleOp"> { + let summary = "Inline all Cluster op based in the parent region"; + let constructor = "TFDevice::CreateXlaInlineDeviceOpsPass()"; + let dependentDialects = ["tf_device::TensorFlowDeviceDialect"]; +} + +def XlaRewritePass : Pass<"tf-xla-rewrite", "mlir::ModuleOp"> { + let summary = "Rewrites partition calls into Xla launch ops to make the attached function run on XLA."; + + let description = [{ + This pass rewrites `tf.PartitionedCall` and `tf.StatefulPartitionedCall` + operations with `_xla_compile_device_type` attribute in a + `tf_device.cluster` into `tf.XlaLaunch` operations. This makes the attached + function execute with XLA. `tf.XlaLaunch` requires resource-type arguments + come at the end, so this pass rewrites the called function if necessary. + This pass assumes there are no nested `tf_device.cluster`s so we don't end + up creating nested XLA launch ops. + + For example, the `tf.PartitionedCall` operation in the following code + + ```mlir + func.func @convert_partitioned_call_with_resources(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.PartitionedCall"(%arg0, %arg1) {_xla_compile_device_type = "CPU", f = @pcall_func_with_resources} : (tensor, tensor) -> (tensor) + %0 = "tf_device.cluster"() ({ + %1 = "tf.StatefulPartitionedCall"() {_xla_compile_device_type = "CPU", f = @stateful_pcall_func} : () -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + + func.return %0 : tensor + } + + func.func @pcall_func_with_resources(%arg0 : tensor, %arg1: tensor) -> tensor { + func.return %arg1 : tensor + } + ``` + + will be replaced by a `tf.XlaLaunch` operation. + + ```mlir + func.func @convert_partitioned_call_with_resources(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf_device.cluster"() ({ + %1 = "tf.XlaLaunch"(%arg1, %arg0) {_xla_compile_device_type = "CPU", function = @pcall_func_with_resources_0, operand_segment_sizes = array} : (tensor, tensor) -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + + return %0 : tensor + } + + func.func @pcall_func_with_resources_0(%arg0: tensor, %arg1: tensor) -> tensor { + return %arg0 : tensor + } + ``` + Notice that the called function is rewritten, with the order of its parameters changed. + }]; + + let constructor = "TFDevice::CreateXlaRewritePass()"; + let dependentDialects = ["tf_device::TensorFlowDeviceDialect"]; +} + +def XlaValidateInputsPass : Pass<"tf-xla-validate-inputs", "ModuleOp"> { + let summary = "Validtes inputs to the TF CPU/GPU bridge"; + let description = [{ + This pass checks that the IR has valid input to CPU/GPU TF/XLA bridge. + }]; + let constructor = "TFDevice::CreateXlaValidateInputsPass()"; + let dependentDialects = ["tf_device::TensorFlowDeviceDialect"]; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_executor_to_functional.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_executor_to_functional.cc new file mode 100644 index 0000000000000000000000000000000000000000..d265c429147ca32c8b6c3bc1e8d86b8b0659d05a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_executor_to_functional.cc @@ -0,0 +1,91 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" + +namespace mlir { + +namespace { + +#define GEN_PASS_DEF_EXECUTORDIALECTTOFUNCTIONALPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct ExecutorDialectToFunctionalConversion + : public impl::ExecutorDialectToFunctionalPassBase< + ExecutorDialectToFunctionalConversion> { + void runOnOperation() override; +}; + +// Extracts inner ops of tf_executor.island ops in a tf_executor.graph, in the +// order of ops in tf_executor.graph. +LogicalResult LiftIslandOpInnerOpsFromGraph(tf_executor::GraphOp graph) { + auto graph_position = graph.getOperation()->getIterator(); + Block* parent_block = graph.getOperation()->getBlock(); + for (Operation& op : graph.GetBody().without_terminator()) { + auto island_op = llvm::dyn_cast(op); + if (!island_op) + return op.emitOpError() + << "is not supported for lifting out of tf_executor.graph, " + "expected tf_executor.island"; + + // Move inner ops in island to before the outer graph. + auto& island_body = island_op.GetBody().getOperations(); + parent_block->getOperations().splice(graph_position, island_body, + island_body.begin(), + std::prev(island_body.end())); + // Forward island fetches (tf_executor.yield operands) to island op result + // uses. + for (auto result : + llvm::zip(island_op.getOutputs(), island_op.GetYield().getFetches())) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + } + + // Forward graph fetches (tf_executor.fetch operands) to graph op result uses. + for (auto result : + llvm::zip(graph.getResults(), graph.GetFetch().getFetches())) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + graph.erase(); + return success(); +} + +void ExecutorDialectToFunctionalConversion::runOnOperation() { + auto result = getOperation().walk([](tf_executor::GraphOp graph) { + if (failed(LiftIslandOpInnerOpsFromGraph(graph))) + return WalkResult::interrupt(); + + return WalkResult::advance(); + }); + if (result.wasInterrupted()) signalPassFailure(); +} +} // end anonymous namespace + +std::unique_ptr> +CreateExecutorDialectToFunctionalConversionPass() { + return std::make_unique(); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_functional_to_executor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_functional_to_executor.cc new file mode 100644 index 0000000000000000000000000000000000000000..92b802fc478243057054aa0b8ca5d97949fef473 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_functional_to_executor.cc @@ -0,0 +1,113 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" + +#define DEBUG_TYPE "tf-functional-to-executor" + +namespace mlir { + +namespace { +// This pass converts mlir functions consisting of mlir ops into a tf_executor +// dialect as a single island. +// Result like so: +// func @my_fn(%argi...) -> (result_t) { +// %results:[[n_args]] = tf_executor.graph { +// %island_results:[[nargs + 1]] = tf_executor.island { +// ... original ops ... +// tf_executor.yield %results... +// } +// tf_executor.fetch %island_results#... +// } +// return %graph_results#... +// } + +#define GEN_PASS_DEF_FUNCTIONALTOEXECUTORDIALECTCONVERSIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct FunctionalToExecutorDialectConversion + : public impl::FunctionalToExecutorDialectConversionPassBase< + FunctionalToExecutorDialectConversion> { + void runOnOperation() override; +}; +} // end anonymous namespace + +void FunctionalToExecutorDialectConversion::runOnOperation() { + auto func = getOperation(); + if (func.isExternal()) return; + if (!llvm::hasSingleElement(func)) { + LLVM_DEBUG(llvm::dbgs() << "Expect single block function, skip conversion " + "to tf_executor dialect\n"); + return; + } + auto loc = func.getLoc(); + mlir::Block& body = func.front(); + // Find region of interest and ReturnOp. + auto copy_range = body.without_terminator(); + if (copy_range.begin() != copy_range.end() && + std::next(copy_range.begin()) == copy_range.end() && + isa(*copy_range.begin())) { + // Already a graph. + return; + } + + auto return_op = dyn_cast(body.getTerminator()); + if (!return_op) { + LLVM_DEBUG(llvm::dbgs() << "Expect function to end with return\n"); + return; + } + // Build GraphOp. + OpBuilder builder(&body, body.begin()); + auto graph_op = tf_executor::GraphOp::create( + builder, loc, func.getFunctionType().getResults()); + graph_op.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&graph_op.GetBody()); + auto island = tf_executor::IslandOp::create( + builder, loc, func.getFunctionType().getResults(), + tf_executor::ControlType::get(&getContext()), ArrayRef()); + // Create Fetch. + ValueRange to_fetch = island.getResults(); + if (to_fetch.size() != 1) { + // Drop control result for fetch. + to_fetch = to_fetch.drop_back(); + } + tf_executor::FetchOp::create(builder, loc, to_fetch); + // Build Island. + island.getBody().push_back(new Block); + island.getBody().front().getOperations().splice( + island.getBody().front().begin(), body.getOperations(), + copy_range.begin(), copy_range.end()); + builder.setInsertionPointToEnd(&island.getBody().front()); + tf_executor::YieldOp::create(builder, loc, return_op.getOperands()); + for (auto item : llvm::enumerate(graph_op.getResults())) { + return_op.setOperand(item.index(), item.value()); + } +} + +std::unique_ptr> +CreateFunctionalToExecutorDialectConversionPass() { + return std::make_unique(); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..9a946f463665f002092a976a449b0db949951e16 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc @@ -0,0 +1,215 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/common_runtime/optimization_registry.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/public/session_options.h" + +#define DEBUG_TYPE "run-tf-graph-optimization" + +namespace tensorflow { +namespace { +// Creates a pass to convert MLIR to Graph, run user-specified Graph +// Optimization Passes and convert back to MLIR. +// Constraints: This pass expects that all operations in the MLIR module either +// belong to 'tf' or '_tf' dialect. The output is in '_tf' dialect. +class GraphOptPass + : public mlir::PassWrapper> { + void getDependentDialects(mlir::DialectRegistry& registry) const override { + mlir::RegisterAllTensorFlowDialects(registry); + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(GraphOptPass) + + explicit GraphOptPass(std::vector passes) + : passes_(std::move(passes)) {} + + protected: + // For MLIR LIT tests only. + GraphOptPass() = default; + + void runOnOperation() override; + + // The passes to run on the module. + std::vector passes_; +}; +} // anonymous namespace + +void GraphOptPass::runOnOperation() { + mlir::ModuleOp module_in = getOperation(); + mlir::MLIRContext& ctx = getContext(); + + // Convert MLIR to Graph + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphExportConfig confs; + auto graph = std::make_unique(flib_def); + absl::flat_hash_set control_ret_nodes; + absl::Status status = tensorflow::tf2xla::v2::ConvertTfExecutorToGraph( + module_in, confs, &graph, &flib_def, &control_ret_nodes); + if (!status.ok()) { + mlir::emitError(mlir::UnknownLoc::get(&ctx)) << status.message(); + return signalPassFailure(); + } + + // Run each of the passes that were selected. + GraphConstructorOptions opts; + opts.allow_internal_ops = true; + opts.expect_device_spec = false; + + GraphOptimizationPassOptions options; + SessionOptions sess_options; + options.graph = &graph; + options.flib_def = &flib_def; + options.session_options = &sess_options; + + for (auto pass : passes_) { + assert(pass != nullptr); + absl::Status status = pass->Run(options); + if (!status.ok()) { + mlir::emitError(mlir::UnknownLoc::get(&ctx)) + << pass->name() << ": " << status.message(); + return signalPassFailure(); + } + } + + // Convert Graph to MLIR + GraphDebugInfo debug_info; + GraphImportConfig specs; + auto module_or_status = tensorflow::tf2xla::v2::ConvertGraphToTfExecutor( + **options.graph, debug_info, flib_def, specs, &ctx); + if (!module_or_status.ok()) { + mlir::emitError(mlir::UnknownLoc::get(&ctx)) + << module_or_status.status().message(); + return signalPassFailure(); + } + auto module_out = std::move(module_or_status).value(); + + // We cannot replace the module in a ModulePass. So we simply copy the + // operation list from module_out to module_in. + auto& module_in_ops = module_in.getBody()->getOperations(); + module_in_ops.clear(); + module_in_ops.splice(module_in_ops.end(), + module_out->getBody()->getOperations()); +} + +// Returns a vector of passes from their names. If a pass is not found, then the +// corresponding return entry is null. +static std::vector FindRegisteredPassesByName( + const std::vector& pass_names) { + std::vector pass_ids(pass_names.size(), nullptr); + + for (const auto& group : OptimizationPassRegistry::Global()->groups()) { + for (const auto& phase : group.second) { + for (const auto& pass : phase.second) { + // Iterate over the pass_names_ and insert the pass pointer at all the + // corresponding indices in the pass_ids vector. + auto iter = pass_names.begin(); + while ((iter = std::find(iter, pass_names.end(), pass->name())) != + pass_names.end()) { + pass_ids[std::distance(pass_names.begin(), iter)] = pass.get(); + iter++; + } + } + } + } + return pass_ids; +} + +class GraphOptByNamePass : public GraphOptPass { + public: + // For MLIR LIT tests only. + GraphOptByNamePass() = default; + + explicit GraphOptByNamePass(const std::vector& pass_names) { + passes_ = FindRegisteredPassesByName(pass_names); + } + + llvm::StringRef getArgument() const final { + return "run-tf-graph-optimization"; + } + + llvm::StringRef getDescription() const final { + return "runs passes registered as tensorflow::GraphOptimizationPass"; + } + + protected: + mlir::Pass::ListOption cl_pass_list_{ + *this, "graph-passes", + llvm::cl::desc("comma separated list of GraphOptimizationPass to run.")}; + + private: + void runOnOperation() override { + if (!cl_pass_list_.empty()) { + passes_ = FindRegisteredPassesByName(cl_pass_list_); + + // Verify all passes requested were registered/found. + for (auto pass_it : llvm::enumerate(passes_)) { + if (pass_it.value() == nullptr) { + mlir::emitError(mlir::UnknownLoc::get(&getContext())) + << "could not find pass " << cl_pass_list_[pass_it.index()]; + return signalPassFailure(); + } + } + } + return GraphOptPass::runOnOperation(); + } +}; + +} // namespace tensorflow + +std::unique_ptr> +tensorflow::CreateTensorFlowGraphOptimizationPass( + std::vector tf_passes) { + return std::make_unique(std::move(tf_passes)); +} + +std::unique_ptr> +tensorflow::CreateTensorFlowGraphOptimizationPass( + const std::vector& pass_names) { + return std::make_unique(pass_names); +} + +void tensorflow::RegisterGraphOptimizationPasses() { + ::mlir::registerPass([]() -> std::unique_ptr<::mlir::Pass> { + return std::make_unique(); + }); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..a0536e0e61f6e063348c8eb3a810cb0bb9c22847 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.h @@ -0,0 +1,49 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_GRAPH_OPTIMIZATION_PASS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_GRAPH_OPTIMIZATION_PASS_H_ + +#include +#include +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/core/common_runtime/optimization_registry.h" + +namespace tensorflow { + +// Create a module pass that will execute the given TF GraphOptimization passes +// in sequence. +// Pass requires that the module ran on is convertible to TF Graph. +std::unique_ptr> +CreateTensorFlowGraphOptimizationPass( + std::vector tf_passes); + +// Same as above but pass names instead of the passes provided. The registered +// passes are queried, if a TF graph optimization pass is not found in registry +// then the pass fails. +// Pass requires that the module ran on is convertible to TF Graph. +std::unique_ptr> +CreateTensorFlowGraphOptimizationPass( + const std::vector& pass_names); + +// Register the pass for command line testing. +void RegisterGraphOptimizationPasses(); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_GRAPH_OPTIMIZATION_PASS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..7eab6cfc0f36707c70c262a8cfd1595f8ed0f315 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.td @@ -0,0 +1,2341 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +// TF dialect passes. + +def TensorflowGPUFusion : Pass<"tf-gpu-op-fusion", "mlir::func::FuncOp"> { + let summary = "Fusion optimization for GPU targets"; + let description = [{ + This pass is performing fusion specific to GPU targets. This is an ad-hoc + pass for now, but should be integrated with some notion of "target" in the + MLIR pipeline in the future. + }]; + let constructor = "TF::CreateGpuOpFusionPass()"; +} + +def BatchMatMulToEinsumPass : Pass<"tf-batch-matmul-to-tf-einsum", "mlir::func::FuncOp"> { + let summary = "Replace TF BatchMatMul op by TF Einsum op."; + let constructor = "TF::CreateBatchMatMulToEinsumPass()"; +} + +def CanonicalizeCompileAndReplicateAttributesPass : Pass<"tf-canonicalize-compile-and-replicate-attributes", "mlir::func::FuncOp"> { + let summary = "Canonicalize compilation and replication attributes."; + + let description = [{ + A pass that converts existing compilation and replication attributes into + unified attributes. For example, `_tpu_replicate="cluster"` in the + following code + + ```mlir + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], name = "TPUReplicateMetadata", num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_tpu = true, use_spmd_for_xla_partitioning = false} : () -> () + ``` + + wll be replaced by `_replication_info="cluster"` and `_xla_compile_device_type="TPU"`. + + ```mlir + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_replication_info = "cluster", _xla_compile_device_type = "TPU", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], name = "TPUReplicateMetadata", num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + ``` + + `_XlaMustCompile=true` in the following code + + ```mlir + %outputs_67, %control_68 = tf_executor.island wraps "tf.PartitionedCall"(%arg0, %outputs_0) {_XlaMustCompile = true, _collective_manager_ids = [], _read_only_resource_inputs = [], config = "", config_proto = "\0A\07\0A\03CPU\10\01\0A\07\0A\03GPU\10\00\0A\07\0A\03TPU\10\02\0A\0E\0A\0ATPU_SYSTEM\10\012\02J\008\01\82\01\05h\01\88\01\01", device = "", executor_type = "", f = @__inference__jit_compiled_convolution_op_1510} : (tensor<4x32x32x8xf32>, tensor<*xf32>) -> tensor<*xf32> + ``` + + will be replaced by `_xla_compile_device_type`, with its value set to the value of `device`. + + ```mlir + %outputs_67, %control_68 = tf_executor.island wraps "tf.PartitionedCall"(%arg0, %outputs_0) {_collective_manager_ids = [], _read_only_resource_inputs = [], _xla_compile_device_type = "", config = "", config_proto = "\0A\07\0A\03CPU\10\01\0A\07\0A\03GPU\10\00\0A\07\0A\03TPU\10\02\0A\0E\0A\0ATPU_SYSTEM\10\012\02J\008\01\82\01\05h\01\88\01\01", device = "", executor_type = "", f = @__inference__jit_compiled_convolution_op_1510} : (tensor<4x32x32x8xf32>, tensor<*xf32>) -> tensor<*xf32> + ``` + }]; + + let constructor = "TF::CreateCanonicalizeCompileAndReplicateAttributesPass()"; +} + +def ClusterTFOpsByHostPass : Pass<"cluster-tf-ops-by-host", "ModuleOp"> { + let summary = "Cluster the TensorFlow ops by host so that each function only " + "contains ops placed on the same host"; + + let constructor = "TF::CreateClusterTFOpsByHostPass()"; +} + +def ConvertToLegacyCompileAndReplicateAttributesPass : Pass<"tf-convert-to-legacy-compile-and-replicate-attributes", "mlir::func::FuncOp"> { + let summary = "Convert unified compilation and replication attributes back to legacy attributes."; + + let description = [{ + This transformation pass converts unified compilation and replication + attributes (`_replication_info` and `_xla_compile_device_type`) into legacy + attributes. This ensures the unified attributes do not get exposed outside + of the MLIR bridge with V1 pipeline in some cases. The pass expects to have + either none or both of the unified attributes present in an op for the + conversion to happen. Otherwise it will fail. + + For example, `_replication_info="cluster"` and + `_xla_compile_device_type="TPU"` in the following code + + ```mlir + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_replication_info = "cluster", _xla_compile_device_type = "TPU", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], name = "TPUReplicateMetadata", num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + ``` + + wll be replaced by `_tpu_replicate="cluster"` as follows, + + + ```mlir + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], name = "TPUReplicateMetadata", num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_tpu = true, use_spmd_for_xla_partitioning = false} : () -> () + ``` + }]; + + let constructor = "TFTPU::CreateConvertToLegacyCompileAndReplicateAttributesPass()"; +} + +def ResourceDeviceInferencePass : Pass<"tf-resource-device-inference", "ModuleOp"> { + let summary = "Propagates the device attribute on resources from callers to " + "callees."; + let constructor = "TF::CreateResourceDeviceInferencePass()"; + let description = [{ + A pass that propagates device assignment of resources on a module. It + performs in-function propagation, as well as cross-function propagation from + callers to callees. + + This pass changes the module by adding "tf.device" attribute to function + arguments and adding "device" attribute to TF ops. + + For example, given the function + + ```mlir + !tf_res = type tensor<*x!tf_type.resource>> + + func @test(%arg0: !tf_res {tf.device = "/TPU:0"}) { + tf_executor.graph { + %control = tf_executor.island { + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + return + } + ``` + + Observe how the op inside the island obtains a `/TPU:0` device assignment: + + ```mlir + !tf_res = type tensor<*x!tf_type.resource>> + + func @test(%arg0: !tf_res {tf.device = "/TPU:0"}) { + tf_executor.graph { + %control = tf_executor.island { + %0 = "tf.Identity"(%arg0) {device = "/TPU:0"} : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + return + } + ``` + }]; +} + +def StackOpsDecompositionPass : Pass<"tf-stack-ops-decomposition", "ModuleOp"> { + let summary = "Decompose stack operations into local variable operations. Needs " + "static shapes."; + let constructor = "TF::CreateStackOpsDecompositionPass()"; + let description = [{ + A pass that converts stack operations to tensor operations and read/assign + ops on local variables. A later resource lifting pass can further remove the + local variables. + + This pass requires that the full shape of the stack can be inferred: 1) the + maximum size needs to be a constant and 2) a push op can be found with a + known shape, and all push ops need to have the same shape. + + A stack creation op "tf.StackV2" will be turned in to two zero-initialized + variables, for the buffer and current size. Each push will be turned into + ```mlir + %old_val = "tf.ReadVariableOp"(%buffer) + %old_size = "tf.ReadVariableOp"(%size) + %offsets = "tf.ConcatV2"(%old_size, %other_dims_0s, %const0) + %new_val = "tf.XlaDynamicUpdateSlice"(%old_val, %push_val, %offsets) + "tf.AssignVariableOp"(%buffer, %new_val) + %new_size = "tf.AddV2"(%old_size, %const1) + "tf.AssignVariableOp"(%size, %new_size) + ``` + + and each pop will be turned into + + ```mlir + %old_val = "tf.ReadVariableOp"(%buffer) + %old_size = "tf.ReadVariableOp"(%size) + %new_size = "tf.Sub"(%old_size, %const1) + %offsets = "tf.ConcatV2"(%old_size, %other_dims_0s, %const0) + %slice = "tf.Slice"(%old_val, %offsets, %slice_size_const) + %pop_result = "tf.Reshape"(%slice, %elem_size_const) + "tf.AssignVariableOp"(%size, %new_size) + ``` + + The pass also works across control flow and functional calls. + }]; +} + + + +def UnrollBatchMatMulPass : Pass<"tf-unroll-batch-matmul", "mlir::func::FuncOp"> { + let summary = "Unroll TF BatchMatMul op into Reshape, Slice, MatMul, Pack ops."; + let constructor = "TF::CreateUnrollBatchMatMulPassPass()"; +} + +def ClusterFormationPass : Pass<"tf-device-cluster-formation", "mlir::ModuleOp"> { + let summary = "Form clusters from instructions assigned to same device"; + let constructor = "TFDevice::CreateClusterFormationPass()"; + let dependentDialects = ["tf_device::TensorFlowDeviceDialect"]; + let description = [{ + Clusters operations with the same device assignment id. For each + cluster, creates a "tf_device.device_launch" op with a Region containing the + ops in each cluster and replaces the ops with the new launch op. + + For example, given the following program: + + ```mlir + %2 = "tf.A"(%arg0) : (tensor) -> tensor + %3 = "tf.B"(%2) {device = "tpu0"} : (tensor) -> tensor + %4 = "tf.C"(%2, %3) {device = "tpu0"} : (tensor, tensor) -> tensor + %5 = "tf.D"(%4) : (tensor) -> tensor + ``` + + After the pass, we will have: + + ```mlir + %0 = "tf.A"(%arg0) : (tensor) -> tensor + %1 = "tf_device.launch"() ( { + %3 = "tf.B"(%0) : (tensor) -> tensor + %4 = "tf.C"(%0, %3) : (tensor, tensor) -> tensor + tf_device.return %4 : tensor + }) {device = "tpu0"} : () -> tensor + %2 = "tf.D"(%1) : (tensor) -> tensor + return %2 : tensor + ``` + }]; +} + +def MaterializePassthroughOp : Pass<"tf-materialize-passthrough-op", "mlir::func::FuncOp"> { + let summary = "Materialize the MlirPassthroughOp by replacing it with the " + "MLIR module attached as an attribute"; + let constructor = "TF::CreateMaterializePassthroughOpPass()"; + let description = [{ + A pass that replaces MlirPassthrough ops with the code they have in + their `mlir_module` string attribute. + }]; +} + +def TransformEinsumPass : Pass<"tf-einsum", "mlir::func::FuncOp"> { + let summary = "Transform Einsum to other TF Ops for the supported variants"; + let constructor = "TF::CreateTransformEinsumPass()"; +} + +def LegalizeTFGToTFPass : Pass<"tfe-legalize-tfg", "ModuleOp"> { + let summary = "Legalize from TFG to the TFE dialect"; + let constructor = "TF::CreateLegalizeTFGToTFEPass()"; +} + +def ReplicateToIslandPass : Pass<"tf-replicate-to-island", "mlir::func::FuncOp"> { + let summary = "Lowers device replicate to executor islands"; + let constructor = "TFDevice::CreateReplicateToIslandPass()"; + let options = [ + Option<"legacy_graph_export_", "legacy-graph-export", "bool", + /*default=*/"true", + "Determines whether or not this pass should execute logic that is " + "reserved for the legacy graph export pipeline to maintain expected " + "invariants. In the case of this pass, that means manually propagating " + "controls to lifted parallel execute regions to the graph fetch to " + "ensure the ops execute, as well as determining whether or not the " + "islands created by this pass should be split after the replicated " + "ops have been lifted."> + ]; +} + +def ReplicaIDToDeviceOrdinalPass : Pass<"tf-replica-id-to-device-ordinal", "mlir::func::FuncOp"> { + let summary = "Set device ordinal with replica id"; + let constructor = "TFDevice::CreateReplicaIDToDeviceOrdinalPass()"; + let description = [{ + This pass sets the device ordinal attribute of the ops using the replica id + attribute. This is run immediately after the replica_to_island pass which + sets the replica id attribute of these ops. Note for single chip usecase, + the pass will check if there is one op and sets the device ordinal attribute + to be zero. + }]; +} + +def ConvertReadonlyReferenceVariablesToResourceVariablesPass : + Pass<"tf-readonly-references-to-resources", "mlir::func::FuncOp"> { + let summary = "Convert readonly reference variables to resource variables."; + let constructor = "TF::CreateConvertReadonlyReferenceVariablesToResourceVariablesPass()"; +} + +def ReplicateTensorListInitOpsPass : Pass<"tf-replicate-tensor-list-init-ops", "mlir::func::FuncOp"> { + let summary = + "Replicate TensorList init ops for correct shape assignments in shape inference"; + + let description = [{ + If we pass same TensorList to a while op as multiple arguments or just use + the same TensorList at multiple places and assign different + TensorListSetItem to elements of TensorList, the shape inference is then + unable to identify the Shape of these args and thus the input TensorList + shape is unidentifiable. + All of these args are supposed to be independent and not related to original + creation of TensorList. + + This pass will create multiple instances of TensorList for each arg of the + while op and each use and thus there will be not a conflict in resolving the + shape of these different inputs. + }]; + + let constructor = "TF::CreateReplicateTensorListInitOpsPass()"; +} + +def TensorFlowShapeInferencePass : Pass<"tf-shape-inference", "ModuleOp"> { + let summary = + "Shape inference on TF dialect and ops implementing InferTypeOpInterface"; + + let description = [{ + Fixed point shape refinement pass that utilizes the shape functions + registered on ops using the InferTypeOpInterface as well as by bridging to + the TensorFlow op registry's shape functions. This is an interprocedural + pass that propagates information across function calls/control flow + operations where possible (the GuaranteeAllFuncsOneUsePass is often run + before this pass to enable more propagation opportunities). It refines + both the outermost element type of tensors as well as the nested component + type (e.g., for tensor lists). + + During shape refinement this pass may insert additional cast operations as + well as fold some constant shape computations to enable more exact shape + inference. Therefore it does do some mutation of the graph. Constant folding + required to produce more exact shapes is also performed but these values + are only kept in the context rather than the ops folded/IR mutated. + }]; + + let constructor = "TF::CreateTFShapeInferencePass()"; + + let options = [ + Option<"max_iterations_", "max-iterations", "int64_t", /*default=*/"10", + "Maximum shape inference iterations">, + Option<"input_arg_shapes_", "input-arg-shapes", "std::string", /*default=*/"", + "Input tensor shapes. Shapes for different tensors are separated by ':', and dimension sizes for the same tensor are separated by ','">, + Option<"enable_stablehlo_propagation_", "enable-stablehlo-propagation", "bool", /*default=*/"false", + "Enables refined shape propagation to the StableHLO module inside XlaCallModule ops.">, + ]; +} + +def StripNoinlineAttributePass : Pass<"tf-strip-noinline-attribute", "ModuleOp"> { + let summary = "Strip the tf._noinline attribute from top-level functions."; + let constructor = "TF::CreateStripNoinlineAttributePass()"; +} + +def ExecutorConvertControlToDataOutputsPass : Pass<"tf-executor-convert-control-to-data-outputs", "ModuleOp"> { + let summary = "Chain control outputs of while loop body"; + + let description = [{ + This pass converts the control outputs of a while loop body function to data + outputs. Thus, inter iteration control dependencies are transformed to + data dependencies. Since data dependencies can express which particular + operations in the while loop body are dependent on which inputs, it captures + inter iteration parallelism in while loop. Control dependencies on the other + hand create a barrier at the end of while loop body thus blocking any + parallelism across iterations. + + For example, the following while loop body has a `%barrier` at the end. + Although there is no data/control dependency between `tf.AssignVariableOp` + for `%arg0` to `tf.AssignVariableOp` for `%arg1` across any iteration, the + while loop body has a control barrier (`%barrier`) at the end which forces + a dependency and the two assign variable ops must wait for each other to + complete before starting the next iteration. Transforming these control + outputs to data outputs removes the dependency between the two assign + variable ops, thus allowing them to run in parallel across iterations. + + Before: + + ```mlir + !tf_res = type tensor>> + func @while_body(%arg0: !tf_res, %arg1: !tf_res, %arg2: tensor, %arg3: tensor) -> (!tf_res, !tf_res, tensor, tensor) { + %graph:4 = tf_executor.graph { + %assign_0_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) : (!tf_res, tensor) -> () + %assign_1_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg1, %arg3) : (!tf_res, tensor) -> () + %add_out, %add_control = tf_executor.island wraps "tf.Add"(%arg2, %arg3) : (tensor, tensor) -> tensor + %mul_out, %mul_control = tf_executor.island wraps "tf.Mul"(%arg2, %arg3) : (tensor, tensor) -> tensor + %barrier = tf_executor.island(%assign_0_control, %assign_1_control, %add_control, %mul_control) wraps "tf.NoOp"() : () -> () + tf_executor.fetch %arg0, %arg1, %add_out, %mul_out, %barrier : !tf_res, !tf_res, tensor, tensor, !tf_executor.control + } + return %graph#0, %graph#1, %graph#2, %graph#3 : !tf_res, !tf_res, tensor, tensor + } + ``` + + After: + + ```mlir + func @while_body(%arg0: !tf_res, %arg1: !tf_res, %arg2: tensor, %arg3: tensor, %chain_0: tensor, %chain_1: tensor) -> (!tf_res, !tf_res, tensor, tensor, tensor, tensor) { + %graph:6 = tf_executor.graph { + %_, %chain_0_src = tf_executor.island wraps "tf.Identity"(%chain_0) : (tensor) -> tensor + %_, %chain_1_src = tf_executor.island wraps "tf.Identity"(%chain_1) : (tensor) -> tensor + %assign_0_control = tf_executor.island(%chain_0_src) wraps "tf.AssignVariableOp"(%arg0, %arg2) : (!tf_res, tensor) -> () + %assign_1_control = tf_executor.island(%chain_1_src) wraps "tf.AssignVariableOp"(%arg1, %arg3) : (!tf_res, tensor) -> () + %add_out, %add_control = tf_executor.island wraps "tf.Add"(%arg2, %arg3) : (tensor, tensor) -> tensor + %mul_out, %mul_control = tf_executor.island wraps "tf.Mul"(%arg2, %arg3) : (tensor, tensor) -> tensor + %chain_0_sink, %_ = tf_executor.island(%assign_0_control) wraps "tf.Identity"(%chain_0) : (tensor) -> tensor + %chain_1_sink, %_ = tf_executor.island(%assign_1_control) wraps "tf.Identity"(%chain_1) : (tensor) -> tensor + tf_executor.fetch %arg0, %arg1, %add_out, %mul_out, %chain_0_sink, %chain_1_sink : !tf_res, !tf_res, tensor, tensor, tensor, tensor + } + return %graph#0, %graph#1, %graph#2, %graph#3, %graph#4, %graph#5 : !tf_res, !tf_res, tensor, tensor, tensor, tensor + } + ``` + }]; + + let constructor = "tf_executor::CreateTFExecutorConvertControlToDataOutputsPass()"; + + let options = [ + Option<"composite_tpuexecute_side_effects_", "composite-tpuexecute-side-effects", "bool", + /*default=*/"false", + "Enables certain TPUExecute ops to run in parallel if they only " + "operate on resources that live on composite devices."> + ]; +} + +def ExecutorUpdateControlDependenciesPass : Pass<"tf-executor-update-control-dependencies", "ModuleOp"> { + let summary = "Computes and applies all necessary control dependencies based on side effect analysis."; + let description = [{ + This pass is intended to run after the split_into_island_per_op + pass. That pass splits up multi-op islands into multiple individual islands + wrapping a single op without applying any control deps between the new + islands. So, this pass is needed in order to make preservation of the + semantic ordering relationships between ops as determined by side effect + analysis explicit in the IR. + + Example: original program: + + ```mlir + func.func @example(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + %read0, %read0_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %assign0_control = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read1, %read1_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %print, %print_control = tf_executor.island wraps "tf.Print"(%read1) { message = "read1 value" } : (tensor<32xf32>) -> (tensor<32xf32>) + tf_executor.fetch %read1#0 : tensor<32xf32> + } + func.return %graph : tensor<32xf32> + } + ``` + + will be converted by this pass into: + + ```mlir + func.func @example(%arg0: tensor<*x!tf_type.resource>>, %arg1: tensor<32xf32>) -> tensor<32xf32> { + %0 = tf_executor.graph { + %read0, %read0_control = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %assign0_control = tf_executor.island(%read0_control) wraps "tf.AssignVariableOp"(%arg0, %arg1) : (tensor<*x!tf_type.resource>>, tensor<32xf32>) -> () + %read1, %read1_control = tf_executor.island(%assign0_control) wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor<32xf32> + %print, %print_control = tf_executor.island(%read1_control) wraps "tf.Print"(%read1) {message = "read1 value"} : (tensor<32xf32>) -> tensor<32xf32> + tf_executor.fetch %read1, %print_control : tensor<32xf32>, !tf_executor.control + } + return %0 : tensor<32xf32> + } + ``` + }]; + let constructor = "tf_executor::CreateTFExecutorUpdateControlDependenciesPass()"; +} + +def ExecutorCheckControlDependenciesPass : Pass<"tf-executor-check-control-dependencies", "ModuleOp"> { + let summary = "Checks control dependencies"; + + let description = [{ + This pass analyzes control dependencies between islands and warns about + dependencies that are not explainable by side effects of the involved ops. + More precisely, for every minimal unexplainable control dependency path + we emit op warnings for all involved ops. The pass does not report + intermediate dummy ops for grouping control dependencies (Identity, NoOp), + unless they are part of an unexplainable path between other ops. + This pass is useful to understand control dependency conservatism for a + given MLIR module. + + For example, the following function + ```mlir + func.func @path_with_intermediate_ops( + %arg0: tensor>>, + %arg1: tensor>>, + %arg2: tensor) -> () { + tf_executor.graph { + %island1 = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) : (tensor>>, tensor) -> () + %island2 = tf_executor.island(%island1) wraps "tf.NoOp"() : () -> () + %island3 = tf_executor.island(%island2) wraps "tf.NoOp"() : () -> () + %island4 = tf_executor.island(%island3) wraps "tf.AssignVariableOp"(%arg1, %arg2) : (tensor>>, tensor) -> () + tf_executor.fetch + } + func.return + } + ``` + produces the following warnings + ```mlir + 6:45: warning: unexpected control dependency path: path 0, node 0 (source) + %island1 = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) : (tensor>>, tensor) -> () + ^ + 6:45: note: see current operation: %control = tf_executor.island wraps "tf.AssignVariableOp"(%arg0, %arg2) : (tensor>>, tensor) -> () + 7:55: warning: unexpected control dependency path: path 0, node 1 (intermediate) + %island2 = tf_executor.island(%island1) wraps "tf.NoOp"() : () -> () + ^ + 7:55: note: see current operation: %control_0 = tf_executor.island(%control) wraps "tf.NoOp"() : () -> () + 8:55: warning: unexpected control dependency path: path 0, node 2 (intermediate) + %island3 = tf_executor.island(%island2) wraps "tf.NoOp"() : () -> () + ^ + 8:55: note: see current operation: %control_1 = tf_executor.island(%control_0) wraps "tf.NoOp"() : () -> () + 9:55: warning: unexpected control dependency path: path 0, node 3 (target) + %island4 = tf_executor.island(%island3) wraps "tf.AssignVariableOp"(%arg1, %arg2) : (tensor>>, tensor) -> () + ^ + 9:55: note: see current operation: %control_2 = tf_executor.island(%control_1) wraps "tf.AssignVariableOp"(%arg1, %arg2) : (tensor>>, tensor) -> () + ``` + because the first and last `AssignVariableOp`s access different resources + and therefore should be independent. Note that the `NoOp`s are considered + as intermediate ops for control dependency grouping. + }]; + + let constructor = "tf_executor::CreateTFExecutorCheckControlDependenciesPass()"; +} + +def ExecutorGraphPruningPass : Pass<"tf-executor-graph-pruning", "mlir::func::FuncOp"> { + let summary = "Prunes unreachable ops in a tf_executor.graph"; + + let description = [{ + This pass removes ops from a `tf_executor.graph` that are not transitively, via + data or control dependencies, connected to the associated `tf_executor.fetch` + op. The order of ops will be preserved. Functions named `main` with no + `tf.entry_function` attribute will not be pruned, as such graphs/functions may + have been imported from a V1 TensorFlow graph, where feeds/fetches/targets are + not provided at certain stages of IR transformation (e.g. pre-placement). + + Option `ops-to-preserve` allows to specify ops that should not be pruned, + regardless of their reachability. + + For example, the following: + + ```mlir + func @graph(%arg0: tensor, %arg1: tensor) -> tensor { + %graph = tf_executor.graph { + %transitive_reachable_data:2 = tf_executor.island wraps "tf.Identity"(%arg0) : (tensor) -> tensor + %reachable_data:2 = tf_executor.island wraps "tf.Identity"(%transitive_reachable_data#0) : (tensor) -> tensor + %unreachable_data:2 = tf_executor.island wraps "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %transitive_reachable_control = tf_executor.island wraps "tf.NoOp"() : () -> () + %reachable_control = tf_executor.island(%transitive_reachable_control) wraps "tf.NoOp"() : () -> () + %unreachable_control = tf_executor.island wraps "tf.NoOp"() : () -> tensor + tf_executor.fetch %reachable_data#0, %reachable_control : tensor, !tf_executor.control + } + return %graph : tensor + } + ``` + + will be transformed into: + + ```mlir + func @graph(%arg0: tensor, %arg1: tensor) -> tensor { + %graph = tf_executor.graph { + %transitive_reachable_data:2 = tf_executor.island wraps "tf.Identity"(%arg0) : (tensor) -> tensor + %reachable_data:2 = tf_executor.island wraps "tf.Identity"(%transitive_reachable_data#0) : (tensor) -> tensor + %transitive_reachable_control = tf_executor.island wraps "tf.NoOp"() : () -> () + %reachable_control = tf_executor.island(%transitive_reachable_control) wraps "tf.NoOp"() : () -> () + tf_executor.fetch %reachable_data#0, %reachable_control : tensor, !tf_executor.control + } + return %graph : tensor + } + ``` + }]; + + let constructor = "tf_executor::CreateTFExecutorGraphPruningPass()"; + + let options = [ + ListOption<"ops_to_preserve_", "ops-to-preserve", "std::string", + "Comma separated list of ops that should not be pruned " + "regardless of reachability"> + ]; +} + +def ExecutorDialectToFunctionalPass : Pass<"tf-executor-to-functional-conversion", "mlir::func::FuncOp"> { + let summary = "Lifts tf_executor.island inner ops from a tf_executor.graph"; + + let description = [{ + This pass converts tf_executor.graphs consisting of only tf_executor.islands and + a tf_executor.fetch into a sea of nodes consisting of TensorFlow Dialect ops by + lifting such ops out of a tf_executor.graph's tf_executor.islands. If V1 control + flow ops are present in a tf_executor.graph, an error will be returned. + + For example, the following: + + ```mlir + func @my_fn(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %graph_results:2 = tf_executor.graph { + %island_0_result, %island_0_control = tf_executor.island { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + tf_executor.yield %identity : tensor + } + %island_1_result, %island_1_control = tf_executor.island { + %identity_n:2 = "tf.IdentityN"(%arg1, %island_0_result) : (tensor, tensor) -> (tensor, tensor) + tf_executor.yield %identity_n#0 + } + tf_executor.fetch %island_0_result, %island_1_result : tensor, tensor + } + return %graph_results#0, %graph_results#1 : tensor, tensor + } + ``` + + will be transformed into: + + ```mlir + func @my_fn(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + %identity_n:2 = "tf.IdentityN"(%arg1, %identity) : (tensor, tensor) -> (tensor, tensor) + return %identity, %identity_n#0 : tensor, tensor + } + ``` + }]; + + let constructor = "CreateExecutorDialectToFunctionalConversionPass()"; +} + +def ExecutorIslandCoarseningPass : Pass<"tf-executor-island-coarsening", "mlir::func::FuncOp"> { + let summary = "Walks tf_executor::GraphOp and merges individual tf_executor::IslandOps."; + let description = [{ + This pass performs whole graph analysis for a graph encapsulated into tf_executor::GraphOp. + The analysis identifies all IslandOps within the graph which could be merged together. + The goal is to merge as many islands as possible. + Once analysis is completed, the pass merges all IslandOps in a single scan. + + For example given the following program with two disjunct islands: + + ```mlir + func @test(%arg0 : tensor) -> tensor { + %0 = tf_executor.graph { + %1:2 = tf_executor.island { + %3 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %3 : tensor + } + %2:2 = tf_executor.island(%1#1) { + %4 = "tf.opB"() : () -> tensor + tf_executor.yield %4 : tensor + } + tf_executor.fetch %2#0 : tensor + } + return %0 : tensor + } + ``` + + After running this pass, the two islands are merged: + + ```mlir + func @test(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island { + %1 = "tf.opA"(%arg0) : (tensor) -> tensor + %2 = "tf.opB"() : () -> tensor + tf_executor.yield %2 : tensor + } + tf_executor.fetch %outputs : tensor + } + return %0 : tensor + } + ``` + }]; + let constructor = "tf_executor::CreateTFExecutorIslandCoarseningPass()"; +} + +def TpuV1BridgeExecutorIslandCoarseningPass : Pass<"tf-executor-tpu-v1-island-coarsening", "ModuleOp"> { + let summary = "Merges TPU clusters IslandOps, intended for V1 compatibility mode"; + let constructor = "tf_executor::CreateTFExecutorTPUV1IslandCoarseningPass()"; + let description = [{ + This pass is a variant of ExecutorIslandCoarseningPass that is limited to + TPU-annotated operations and intended to preserve backward compatibility with + TFv1. + }]; +} + +def TPUBridgeExecutorIslandOutliningPass : Pass<"tf-executor-tpu-v1-island-outlining", "ModuleOp"> { + let summary = "Outline TPU clusters from island into a nested module, so it can " + "be processed like a V2 module, intended for V1 compatibility mode"; + let constructor = "tf_executor::CreateTFExecutorTPUV1IslandOutliningPass()"; + let description = [{ + Extract the islands containing a TPU cluster computation into an outlined + function in a nested module. This will allow to run the usual bridge on this + nested module which now exhibits a more friendly "V2-like" structure. + This is only intended for V1 compatibility mode where the bridge runs without + feed/fetches on session create/extend. + + So given e.g. + + ```mlir + func @test() -> tensor { + %0 = tf_executor.graph { + %output, %control = tf_executor.island { + ... + tf_executor.yield %result : tensor + } + tf_executor.fetch %output : tensor + } + return %0 + } + ``` + + This pass will create an additional function containing the code in + tf_executor.island: + + ```mlir + func nested @_tpu_v1_compat_outlined_func0() -> tensor { + ... + } + ``` + + and will then replace the island with the wrapped call: + + ```mlir + func @test() -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.PartitionedCall"() { + f = @_tpu_v1_compat_outlined::@_tpu_v1_compat_outlined_func0 + } : () -> tensor + tf_executor.fetch %outputs : tensor + } + return %0 : tensor + } + ``` + }]; +} + +def DecomposeReduceDatasetPass : Pass<"tf-decompose-reduce-dataset", "mlir::func::FuncOp"> { + let summary = "Decomposes ReduceDataset op into dataset operations."; + + let description = [{ + Decomposes ReduceDataset op into a while loop that iterates the dataset and calls + into the reduction function. This decomposition is only done if the + ReduceDataset op is marked for compilation with the _xla_compile_device_type + attribute. + + For example, for the following function the ReduceDataset op: + + ```mlir + func.func @single_state_single_dataset_type_no_arguments( + %arg0: tensor, + %arg1: tensor + ) { + %1 = "tf.ReduceDataset"(%arg0, %arg1) { + Targuments = [], + Tstate = [i64], device = "", + f = @__reduce_func_1, f._tf_data_function = true, + output_shapes = [#tf_type.shape<>], + output_types = [i64], use_inter_op_parallelism = true, _xla_compile_device_type="TPU"} : + (tensor, tensor) -> (tensor) + func.return + } + ``` + + with the following reduction function: + + ```mlir + func.func private @__reduce_func_1(%arg0: tensor {tf._user_specified_name = "args_0"}, + %arg1: tensor<32xf32> {tf._user_specified_name = "args_1"}) -> (tensor) + attributes {tf._tf_data_function = true, tf.signature.is_stateful} { + %0 = "tf.JustPretend"(%arg1) : (tensor<32xf32>) -> (tensor) + func.return %0 : tensor + } + ``` + + will be transformed into: + + ```mlir + func.func @single_state_single_dataset_type_no_arguments(%arg0: tensor, %arg1: tensor) { + %0 = "tf.AnonymousIteratorV3"() {output_shapes = [#tf_type.shape<32>], output_types = [f32]} : () -> tensor + "tf.MakeIterator"(%arg0, %0) : (tensor, tensor) -> () + %cst = "tf.Const"() {value = dense : tensor} : () -> tensor + %1:2 = "tf.WhileRegion"(%cst, %arg1) ({ + ^bb0(%arg2: tensor, %arg3: tensor): + "tf.Yield"(%arg2) : (tensor) -> () + }, { + ^bb0(%arg2: tensor, %arg3: tensor): + %2 = "tf.IteratorGetNextAsOptional"(%0) {output_shapes = [#tf_type.shape<32>], output_types = [f32]} : (tensor) -> tensor + %3 = "tf.OptionalHasValue"(%2) : (tensor) -> tensor + %4 = "tf.IfRegion"(%3) ({ + %5 = "tf.OptionalGetValue"(%2) : (tensor) -> tensor<32xf32> + %6 = func.call @__reduce_func_1(%arg3, %5) {_xla_compile_device_type = "TPU"} : (tensor, tensor<32xf32>) -> tensor + "tf.Yield"(%6) : (tensor) -> () + }, { + "tf.Yield"(%arg3) : (tensor) -> () + }) {_lower_using_switch_merge = true, is_stateless = false} : (tensor) -> tensor + "tf.Yield"(%3, %4) : (tensor, tensor) -> () + }) {_lower_using_switch_merge = true, is_stateless = false, parallel_iterations = 10 : i64} : (tensor, tensor) -> (tensor, tensor) + return + } + ``` + }]; + + let constructor = "TF::CreateDecomposeReduceDatasetPass()"; +} + +def ExecutorTPUV1IslandInliningPass : Pass<"tf-executor-tpu-v1-island-inlining", "ModuleOp"> { + let summary = "Inline calls to the nested TPU module."; + + let description = [{ + This pass inlines the islands calling into the nested module that was + outlined, thus reversing the effect of the + `-tf-executor-tpu-v1-island-outlining` pass. + + For example, the following: + ```mlir + module { + func @foo(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.PartitionedCall"(%arg0) {f = @_tpu_v1_compat_outlined::@bar} : (tensor) -> tensor + tf_executor.fetch %outputs : tensor + } + return %0 : tensor + } + module @_tpu_v1_compat_outlined { + func nested @bar(%arg0: tensor) -> tensor { + %0 = "tf.opA"(%arg0) : (tensor) -> tensor + return %0 : tensor + } + } + } + ``` + + will be transformed into: + + ```mlir + module { + func @foo(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island { + %1 = "tf.opA"(%arg0) : (tensor) -> tensor + tf_executor.yield %1 : tensor + } + tf_executor.fetch %outputs : tensor + } + return %0 : tensor + } + } + ``` + }]; + + let constructor = "tf_executor::CreateTFExecutorTPUV1IslandInliningPass()"; +} + +def TPUPartitionedOpConversionPass : Pass<"tf-tpu-partitioned-op-conversion", "mlir::func::FuncOp"> { + let summary = "Rewrite all TPU Partitioned ops into their V2 counterparts."; + + let constructor = "TFTPU::CreateTPUPartitionedOpConversionPass()"; +} + +def ClusterConstantSinkingPass : Pass<"tf-device-constant-sinking", "mlir::func::FuncOp"> { + let summary = "Sinks constants implicitly captured in a tf_device.cluster region."; + + let description = [{ + This pass sinks implicitly captured constants (`tf.Const` ops) used by and into + a `tf_device.cluster` region. Performing this prior to outlining will reduce the + number of arguments of the outlined function. + + For example, the following: + + ```mlir + func @cluster() -> tensor { + %const = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %cluster = "tf_device.cluster"() ( { + %identity = "tf.Identity"(%const) : (tensor) -> tensor + tf_device.return %identity : tensor + }) : () -> (tensor) + return %cluster : tensor + } + ``` + + will be transformed into: + + ```mlir + func @cluster() -> tensor { + %cluster = "tf_device.cluster"() ( { + %const = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %identity = "tf.Identity"(%const) : (tensor) -> tensor + tf_device.return %identity : tensor + }) : () -> (tensor) + return %cluster : tensor + } + ``` + }]; + + let constructor = "TFDevice::CreateClusterConstantSinkingPass()"; +} + +def ClusterOutliningPass : Pass<"tf-device-cluster-outlining", "ModuleOp"> { + let summary = "Outlines regions of tf_device.cluster operations"; + + let description = [{ + This pass outlines the body of a `tf_device.cluster` into a function and + replaces the `tf_device.cluster` op with an equivalent `tf_device.cluster_func` + op. Implicit operands will be captured and materialized as explicit arguments to + the newly created functions and associated `tf_device.cluster_func` ops. + + For example, the following: + + ```mlir + func @computation(%arg0: tensor) -> tensor { + %cluster = "tf_device.cluster"() ( { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + tf_device.return %identity : tensor + }) : () -> (tensor) + return %cluster : tensor + } + ``` + + will be transformed into: + + ```mlir + func @computation(%arg0: tensor) -> tensor { + %cluster = "tf_device.cluster_func"(%arg0) {func = @_func} : (tensor) -> tensor + return %cluster : tensor + } + + func @_func(%arg0: tensor) -> tensor { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + return %identity : tensor + } + ``` + }]; + + let constructor = "TFDevice::CreateClusterOutliningPass()"; +} + +def ConvertTfControlFlowToScfPass : Pass<"convert-tf-control-flow-to-scf", "ModuleOp"> { + let summary = "Convert TensorFlow control flow to SCF."; + + let description = [{ + This pass can be used for all direct control flow lowerings from the TensorFlow + dialect to the SCF dialect. + }]; + + let dependentDialects = ["scf::SCFDialect", "tensor::TensorDialect"]; + + let constructor = "TF::createConvertTfControlFlowToScfPass()"; +} + +def LaunchOutliningPass : Pass<"tf-device-launch-outlining", "ModuleOp"> { + let summary = "Outlines regions of tf_device.launch operations"; + + let description = [{ + This pass outlines the body of a `tf_device.launch` into a function and + replaces the `tf_device.launch` op with an equivalent `tf_device.launch_func` + op. Implicit operands will be captured and materialized as explicit arguments to + the newly created functions and associated `tf_device.launch_func` ops. The + `device` attribute from the `launch` op is transferred to `launch_func`. + + For example, the following: + + ```mlir + func @computation(%arg0: tensor) -> tensor { + %launch = "tf_device.launch"() ( { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + tf_device.return %identity : tensor + }) {device = "some_device"} : () -> (tensor) + return %launch : tensor + } + ``` + + will be transformed into: + + ```mlir + func @computation(%arg0: tensor) -> tensor { + %launch = "tf_device.launch_func"(%arg0) {device = "some_device", func = @_func} : (tensor) -> tensor + return %launch : tensor + } + + func @_func(%arg0: tensor) -> tensor { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + return %identity : tensor + } + ``` + }]; + + let constructor = "TFDevice::CreateLaunchOutliningPass()"; + + let options = [ + Option<"globally_unique_func_names_", "globally-unique-func-names", "bool", + /*default=*/"true", + "If true, the pass adds extra identifiers to make function names " + "globally unique within a process, not just within a module."> + ]; +} + +def ConvertLaunchFuncToTFCallPass : Pass<"tf-device-convert-launch-func-to-tf-call", "ModuleOp"> { + let summary = "Rewrites tf_device::LaunchFuncOp to TF::PartitionedCallOp"; + + let description = [{ + This pass converts tf_device::LaunchFuncOp into an equivalent + TF::PartitionedCallOp so that it can be exported to TensorFlow GraphDef. + }]; + + let constructor = "TFDevice::CreateConvertLaunchFuncToTFCallPass()"; +} + +def MarkInputOutputAliasesPass : Pass<"tf-device-mark-input-output-aliases", "ModuleOp"> { + let summary = "Marks device cluster inputs-output pairs that read/write to the same variable as aliases"; + + let description = [{ + This pass analyzes the inputs and outputs to device cluster and marks those + input-output pairs as aliases (using `tf.aliasing_output` attribute) which read + and write to the same resource. This aliasing information can then be propagated + to XLA compiler for input/output buffer space optimizations. + }]; + + let constructor = "TFDevice::CreateMarkInputOutputAliasesPass()"; +} + +def TPUResourceReadForWritePass : Pass<"tf-tpu-resource-read-for-write", "ModuleOp"> { + let summary = "Inserts tf.ReadVariableOp inputs to a TPU cluster for resource writes with no reads"; + + let description = [{ + This pass materializes `tf.ReadVariableOp` inputs to an outlined TPU computation + for resource variables where only writes are present so later in the pipeline + such resource variables can be fused with generated `tf.TPUExecute` ops, which + only supports resource variable read or read + write. For all TPU computations, + resource variables are required to be initialized prior to execution. Write only + resource variable uses can be generated currently via packed tensor uses. + + For example, the following: + + ```mlir + func @write_only_resource(%value: tensor, %resource: tensor<*x!tf_type.resource>>) { + %0 = "tf_device.cluster_func"(%value) {func = @cluster} : (tensor) -> tensor + "tf.AssignVariableOp"(%resource, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + return + } + + func @cluster(%arg0: tensor) -> tensor { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + return %identity : tensor + } + ``` + + will be transformed into: + + ```mlir + func @write_only_resource(%value: tensor, %resource: tensor<*x!tf_type.resource>>) { + %resource_read = "tf.ReadVariableOp"(%resource) : (tensor<*x!tf_type.resource>>) -> tensor + %0 = "tf_device.cluster_func"(%value, %resource_read) {func = @cluster} : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%resource, %0) : (tensor<*x!tf_type.resource>>, tensor) -> () + return + } + + func @cluster(%arg0: tensor, %arg1: tensor) -> tensor { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + return %identity : tensor + } + ``` + }]; + + let constructor = "TFTPU::CreateTPUResourceReadForWritePass()"; +} + +def HoistReplicateInvariantResourceWritesPass : Pass<"tf-hoist-replicate-invariant-resource-writes", "mlir::func::FuncOp"> { + let summary = "Hoists writes to replicate invariant resource variables."; + + let description = [{ + This pass hoists replicate invariant resource variable writes outside + tf_device.replicate op. These may have been inserted by other passes such as + resource op lifting. However, if the resource variable is not replicated, writes + to such variables for each replica are redundant and can be replaced by writing + a single value from first replica. + + The benefit of this optimization is reduced memory requirement on host. For + multiple writes (one from each replica) to such variables, the host would + allocate buffer space to receive the device output from all replicas, which is + not required. We can use the output of first replica in such cases. + }]; + + let constructor = "TF::CreateHoistReplicateInvariantResourceWritesPass()"; +} + +def FunctionalControlFlowToRegionsPass : Pass<"tf-functional-control-flow-to-regions", "ModuleOp"> { + let summary = "Transforms functional control flow operations to their region-based counterparts"; + + let description = [{ + This pass transforms functional control flow operations in the TensorFlow + dialect to their region-based counterparts, i.e., `tf.If` is transformed to + `tf.IfRegion` and `tf.While` is transformed to `tf.WhileRegion`. + + For example, this functional operation + + ```mlir + %0 = "tf.If"(%arg0, %arg1) { + then_branch = @then_branch_func, else_branch = @else_branch_func, is_stateless = false + } : (tensor, tensor<*xf32>) -> tensor<*xf32> + ``` + + will be transformed into this region-based operation + + ```mlir + %0 = "tf.IfRegion"(%arg0) ( { + %1 = call @then_branch_func(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %1 = call @else_branch_func(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<*xf32> + ``` + }]; + + let constructor = "TF::CreateTFFunctionalControlFlowToRegions()"; + + let options = [ + Option<"allow_passthrough_args_", "allow-passthrough-args", "bool", + /*default=*/"false", + "Allows to switch on passthrough args for control flow logic. This " + "allows the pass to create blocks that better match MLIR interfaces, " + "but might break assumptions in existing passes."> + ]; +} + +def RegionControlFlowToFunctionalPass : Pass<"tf-region-control-flow-to-functional", "ModuleOp"> { + let summary = "Transforms region-based control flow operations to their functional counterparts"; + + let description = [{ + This pass transforms region-based control flow operations in the TensorFlow + dialect to their functional counterparts, i.e., `tf.IfRegion` is transformed to + `tf.If` and `tf.WhileRegion` is transformed to `tf.While`. + + For example, this region-based operation + + ```mlir + %0 = "tf.IfRegion"(%arg0) ( { + %1 = call @then_branch_func(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }, { + %1 = call @else_branch_func(%arg1) : (tensor<*xf32>) -> tensor<*xf32> + "tf.Yield"(%1) : (tensor<*xf32>) -> () + }) {is_stateless = false} : (tensor) -> tensor<*xf32> + ``` + + will be transformed into this functional operation + + ```mlir + %0 = "tf.If"(%arg0, %arg1) { + then_branch = @then_branch_func, else_branch = @else_branch_func, is_stateless = false + } : (tensor, tensor<*xf32>) -> tensor<*xf32> + ``` + }]; + + let constructor = "TF::CreateTFRegionControlFlowToFunctional()"; +} + +def TPUReorderReplicateAndPartitionedInputsPass : Pass<"tf-tpu-reorder-replicate-partitioned-inputs", "mlir::func::FuncOp"> { + let summary = "Reorder replicated and partitioned input ops."; + + let description = [{ + This pass rewrites how data parallelism and model parallelism is expressed for + inputs. It reorders `tf.TPUPartitionedInput` (model parallelism) and + `tf.TPUReplicatedInput` (data parallelism) ops. It transforms a DAG where + multiple `tf.TPUPartitionedInput` ops are feeding into a single + `tf.TPUReplicatedInput` into a DAG where multiple `tf.TPUReplicatedInput` ops + are feeding into a single `tf.TPUPartitionedInput`. Transforming the IR in such + a manner will allow subsequent cluster formation pass to handle IR with both + data and model parallelism in an easier manner. + + For example, the following: + + ```mlir + !rtype = type tensor>> + func @data_and_model_parallelism(%arg0: !rtype, %arg1: !rtype, %arg2: !rtype, %arg3: !rtype) -> !rtype { + %pi_0 = "tf.TPUPartitionedInput"(%arg0, %arg1) {_XlaSharding = "", device = "", partition_dim = -1 : i64} : (!rtype, !rtype) -> !rtype + %pi_1 = "tf.TPUPartitionedInput"(%arg2, %arg3) {_XlaSharding = "", device = "", partition_dim = -1 : i64} : (!rtype, !rtype) -> !rtype + %ri = "tf.TPUReplicatedInput"(%pi_0, %pi_1) : (!rtype, !rtype) -> !rtype + return %ri : !rtype + } + ``` + + will be transformed into: + + ```mlir + !rtype = type tensor>> + func @data_and_model_parallelism(%arg0: !rtype, %arg1: !rtype, %arg2: !rtype, %arg3: !rtype) -> !rtype { + %ri_0 = "tf.TPUReplicatedInput"(%arg0, %arg2) : (!rtype, !rtype) -> !rtype + %ri_1 = "tf.TPUReplicatedInput"(%arg1, %arg3) : (!rtype, !rtype) -> !rtype + %pi = "tf.TPUPartitionedInput"(%ri_0, %ri_1) {_XlaSharding = "", device = "", partition_dim = -1 : i64} : (!rtype, !rtype) -> !rtype + return %pi : !rtype + } + ``` + }]; + + let constructor = "TFTPU::CreateTPUReorderReplicateAndPartitionedInputsPass()"; +} + +def TPUResourceReadsWritesPartitioningPass : Pass<"tf-tpu-resource-partition", "mlir::func::FuncOp"> { + let summary = "Partitions unpartitioned resource read/write to partitioned resource variables."; + + let description = [{ + This pass creates individual resource reads/writes from the unpartitioned + resource variable (from `tf.TPUPartitionedInput`) to individual partitioned + resource variables (`tf.TPUPartitionedInput` operands). As resource op + decomposition/lifting occurs with the unpartitioned resource variables, + transforming the IR in such a manner will allow for subsequent passes to operate + on individual resource variable handles per core/device. + + For example, the following: + + ```mlir + func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %partitioned_variable = "tf.TPUPartitionedInput"(%arg0, %arg1) {N = 2 : i64, _XlaSharding = "", partition_dim = -1 : i64} : (tensor>>, tensor>>) -> tensor>> + %read = "tf.ReadVariableOp"(%partitioned_variable) : (tensor>>) -> tensor + %computation = "tf_device.cluster_func"(%read) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + "tf.AssignVariableOp"(%partitioned_variable, %computation) : (tensor>>, tensor) -> () + return + } + + func @computation(%arg0: tensor) -> tensor { + return %arg0: tensor + } + ``` + + will be transformed into: + + ```mlir + func @cluster(%arg0: tensor>>, %arg1: tensor>>) { + %read0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %read1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %partitioned_input = "tf.TPUPartitionedInput"(%read0, %read1) {N = 2 : i64, _XlaSharding = "", partition_dim = -1 : i64} : (tensor, tensor) -> tensor + %computation = "tf_device.cluster_func"(%partitioned_input) {func = @computation, use_spmd_for_xla_partitioning = true} : (tensor) -> tensor + %partitioned_output:2 = "tf.TPUPartitionedOutput"(%computation) {N = 2 : i64, _XlaSharding = "", partition_dim = -1 : i64} : (tensor) -> (tensor, tensor) + "tf.AssignVariableOp"(%arg0, %partitioned_output#0) : (tensor>>, tensor) -> () + "tf.AssignVariableOp"(%arg1, %partitioned_output#1) : (tensor>>, tensor) -> () + return + } + + func @computation(%arg0: tensor) -> tensor { + return %arg0: tensor + } + ``` + }]; + + let constructor = "TFTPU::CreateTPUResourceReadsWritesPartitioningPass()"; +} + +def VerifySuitableForExportPass : Pass<"tf-verify-for-export", "ModuleOp"> { + let summary = "Verify module is suitable for export back to TF Graph"; + let description = [{ + Verifies whether all functions in module are of single tf_executor.graph and + each tf_executor.island in tf_executor.graph only has a single op. + }]; + + let constructor = "TF::CreateVerifySuitableForExportPass()"; +} + +def PrepareTpuComputationForTfExportPass : Pass<"prepare-tpu-computation-for-tf-export", "ModuleOp"> { + let summary = "Prepare TPU computation to be legal for export to TensorFlow"; + let description = [{ + Prepares TPU computation module attached to _TPUCompileMlir op for + TensorFlow graph export by making transformation such as replacing or + removing MLIR or XLA specific attributes that are not legal in TensorFlow + graph. + }]; + + let constructor = "TF::CreatePrepareTpuComputationForTfExportPass()"; +} + +def TensorDeviceCopyConversionPass : Pass<"tf-tensor-device-copy", "mlir::func::FuncOp"> { + let summary = "Fold the tf.Identity op and the tf.IdentityN op if the op has the same device as its operand"; + + let constructor = "TF::CreateTensorDeviceCopyConversionPass()"; +} + +def TensorArrayOpsDecompositionPass : Pass<"tf-tensor-array-ops-decomposition", "ModuleOp"> { + let summary = "Decompose tensor array operations into local variable operations."; + + let description = [{ + A pass that converts tensor array operations to tensor operations and + read/assign ops on local variables. A later resource lifting pass can further + remove the local variables. + + This pass requires that the full shape of the tensor array can be inferred: + 1) the size needs to be a constant, 2) it specifies the full element shape, + or that can be inferred from a later write, and 3) all elements have the same + shape. + }]; + let dependentDialects = ["tensor::TensorDialect"]; + + let constructor = "TF::CreateTensorArrayOpsDecompositionPass()"; +} + +def TensorFlowOptimizePass : Pass<"tf-optimize", "mlir::func::FuncOp"> { + let summary = "Optimize TensorFlow module"; + + let constructor = "TF::CreateTFOptimizePass()"; +} + +def PromoteResourcesToArgsPass : Pass<"tf-promote-resources-to-args", "ModuleOp"> { + let summary = "Promote resources reads/writes to function inputs/outputs."; + let description = [{ + This pass promotes resource accesses in function(s) (by default, the main) + to input arguments and outputs of the function(s). + + Two types of resources are supported: + (1) A function argument of TF::ResourceType type (this pass). + (2) A VarHandleOp in the function (tf-promote-var-handles-to-args). + + After the pass, + + . The function will have an input argument for each resource that is + already provided as an input argument or is read. The type of the input + argument will become the shape of the value represented by the resource. + + . The function will have an output for each resource that is written. The + type of the output will become the shape of the resource. + + The information of variable identification and input-output alising is + recorded as named attributes of the input argument or output: + + . 'tf.resource_name' matches 'shared_name' of VarHandleOp, which represents + the identifier of the corresponding resource. This attribute is added to + an input argument if the initial value of the resource is read, or to the + output if the initial value is not read. + + . 'tf.aliasing_output' is the index of the function output that is an alias + of the input argument. This attribute is added only to the input argument + when the initial value of the corresponding resource is read, and the + resource is written later. + + Assumption of this pass: + . Compound resource operations have already been decomposed. + . Dead functions have already been removed, as resource arguments in dead + functions can cause the pass to fail. + }]; + + let constructor = "TF::CreatePromoteResourcesToArgsPass()"; + +let options = [ + ListOption<"functions_", "functions", "std::string", + "Comma separated list of functions whose resources " + "read/writes should be promoted to function inputs/outputs."> + ]; +} + +def PromoteVarHandlesToArgsPass : Pass<"tf-promote-var-handles-to-args", "ModuleOp"> { + let summary = "Promote tf.VarHandleOps to function arguments."; + let description = [{See joint description in promote resources to args.}]; + + let constructor = "TF::CreatePromoteVarHandlesToArgsPass()"; +} + +def ExtractTPUCopyWithDynamicShapeOpPass : Pass<"tf-extract-tpu-copy-with-dynamic-shape-op", "mlir::func::FuncOp"> { + let summary = "Extract the TPUCopyWithDynamicShapeOp out of the host launch and place it on device launch"; + + let description = [{ + This pass looks for TPUCopyWithDynamicShapeOp which wraps in a + `tf_device.launch` with host device attribute. It extracts the ops and wrap + them in `tf_device.launch` with tpu device attribute so that ops can be + run on TPU instead of CPU while still being compiled on host. + }]; + + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; + + let constructor = "TFTPU::CreateExtractTPUCopyWithDynamicShapeOpPass()"; +} + +def TPUAnnotateDynamicShapeInputsPass : Pass<"tf-tpu-annotate-dynamic-shape-inputs", "ModuleOp"> { + let summary = "Annotate the inputs returned by TPUCopyWithDynamicShapeOp with dynamic shape"; + + let description = [{ + This pass looks for the usage of the result of TPUCopyWithDynamicShapeOp + and sets the shape of these inputs to be dynamic shaped. This will ensure + that the generated HLO program is correctly reflecting the dynamic shape. + }]; + + // Required for mhlo bounded shape extension. + let dependentDialects = ["mhlo::MhloDialect"]; + let constructor = "TFTPU::CreateTPUAnnotateDynamicShapeInputsPass()"; +} + +def ReplicateInvariantOpHoistingPass : Pass<"tf-replicate-invariant-op-hoisting", "mlir::func::FuncOp"> { + let summary = "Hoists replicate invariant operations out of replicate"; + + let description = [{ + This pass looks for replicate invariant ops in a `tf_device.replicate` op + region and hoists them out. It also makes `tf.Shape` ops replicate invariant + if possible. This currently updates or replaces `tf.Shape` ops of replicated + arguments, either tensors or resources. + + The primary benefit of the pass is to hoist `num_replicas` `_TPUCompile`s + into a single `_TPUCompile`. + + This pass assumes that when a `tf.Shape` directly inputs from `replicate` + params, then it is the same shape across replicas. + + For example, the following + + ```mlir + tf_device.replicate([%0, %1] as %ri: tensor<*xi32>) {n = 2 : i32} { + %2 = "tf.Shape"(%ri) : (tensor<*xi32>) -> tensor + tf_device.return + } + ``` + + gets converted to + + ```mlir + tf_device.replicate([%0, %1] as %ri: tensor<*xi32>) {n = 2 : i32} { + %2 = "tf.Shape"(%0) : (tensor<*xi32>) -> tensor + tf_device.return + } + ``` + + and for resource variables the following + + ```mlir + tf_device.replicate([%0, %1] as %ri: tensor<*x!tf_type.resource>) {n = 2 : i32} { + %2 = "tf.ReadVariableOp"(%ri) : tensor<*x!tf_type.resource> -> tensor<*xi32> + %3 = "tf.Shape"(%2) : (tensor<*xi32>) -> tensor + tf_device.return + } + ``` + + gets converted to + + ```mlir + tf_device.replicate([%0, %1] as %ri: tensor<*x!tf_type.resource>) {n = 2 : i32} { + %2 = "tf.ReadVariableOp"(%ri) : tensor<*x!tf_type.resource> -> tensor<*xi32> + %3 = "tf.VariableShape"(%0) : (tensor<*x!tf_type.resource>) -> tensor + tf_device.return + } + ``` + }]; + + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; + + let constructor = "TFDevice::CreateReplicateInvariantOpHoistingPass()"; +} + +def LowerQuantizedPass : Pass<"tf-lower-quantized", "mlir::func::FuncOp"> { + let summary = "Lowers ops that require quantized input or output."; + + let description = [{ + This pass rewrites all ops that have at least one input or output that must + be a quantized type to ops whose inputs and outputs allow non-quantized + types. Examples of quantized types are TF_Qint8 or TF_Quint8. + + An example is TF_DequantizeOp, which converts a quantized type to a float. + This op is rewritten to generic ops that perform the scale and shift + and can operate on non-quantized types. + + Currently, TF_DequantizeOp is the only op with a lowering that falls + in this category. When more lowerings are added (e.g. QuantizeV2Op), + they should be added to this pass. + }]; + + let constructor = "TF::CreateLowerQuantizedPass()"; +} + +def TPUHostComputationExpansionPass : Pass<"tf-tpu-host-computation-expansion", "mlir::func::FuncOp"> { + let summary = "Expands host computation before and after TPU computation."; + + let description = [{ + This pass expands outside compilation attributes to Identity/Cast ops + at the head of TPU computation if it's only used by outside compiled ops. + }]; + + let constructor = "TFTPU::CreateTPUHostComputationExpansionPass()"; +} + +def TPUUpdateEmbeddingEnqueueOpInputsPass : Pass<"tf-tpu-update-embedding-enqueue-op-inputs", "mlir::func::FuncOp"> { + let summary = "Updates inputs to TPU embedding enqueue ops depending on whether graph is in training mode or in evaluation mode."; + + let description = [{ + Updates inputs to TPU embedding enqueue ops depending on whether graph + is in training mode or in evaluation mode. + }]; + + let constructor = "TFTPU::CreateTPUUpdateEmbeddingEnqueueOpInputsPass()"; +} + +def DropWhileShapeInvariantPass : Pass<"tf-drop-while-shape-invariant", "mlir::func::FuncOp"> { + let summary = "Drop `shape_invariant` attribute from While/WhileRegion ops."; + + let description = [{ + Drop `shape_invariant` attribute from tf.While and tf.WhileRegion op. This + would allow shape inference pass to further refine operand/result shapes of + these ops. This is only safe to do when compiling to XLA. + }]; + + let constructor = "TF::CreateDropWhileShapeInvariantPass()"; +} + +def DropWhileShapeInvariantInDeviceClusterPass : Pass<"tf-drop-while-shape-invariant-in-device-cluster", "mlir::func::FuncOp"> { + let summary = "Drop `shape_invariant` attribute from While/WhileRegion ops inside device cluster."; + + let description = [{ + Drop `shape_invariant` attribute from tf.While and tf.WhileRegion op only + inside device cluster. This would allow shape inference pass to further + refine operand/result shapes of these ops. This is only safe to do when + compiling to XLA. + }]; + + let constructor = "TF::CreateDropWhileShapeInvariantInDeviceClusterPass()"; +} + +def TPUCleanupClusterAttributesPass : Pass<"tf-tpu-cleanup-cluster-attributes", "ModuleOp"> { + let summary = "Eliminate _replication_info and other attributes from ops in a cluster"; + + let description = [{ + This pass eliminate `_replication_info` and `device` attribute on operations + that are contained in a tf_device.cluster op. + }]; + + let constructor = "TFTPU::CreateTPUClusterCleanupAttributesPass()"; +} + +def TPUSpaceToDepthPass : Pass<"tf-tpu-space-to-depth-pass", "ModuleOp"> { + let summary = "Applies automatic space to depth transform for the first or frontier convolutions consume host inputs on TPU."; + + let description = [{ + Automatic space to depth transform is done by adding space to depth transform op after host input + and applying space to depth transform for the first convolution and its backprop filter on TPU. + + For example, original program: + + ```mlir + module { + func @while_body { + %input = "tf.IteratorGetNext"(...) {device = "/CPU:0"}: -> tensor<2x224x224x3xf32> + %device_launch = "tf_device.cluster_func"(%input,...) {func = @_func,...) + return ... + } + func @_func(%input: tensor<2x224x224x3xf32>, %filter: tensor<7x7x3x64xf32>) { + %6 = "tf.Conv2D"(%input, %filter) {strides = [1, 2, 2, 1]}: (tensor<2x230x230x3xf32>, tensor<7x7x3x64xf32>) -> tensor<2x112x112x64xf32> + } + } + ``` + + The program will be transformed into: + + ```mlir + module { + func @while_body { + %input = "tf.IteratorGetNext"(...) {device = "/CPU:0"} -> tensor<2x224x224x3xf32> + %space_to_depth = "tf.SpaceToDepth"(%input) {block_size = 2, ...}: (tensor<2x224x224x3xf32>) -> tensor<2x112x112x12xf32> + %device_launch = "tf_device.cluster_func"(%space_to_depth,...) {func = @_func,...) + return ... + } + func @_func(%input: tensor<2x112x112x12xf32>, %filter: tensor<7x7x3x64xf32>) { + %filter_transform = "tf.Pad/tf.Transpose/tf.Reshape"(%filter): tensor<7x7x3x64xf32>) -> tensor<4x4x12x64xf32> + %conv = "tf.Conv2D"(%input, %filter_transfrom) {strides = [1, 1, 1, 1]}: (tensor<2x112x112x12xf32>, tensor<4x4x12x64xf32>) -> tensor<2x112x112x64xf32> + } + } + ``` + + This way, the first convolution with 3 feature dimension will be transformed + to 12 feature dimension, which has better performance on TPU. + }]; + + let constructor = "TFTPU::CreateTPUSpaceToDepthPass()"; +} + +def MergeControlFlowPass : Pass<"tf-merge-control-flow", "ModuleOp"> { + let summary = "Merges IfRegion ops together with a common predicate."; + + let description = [{ + This pass merges IfRegion ops together if they have the same predicate and it + is safe to do so (there are no intermediate dependencies, they are in the + same block, etc). + + For example: + + ```mlir + "tf.IfRegion"(%0) ( { + %2 = "tf.A"() : () -> (tensor) + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = true } : (tensor) -> () + "tf.IfRegion"(%0) ( { + %2 = "tf.B"() : () -> (tensor) + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = true } : (tensor) -> () + ``` + + Would be transformed to: + + ```mlir + "tf.IfRegion"(%0) ( { + %2 = "tf.A"() : () -> (tensor) + %3 = "tf.B"() : () -> (tensor) + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = true } : (tensor) -> () + ``` + }]; + + let constructor = "TFDevice::CreateMergeControlFlowPass()"; +} + +def TensorListOpsDecompositionPass : Pass<"tf-tensor-list-ops-decomposition", "ModuleOp"> { + let summary = "Decomposes TensorList operations into generic operations on tensors."; + + let description = [{ +This pass rewrites TensorList operations into generic and non-mutating +operations on tensors. This results in operations that can be legalized to XLA. + +The list is converted to a single large tensor that includes all list elements, +with a new first dimension for the list index. List update operations are +converted to operations that create a new tensor representing the list. + +In the current implementation, the resulting operations are statically shaped, +which means it must be possible to infer a bound on the full shape of the +TensorList. That is, the `element_shape` and `num_elements` arguments to a +tensor list creation op are constant. + +A tensor list creation op `tf.EmptyTensorList`/`tf.TensorListReserve` will be +turned in to a zero-initialized buffer, and the size is initialized to 0 +for `tf.EmptyTensorList` or the specified size for `tf.TensorListReserve`. +Each push will be turned into `tf.XlaDynamicUpdateSlice` with the incremented +size, and each pop will be turned into a `tf.Slice` and a copy of the buffer +with decremented size. Each `tf.TensorListSetItem` will be turned into a +`tf.XlaDynamicUpdateSlice` with unchanged size, and each `tf.TensorListGetItem` +will be rewritten to a `tf.Slice`. + +The pass also works across control flow and functional calls. + +For example, the TensorList ops in the following function: + +```mlir +func @main(%arg0: tensor<8x4xf32>) { + %elem_shape = "tf.Const"() {value = dense<[8, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + %max_size = "tf.Const"() {value = dense<10> : tensor} : () -> tensor + %tl = "tf.EmptyTensorList"(%elem_shape, %max_size) : (tensor<2xi32>, tensor) -> tensor>> + %push = "tf.TensorListPushBack"(%tl, %arg0) : (tensor>>, tensor<8x4xf32>) -> tensor>> + return +} +``` + +will be transformed to: + +```mlir +func @main(%arg0: tensor<8x4xf32>) { + // EmptyTensorList lowering + %emptyi = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %emptyf = "tf.Cast"(%emptyi) : (tensor) -> tensor + %size_shape = "tf.Const"() {value = dense<[10, 8, 4]> : tensor<3xi32>} : () -> tensor<3xi32> + %tl = "tf.BroadcastTo"(%emptyf, %size_shape) : (tensor, tensor<3xi32>) -> tensor<10x8x4xf32> + // TensorListPushBack lowering + %index_in_list = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %arg0_shape = "tf.Const"() {value = dense<[1, 8, 4]> : tensor<3xi32>} : () -> tensor<3xi32> + %arg0_reshaped = "tf.Reshape"(%arg0, %arg0_shape) : (tensor<8x4xf32>, tensor<3xi32>) -> tensor<1x8x4xf32> + %zeroi2 = "tf.Const"() {value = dense<0> : tensor<2xi32>} : () -> tensor<2xi32> + %axis = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %start_indices = "tf.ConcatV2"(%index_in_list, %zeroi2, %axis) : (tensor<1xi32>, tensor<2xi32>, tensor) -> tensor<3xi32> + %push = "tf.XlaDynamicUpdateSlice"(%tl, %arg0_reshaped, %start_indices) : (tensor<10x8x4xf32>, tensor<1x8x4xf32>, tensor<3xi32>) -> tensor<10x8x4xf32> + %one = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %next_index_in_list = "tf.AddV2"(%index_in_list, %one) : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + return +} +``` + }]; + + let constructor = "TF::CreateTensorListOpsDecompositionPass()"; +} + +def TPUParallelExecuteSinkResourceWritePass : Pass<"tf-tpu-parallel-execute-sink-resource-write", "mlir::func::FuncOp"> { + let summary = "Moves tf.AssignVariableOp consumers of tf_device.parallel_execute " + "into tf_device.parallel_execute regions"; + + let constructor = "TFTPU::CreateTPUParallelExecuteSinkResourceWritePass()"; +} + +def RewriteTPUEmbeddingOpsPass : Pass<"tf-rewrite-tpu-embedding-ops", "mlir::func::FuncOp"> { + let summary = "Rewrites TPU embedding send/recv ops by adding TPU embedding " + "deduplication data"; + + let constructor = "TF::CreateRewriteTPUEmbeddingOpsPass()"; +} + +def InitTextFileToImportPass : Pass<"tf-init-text-file-to-import", "mlir::func::FuncOp"> { + let summary = "convert InitializeTableFromTextFileV2 ops to LookupTableImportV2Op " + "to remove the dependency on asset files"; + + let constructor = "TF::CreateInitTextFileToImportPass()"; + let options = [ + Option<"saved_model_dir_", "tf-saved-model-dir", "std::string", /*default=*/"", + "Directory containing the model exported as a TensorFlow " + "SavedModel. If your model is not based on the TensorFlow " + "SavedModel, use an empty value."> + ]; +} + +def FusedKernelMatcherPass : Pass<"tf-fused-kernel-matcher", "mlir::func::FuncOp"> { + let summary = "Matches computations corresponding to optimized fused kernels"; + let constructor = "TF::CreateFusedKernelMatcherPass()"; +} + +def TFDataOptimizationPass : Pass<"tf-data-optimization", "mlir::func::FuncOp"> { + let summary = "Performs tf.data optimizations"; + let constructor = "TF::CreateTFDataOptimizationPass()"; +} + +def FunctionalControlFlowToCFGPass : Pass<"tf-functional-control-flow-to-cfg", "mlir::func::FuncOp"> { + let summary = "Transform functional control flow Ops to MLIR Control Form Graph " + "(CFG) form"; + let constructor = "TF::CreateTFFunctionalControlFlowToCFG()"; + let dependentDialects = ["tensor::TensorDialect"]; +} + +def BreakUpIslandsPass : Pass<"tf-executor-break-up-islands", "mlir::func::FuncOp"> { + let summary = "Transform from TF control dialect to TF executor dialect."; + let constructor = "CreateBreakUpIslandsPass()"; + let dependentDialects = ["mlir::tf_executor::TensorFlowExecutorDialect"]; +} + +def SplitIntoIslandPerOpPass : Pass<"tf-executor-split-into-island-per-op", "mlir::func::FuncOp"> { + let summary = "Transform from TF control dialect to TF executor dialect."; + let description = [{ + Splits an island with multiple ops into multiple islands (one per op). Does + not create any control dependencies between new islands, and does not + propagate control dependencies that potentially existed between the old + islands into the new islands. Maintains existing data dependencies between + ops wrapped by the new islands. + + Example: original program: + + ```mlir + func.func @dangling_print(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %graph:2 = tf_executor.graph { + %island1:3 = tf_executor.island { + %add1 = "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %add2 = "tf.Add"(%add1, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %res = "tf.Print"(%add2) { message = "add result" } : (tensor<*xi32>) -> (tensor<*xi32>) + tf_executor.yield %add1, %add2 : tensor<*xi32>, tensor<*xi32> + } + tf_executor.fetch %island1#0, %island1#1 : tensor<*xi32>, tensor<*xi32> + } + func.return %graph#0, %graph#1 : tensor<*xi32>, tensor<*xi32> + } + ``` + + will be converted by this pass into: + + ```mlir + func.func @dangling_print(%arg0: tensor<*xi32>, %arg1: tensor) -> (tensor<*xi32>, tensor<*xi32>) { + %0:2 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Add"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.Add"(%outputs, %arg1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.Print"(%outputs_0) {message = "add result"} : (tensor<*xi32>) -> tensor<*xi32> + tf_executor.fetch %outputs, %outputs_0 : tensor<*xi32>, tensor<*xi32> + } + return %0#0, %0#1 : tensor<*xi32>, tensor<*xi32> + } + ``` + }]; + let constructor = "mlir::TF::CreateSplitIntoIslandPerOpPass()"; + let dependentDialects = ["mlir::tf_executor::TensorFlowExecutorDialect"]; +} + +def FunctionalToExecutorDialectConversionPass : Pass<"tf-functional-to-executor-conversion", "mlir::func::FuncOp"> { + let summary = "Transform from func op to TF executor dialect."; + let constructor = "CreateFunctionalToExecutorDialectConversionPass()"; + let dependentDialects = ["mlir::tf_executor::TensorFlowExecutorDialect"]; +} + +def SimpleTFDeviceAssignmentPass : Pass<"tf-simple-device-assignment", "mlir::func::FuncOp"> { + let summary = "Simple device assignment in TF dialect."; + let constructor = "TF::CreateSimpleTFDeviceAssignmentPass()"; + let options = [ + Option<"default_device_", "default-device", "std::string", /*default=*/"\"cpu\"", + "The default device to assign."> + ]; + let description = [{ + Assigns the default device to all ops that have an empty (or + nonexistent) device attribute. + + For example, if we have the code + + ```mlir + %0 = "tf.Const"() {value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + %1 = "tf.Const"() {device = "", value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + %2 = "tf.Const"() {device = "baz", value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + ``` + + then running this pass with 'default-device=foobar', we get: + + ```mlir + %0 = "tf.Const"() {device = "foobar" value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + %1 = "tf.Const"() {device = "foobar", value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + %2 = "tf.Const"() {device = "baz", value = dense<[[42.0]]> : tensor<1x1xf32>} : () -> tensor<1x1xf32> + ``` + }]; +} + +def TFDeviceAssignmentByFuncAttrPass : Pass<"tf-device-assignment-by-func-attr", "mlir::func::FuncOp"> { + let summary = "Device assignment in TF dialect using the device specified in the function attribute."; + let constructor = "TF::CreateTFDeviceAssignmentByFuncAttrPass()"; +} + +def LayoutAssignmentPass : Pass<"tf-layout-assignment", "mlir::func::FuncOp"> { + let summary = "Layout assignment pass."; + let constructor = "TF::CreateLayoutAssignmentPass()"; + let options = [ + Option<"force_data_format_", "force-data-format", "std::string", /*default=*/"", + "Force data format for all layout sensitive ops."> + ]; +} + +def MoveTransposesPass : Pass<"tf-move-transposes", "mlir::func::FuncOp"> { + let summary = "Move transposes pass."; + let constructor = "TF::CreateMoveTransposesPass()"; + let options = [ + Option<"fold_transpose_in_ops_", "fold-transpose-in-ops", "bool", /*default=*/"true", + "Whether to fold transposes in ops which can support folding.">, + Option<"direction_", "direction", "enum MoveTransposeDirection", + /*default=*/"MoveTransposeDirection::kBegin", "Move transposes to the beginning or" + " the end of the block where they are defined.", + "llvm::cl::values(clEnumValN(MoveTransposeDirection::kBegin, \"begin\", \"beginning of the block\")," + "clEnumValN(MoveTransposeDirection::kEnd, \"end\", \"end of the block\"))"> + ]; +} + +def BroadcastFoldPass : Pass<"tf-broadcast-fold", "mlir::func::FuncOp"> { + let summary = "Fold explicit broadcasts into the following operations if they " + "support implicit broadcasting on their operand."; + let constructor = "TF::CreateBroadcastFoldPass()"; +} + +def ParallelExecuteToIslandsPass : Pass<"tf-parallel-execute-to-islands", "mlir::func::FuncOp"> { + let summary = "Lowers device parallel_execute to executor islands"; + let constructor = "TFDevice::CreateParallelExecuteToIslandsPass()"; + let options = [ + Option<"legacy_graph_export_", "legacy-graph-export", "bool", + /*default=*/"true", + "Determines whether or not this pass should execute logic that is " + "reserved for the legacy graph export pipeline to maintain expected " + "invariants. In the case of this pass, that means manually propagating " + "controls to lifted parallel execute regions to the graph fetch to " + "ensure the ops execute."> + ]; +} + +def ConstantOpDeviceAssignmentPass : Pass<"constant-op-device-assignment", "ModuleOp"> { + let summary = "Assign device for tf.Const ops"; + let constructor = "TF::CreateConstantOpDeviceAssignmentPass()"; +} + +def GuaranteeAllFuncsOneUsePass : Pass<"tf-guarantee-all-funcs-one-use", "ModuleOp"> { + let summary = "Guarantee all FuncOp's have only a single use."; + let constructor = "TF::CreateGuaranteeAllFuncsOneUsePass()"; +} + +def TPUColocateCompositeResourceOpsPass : Pass<"tf-tpu-colocate-composite-resource-ops", "mlir::func::FuncOp"> { + let summary = "Colocate resource with composite device assignment to TPU device."; + let constructor = "TFTPU::CreateTPUColocateCompositeResourceOps()"; + let description = [{ + Pass that co-locates resource ops that use composite device resources + (packed tensors) with the underlying physical TPU device. + + So for example, if we have a function that does (inside a `tf_device.replicate`): + + ```mlir + %0 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + ``` + + Then said `ReadVariableOp` is going to get replaced by: + + ```mlir + %0 = "tf_device.launch"() ( { + %2 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor<4xf32> + tf_device.return %2 : tensor<4xf32> + }) {...} : () -> tensor<4xf32> + ``` + }]; + let dependentDialects = [ + "mhlo::MhloDialect", + "tf_device::TensorFlowDeviceDialect" + ]; +} + +def TPUDevicePropagationPass : Pass<"tf-tpu-device-propagation", "mlir::func::FuncOp"> { + let summary = "Propagates TPU devices from ops to users"; + let constructor = "TFTPU::CreateTPUDevicePropagationPass()"; +} + +def TPUColocateSplitsPass : Pass<"tf-tpu-colocate-splits", "mlir::func::FuncOp"> { + let summary = "Colocates each Split op with its predecessor"; + let constructor = "TFTPU::CreateTPUColocateSplitsPass()"; + let description = [{ + It is beneficial for performance to assign a `Split` op to the same device + as its predecessor. This is because the weight of cut edges is always + minimized when the `Split` is with its predecessor. This colocation + constraint will be used by the placer graph optimization to assign a device + to the op. + + This pass should run in the export pipeline after tf-replicate-to-island so + each replica has its own distinct (predecessor, Split) pair. + + The colocation class (`_class`) of the `Split` is set to the same class as + its predecessor: + + ```mlir + %outputs1:2, %control1 = tf_executor.island wraps "tf.IteratorGetNext"(%arg) + {_class = ["loc:@dataset_iterator_1"]} + %outputs2:2, %control2 = tf_executor.island wraps "tf.Split"(%outputs0, %outputs1#1) + {_class = ["loc:@dataset_iterator_1", num_split = 2 : i32} + ``` + }]; +} + +def TPUIdentityPruningPass : Pass<"tf-tpu-identity-pruning", "ModuleOp"> { + let summary = "Removes Identity/IdentityN ops from the TPU computation"; + let constructor = "TFTPU::CreateTPUIdentityPruningPass()"; +} + +def TPUDynamicLayoutPass : Pass<"tf-tpu-dynamic-layout-pass", "ModuleOp"> { + let summary = "Inserts TPU layout ops to determine layout at run time."; + let constructor = "TFTPU::CreateTPUDynamicLayoutPass()"; + let description = [{ + A pass that allows TPU input layout to be determined after JIT compilation. + This is done by adding run-time ops that interpret compilation result and + copy the input to device with that layout. + + Example: original program: + + ```mlir + %input = "tf.IteratorGetNext"(...) {device = "/CPU:0"} + %compile:2 = "tf._TPUCompileMlir"(...) + %execute = "tf.TPUExecute"(%input, ..., %compile#1) {device = "/TPU:0"} + ``` + + Without this pass, later TF graph partitioning passes will insert send/recv + between %input and %execute and data will be copied to device in a fixed + layout. With this pass, the program will be transformed into: + + ```mlir + %input = "tf.IteratorGetNext"(...) {device = "/CPU:0"} + %compile:2 = "tf._TPUCompileMlir"(...) + %get_layout = "tf.TPUGetLayoutOp"(%compile#1) {...} + %copy_to_device = "tf.TPUCopyWithLayout"(%input, %get_layout) + {device = "/TPU:0"} + %execute = "tf.TPUExecute"(%copy_to_device, ..., %compile#1) + {device = "/TPU:0"} + ``` + + This way, %compile will determine the layout, which will be respected by + %copy_to_device. There will not be send/recv ops added by later passes, + because tf.TPUCopyWithLayout accepts a host input and produces a device + output. + }]; +} + +def DeviceIndexSelectorPass : Pass<"tf-device-index-selector", "mlir::func::FuncOp"> { + let summary = "Fold tf.DeviceIndex to constant."; + let constructor = "TF::CreateDeviceIndexSelectorPass()"; +} + +def OrderByDialectPass : Pass<"tf-order-by-dialect", "mlir::ModuleOp"> { + let summary = "Reorders ops so ops of the same dialect are next to each other."; + let description = [{ + Performs a reordering of ops so that + (a) ops of the same dialect are next to each other + (b) order within a dialect is preserved + . + For example, this would transform + %a = "x.f"() + %b = "y.f"(%a) + %c = "x.f"(%a) + to + %a = "x.f"() + %c = "x.f"(%a) + %b = "y.f"(%a) + so that the two "x" dialect instructions are next to each other. + }]; + let constructor = "TF::CreateOrderByDialectPass()"; +} + +def GroupByDialectPass : Pass<"tf-group-by-dialect", "mlir::ModuleOp"> { + let summary = "Groups ops into functions that only contain one dialect."; + let description = [{ + Factors operations into subroutines such that all functions only + contain a single dialect. Which of the dialects are allowed in the + "top" function is configurable. + + For example, the code + x.a() + x.b() + %c = y.c() + x.d(%c) + would be transformed into something like + call @x_1() + %c = call @y_1() + call @x_2(%c) + with @x_1, @x_2 and @y_1 filled in. + }]; + let constructor = "TF::CreateGroupByDialectPass()"; +} + +def RemoveUnusedArgumentsPass : Pass<"tf-remove-unused-arguments", "ModuleOp"> { + let summary = "Removes unused args from private functions & their callers."; + let description = [{ + Removes arguments from functions that aren't used in the function + body, outside of returns. Also adjusts the callers of said functions. + + For example, the code + func.func @f(%arg0, %arg1) { + SomeOpThatUsesArg0(%arg0) + return %arg0 + } + ... + call @x_1(x, y) + + would be transformed into + func.func @f(%arg0) { + return %arg0 + } + ... + call @x_1(x) + + Note that, in the above example, both args would be removed if there + wasn't the "SomeOpThatUsesArg0(%arg0)" line. + }]; + let constructor = "TF::CreateRemoveUnusedArgumentsPass()"; +} + +def RemoveUnusedWhileResultsPass : Pass<"tf-remove-unused-while-results", "mlir::func::FuncOp"> { + let summary = "Removes unused results from tf.WhileRegion ops"; + let description = [{ + Removes unused results from `tf.WhileRegion` ops along with the defining + ops in the body, if it is safe to do so. + Currently, the pass detects results with following properties: + - the result is unused outside of the `tf.WhileRegion` op + - the defining op of the result in the body can be safely removed + - the operand corresponding to the result is not used by any other op in + the condition or body (in particular, there must not be intermediate + pass-through ops like `tf.Identity`) + + + For example, the following pseudo MLIR code (types are left out for + brevity) + ```mlir + func.func @remove_first_result(%arg0, %arg1) { + %0:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2, %arg3): + %1 = "tf.OpA"() {is_stateless = true} + "tf.Yield"(%1) + }, { + ^bb0(%arg2, %arg3): + %1 = "tf.OpB"(%arg2) {is_stateless = true} + %2 = "tf.OpC"(%arg3) {is_stateless = true} + "tf.Yield"(%1, %2) + }) {is_stateless = true} + return %0#1 + } + ``` + would be transformed to + ```mlir + func.func @remove_first_result(%arg0, %arg1) { + %0 = "tf.WhileRegion"(%arg1) ({ + ^bb0(%arg3): + %1 = "tf.OpA"() {is_stateless = true} + "tf.Yield"(%1) + }, { + ^bb0(%arg3): + %1 = "tf.OpC"(%arg3) {is_stateless = true} + "tf.Yield"(%1) + }) {is_stateless = true} + return %0 + } + ``` + (the first result can be removed along with its defining op `tf.OpB`). + + }]; + let constructor = "TF::CreateRemoveUnusedWhileResultsPass()"; +} + +def HoistLoopInvariantPass : Pass<"tf-hoist-loop-invariant", "mlir::func::FuncOp"> { + let summary = "Hoists loop invariant ops to the outside of the loop"; + let description = [{ + Hoists loop invariant to the outside of the loop. The pass is similar to + LoopInvariantCodeMotion pass, but it also hoists ReadVariableOps, + if the variable is read only. + + For example, the following pseudo MLIR code (types are left out for + brevity) + ```mlir + func.func @hoist_loop_invariant(%arg0, %arg1) { + %var = "tf.VarHandleOp"() {container="", shared_name="var_name", device = "/device:CPU:0"} + %results:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2, %arg3): + %0 = "tf.OpA"() {is_stateless = true} + "tf.Yield"(%0) + }, { + ^bb0(%arg2, %arg3): + %1 = "tf.ReadVariableOp"(%var) + %2 = "tf.OpB"(%1) {is_stateless = true} + %3 = "tf.OpC"(%arg2, %2) {is_stateless = true} + %4 = "tf.OpD"(%arg3, %2) {is_stateless = true} + "tf.Yield"(%3, %4) + }) {is_stateless = true} + return %results#0, %results#1 + } + ``` + would be transformed to + ```mlir + func.func @hoist_loop_invariant(%arg0, %arg1) { + %var = "tf.VarHandleOp"() {container="", shared_name="var_name", device = "/device:CPU:0"} + %1 = "tf.ReadVariableOp"(%var) + %2 = "tf.OpB"(%1) {is_stateless = true} + %results:2 = "tf.WhileRegion"(%arg0, %arg1) ({ + ^bb0(%arg2, %arg3): + %0 = "tf.OpA"() {is_stateless = true} + "tf.Yield"(%0) + }, { + ^bb0(%arg2, %arg3): + %3 = "tf.OpC"(%arg2, %2) {is_stateless = true} + %4 = "tf.OpD"(%arg3, %2) {is_stateless = true} + "tf.Yield"(%3, %4) + }) {is_stateless = true} + return %results#0, %results#1 + } + ``` + The `tf.ReadVariableOp` and `tf.OpB` can be hoisted to the outside of + the loop. + + }]; + let constructor = "TF::CreateHoistLoopInvariantPass()"; +} + +def LocalizeVarHandlesPass : Pass<"tf-localize-var-handles", "ModuleOp"> { + let summary = "Creates VarHandleOps next to the operations that use them."; + let description = [{ + Creates VarHandleOps right next to the operations that use them, one + per operation. + This is useful for transformations that only end up with a few small + snippets of remaining TF code, and wish for those snippets to be + self-contained. + For example, this would transform + + "tf_saved_model.global_tensor"() { sym_name = "v" ... } + func @f(%arg0 {tf_saved_model.bound_input = @v}) { + %1 = "tf.ReadVariableOp"(%arg0) + ... + } + + to + + func @f(%arg0 {tf_saved_model.bound_input = @v}) { + %0 = "tf.VarHandleOp"(sym_name = "v") + %1 = "tf.ReadVariableOp"(%0) + ... + } + + Note that this pass might leave behind unused values + (like e.g. %arg0 in the example above), which can later be + pruned using DCE. + }]; + let constructor = "TF::CreateLocalizeVarHandlesPass()"; +} + +def StripTfAttributesPass : Pass<"tf-strip-tf-attributes", "ModuleOp"> { + let summary = "Removes TF specific attributes"; + let description = [{ + Removes attributes that are TF specific (start with "tf.") or that + have a value from the TF dialect. Useful after legalizing TF graphs + to other dialects, to remove any TF remnants. + }]; + let constructor = "TF::CreateStripTfAttributesPass()"; +} + +def NameAnonymousIteratorsPass : Pass<"tf-name-anonymous-iterators", "ModuleOp"> { + let summary = "Converts anonymous iterators to named iterators"; + let description = [{ + This converts AnonymousIterator ops to Iterator, thus giving them a name. + For example, this will convert + %0 = "tf.AnonymousIteratorV3"() {...} + to + %0 = "tf.Iterator"() {shared_name = "_iterator1", ...} + }]; + let constructor = "TF::CreateNameAnonymousIteratorsPass()"; +} + +//===----------------------------------------------------------------------===// +// XlaCallModule +//===----------------------------------------------------------------------===// + +def XlaCallModuleDeserializationPass + : Pass<"tf-xla-call-module-deserialization", "ModuleOp"> { + let summary = "Deserializes StableHLO functions embedded in `tf.XlaCallModule` to top level module"; + + let description = [{ + This pass deserializes the StableHLO bytecodes embedded in tf.XlaCallModule, + then outlines the functions in the deserialized StableHLO module to the top + level MLIR module, with function renamings to avoid naming conflicts. + + After the outlining, it updates tf.XlaCallModule's module attribute to be + empty, adds an `_entry_function` attribute referring to the entry function. + It also adds a `_from_xla_call_module: true` attribute to each lifted + StableHLO function. + }]; + + // These dialects are needed by stablehlo deserialization. + // + // We use tensorflow::XlaCallModuleLoader. + // tensorflow::XlaCallModuleLoader will get or load dialects: + // Func, Stablehlo, Mhlo, Chlo, and Vhlo. + // + // XlaCallModuleLoader uses mlir::stablehlo::deserializePortableArtifact, + // which runs VhloLegalizeToStablehloPass whose depends on dialects: + // Func, Stablehlo, Shape, and Quantization. + // + // If we do not register them here, an error will be + // triggered because we cannot load a dialect while in a + // multi-threaded execution context, and PassManager is + // multi-threaded. + let dependentDialects = [ + "chlo::ChloDialect", + "shape::ShapeDialect", + "stablehlo::StablehloDialect", + "vhlo::VhloDialect", + "quant::QuantDialect", + "sdy::SdyDialect", + ]; + + let constructor = "TF::CreateXlaCallModuleDeserializationPass()"; +} + +def XlaCallModuleSerializationPass + : Pass<"tf-xla-call-module-serialization", "ModuleOp"> { + let summary = "Serializes StableHLO functions from top-level module into `tf.XlaCallModule`'s `module` attribute"; + + let description = [{ + This pass collects StableHLO functions referenced from `tf.XlaCallModule`'s + `_entry_function` attribute into a module, serializes the module into MLIR + bytecode, and embed the bytecode to `tf.XlaCallModule`'s `module` attribute. + + After serialization, this pass removes the `_entry_function` attribute from + `tf.XlaCallModule`, and removes all the serialized stablehlo functions + from the top-level module. + }]; + + let dependentDialects = [ + "stablehlo::StablehloDialect", + "vhlo::VhloDialect", + ]; + + let constructor = "TF::CreateXlaCallModuleSerializationPass()"; +} + +def ColocateTPUCopyWithDynamicShapePass : Pass<"tf-colocate-tpu-copy-with-dynamic-shape", "ModuleOp"> { + let summary = "Adjusts the device annotation of TPUCopyWithDynamicShape ops"; + let description = [{ + Adjusts the device annotation of TPUCopyWithDynamicShape ops such + that it matches that of the subsequent TPUExecute{,AndUpdateVariables} + op. + }]; + let constructor = "TFTPU::CreateColocateTPUCopyWithDynamicShapePass()"; +} + +def PrintPass : Pass<"tf-print", "ModuleOp"> { + let summary = "Prints the current module."; + let description = [{ + This prints the current module (by default, to stderr). The module is + otherwise piped through without changes. + + This complements MLIRs dumping functionality, which can sometimes be hard + to configure. This pass is useful if you know the exact position in the + pipeline where you want to dump, and are able to easily modify the pipeline + source code. + }]; + let constructor = "TF::CreatePrintPass()"; +} + +def MoveTpuCompileToFrontPass : Pass<"tf-move-tpu-compile-to-front", "mlir::func::FuncOp"> { + let summary = "Moves compile ops to the front."; + let description = [{ + Moves all _TPUCompileMlir ops as far to the front as possible. For example, + if we have: + + %0 = "tf.OpA"(...) + %1 = "tf._TPUCompileMlir"(...) + %2 = "tf.OpB"(...) + "SomeRegionOp"() ({ + "tf._TPUCompileMlir"(...) + }) + + then after the pass, this will have been transformed to: + + %1 = "tf._TPUCompileMlir"(...) + "SomeRegionOp"() ({ + "tf._TPUCompileMlir"(...) + }) + %0 = "tf.OpA"(...) + %2 = "tf.OpB"(...) + + Note that this will only move the ops if it's possible to do so. If there's + a data or control dependency prevening the reordering, it will leave the ops + where they are. + }]; + let constructor = "TF::CreateMoveTpuCompileToFrontPass()"; +} + +def DecomposeOptionalsPass : Pass<"tf-decompose-optionals", "ModuleOp"> { + let summary = "Decomposes ops of the 'Optional' family"; + let description = [{ + Turns OptionalFromValue, OptionalGetValue, OptionalNone, + and OptionalHasValue into identities. + + For example, if we have: + + %1 = "tf.OptionalFromValue"(%0) : (tensor) -> !tf_type.variant + %2 = "tf.OptionalGetValue"(%1) : (tensor) -> tensor + + then after the pass, this will have been transformed to: + + %1 = "tf.Identity"(%0) : (tensor) -> tensor + %2 = "tf.Identity"(%1) : (tensor) -> tensor + + }]; + let constructor = "TF::CreateDecomposeOptionalsPass()"; +} + +def OrderForProgramKeyPass : Pass<"tf-order-for-program-key", "ModuleOp"> { + let summary = "Move TPUCompile ops as early as possible."; + let description = [{ + Reorders blocks so that TPUCompileMlir ops (and launches that contain + TPUCompileMlir ops) are placed as early as possible. + + For example, the code + + "SomeOp"() : () -> tensor + %0, %1 = "tf._TPUCompileMlir"(...) + + would be reordered to + + %0, %1 = "tf._TPUCompileMlir"(...) + "SomeOp"() : () -> tensor + + This helps the "-tf-embedding-program-key" pass. The latter connects + certain ops to the output of the nearest CompileMlir op. But if there's + no preceding CompileMlir op, it tries to reorder, but breaks things in + the process. Sorting the ops before the pass helps. + }]; + let constructor = "TF::CreateOrderForProgramKeyPass()"; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..9a3de661c3ff8261434e1579d72174e319c3953e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.cc @@ -0,0 +1,195 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h" + +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tsl/platform/path.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +#define GEN_PASS_DEF_ASSETSINKINGPASS +#define GEN_PASS_DECL_ASSETSINKINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +class AssetSinkingPass : public impl::AssetSinkingPassBase { + public: + AssetSinkingPass() = default; + + explicit AssetSinkingPass(llvm::StringRef saved_model_dir) { + saved_model_dir_ = saved_model_dir.str(); + } + + void runOnOperation() override { + ModuleOp module = getOperation(); + if (!HasTfSavedModelSemantics(module)) { + return; + } + + auto init_op = GetSessionInitializerOp(module); + if (init_op == nullptr || init_op.getInitializers().empty()) { + return; + } + + SymbolTable symbol_table(module); + for (auto initializer : init_op.getInitializers()) { + auto func = symbol_table.lookup( + mlir::cast(initializer).getValue()); + RewriteFunction(symbol_table, func); + } + + // Clean up unused asset ops. + for (auto asset : llvm::make_early_inc_range(module.getOps())) { + if (symbol_table.symbolKnownUseEmpty(asset, module)) { + asset.erase(); + } + } + } + + private: + // Replaces bounded-input arguments of the function with constant ops in the + // body and removes the arguments. + void RewriteFunction(const SymbolTable& symbol_table, func::FuncOp func) { + if (func.getNumArguments() == 0) { + return; + } + + auto builder = OpBuilder::atBlockBegin(&func.front()); + + llvm::SmallDenseMap const_ops; + llvm::BitVector arg_indexes_to_remove(func.getNumArguments()); + + // Replace arguments with const ops. + for (BlockArgument argument : func.getArguments()) { + auto asset = LookupBoundInputOfType( + func, argument.getArgNumber(), symbol_table); + if (asset == nullptr) { + continue; + } + + // Create a const op for the asset if it doesn't already exist. + auto it = const_ops.find(asset.getSymName()); + if (it == const_ops.end()) { + // Asset filenames are relative to the SavedModel directory. + const std::string filename = tsl::io::JoinPath( + saved_model_dir_, absl::string_view(asset.getFilename())); + + RankedTensorType type = RankedTensorType::get( + {}, TF::StringType::get(builder.getContext())); + auto const_op = + TF::ConstOp::create(builder, builder.getUnknownLoc(), + DenseStringElementsAttr::get(type, {filename})); + + it = const_ops.insert({asset.getSymName(), const_op}).first; + } + + argument.replaceAllUsesWith(it->second.getOutput()); + arg_indexes_to_remove.set(argument.getArgNumber()); + } + + // Erase function arguments with bounded input. + CHECK(llvm::succeeded(func.eraseArguments(arg_indexes_to_remove))); + } +}; + +} // namespace + +absl::Status AddSessionInitializerAndInlineCheckpoint( + ModuleOp module, absl::string_view checkpoint_path) { + // The main function should be the only public function. + func::FuncOp main_func = nullptr; + for (auto func : module.getOps()) { + if (func.isPublic()) { + if (main_func != nullptr) { + return absl::InternalError("Only one public function is allowed."); + } + main_func = func; + } + } + if (main_func.getNumArguments() != 1) { + return absl::InternalError("Expected 1 argument for the main function."); + } + OpBuilder builder(module.getBodyRegion()); + // Create SessionInitializerOp; should reference main function. + StringAttr func_name = main_func.getSymNameAttr(); + llvm::SmallVector func_names = { + mlir::SymbolRefAttr::get(builder.getContext(), func_name)}; + tf_saved_model::SessionInitializerOp::create( + builder, module->getLoc(), builder.getArrayAttr(func_names)); + // Create AssetOp; this holds the checkpoint_path. + // TODO(b/318761632): Cleanup usage of string literals, instead use constants. + auto asset_op = tf_saved_model::AssetOp::create( + builder, module->getLoc(), + /*sym_name=*/ + builder.getStringAttr("__tf_saved_model_variables"), // Val unimportant. + /*filename=*/ + builder.getStringAttr(checkpoint_path)); + // Marks the input to be inlined. + main_func.setArgAttr( + 0, "tf_saved_model.bound_input", + SymbolRefAttr::get(builder.getContext(), asset_op.getName())); + // Bound arguments are expected to be of type tensor, not + // tensor. + auto tensor_string_type = + RankedTensorType::get({}, TF::StringType::get(builder.getContext())); + main_func.getArguments().front().setType(tensor_string_type); + main_func.setType( + FunctionType::get(builder.getContext(), {tensor_string_type}, + main_func.getFunctionType().getResults())); + // Name of the main function for the eventual executable needs to be set. + main_func->setAttr(kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr({func_name})); + main_func->setAttr( + kTfSavedModelInitializerTypeAttr, + builder.getStringAttr(kTfSavedModelInitializerRestoreType)); + module->setAttr("tf_saved_model.semantics", + UnitAttr::get(builder.getContext())); + return absl::OkStatus(); +} + +std::unique_ptr> CreateAssetSinkingPass( + llvm::StringRef saved_model_dir) { + return std::make_unique(saved_model_dir); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..17ad6bfea1c00728317fe4e84970354662b450d5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h @@ -0,0 +1,44 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_ASSET_SINKING_PASS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_ASSET_SINKING_PASS_H_ + +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace tf_saved_model { + +// Helper function that sets up a module for an AssetSinkingPass. The sole +// argument of the main function of `module` is prepared to be inlined with +// the value `checkpoint_path`. +// Also adds SessionInitializer op. +absl::Status AddSessionInitializerAndInlineCheckpoint( + ModuleOp module, absl::string_view checkpoint_path); + +// Creates a pass that sinks SavedModel asset filenames to constants. +std::unique_ptr> CreateAssetSinkingPass( + llvm::StringRef saved_model_dir); + +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_ASSET_SINKING_PASS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..85cd30af678bd60e019070b781145054dd623592 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.cc @@ -0,0 +1,325 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h" + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace mlir { +namespace tf_saved_model { + +namespace { +// Attribute name that specifies the input shapes of a function. +constexpr StringRef kTfInputShapesAttr = "tf._input_shapes"; + +// Updates terminator op arguments of 'func' after removing arguments +// specified in 'arguments_to_erase'. +template +void UpdateTerminatorArguments(T& func, + const ArrayRef arguments_to_erase, + llvm::BitVector& erase_indices) { + auto terminator = func.front().getTerminator(); + int num_operands = terminator->getNumOperands(); + erase_indices.resize(num_operands); + for (auto arg_index : arguments_to_erase) { + auto argument = func.getArgument(arg_index); + for (auto& use : argument.getUses()) { + if (llvm::isa(use.getOwner())) { + erase_indices.set(use.getOperandNumber()); + } + } + func.getArgument(arg_index).dropAllUses(); + } + if (llvm::isa(func.front().getTerminator())) { + terminator->eraseOperands(erase_indices); + } +} + +// Erases function arguments indexed at `args_to_erase`. Also applies the +// changes to any relevant function attributes accordingly. +LogicalResult EraseFuncOpArguments(func::FuncOp func_op, + const ArrayRef args_to_erase) { + BitVector args_to_erase_bit_vector(func_op.getNumArguments()); + for (const unsigned i : args_to_erase) args_to_erase_bit_vector.set(i); + + if (failed(func_op.eraseArguments(args_to_erase_bit_vector))) { + return failure(); + } + + // Erases entries in "tf._input_shapes" attribute of `func_op` that correspond + // to the erased arguments. + if (auto input_shapes_attr = + func_op->getAttrOfType(kTfInputShapesAttr); + input_shapes_attr) { + // Construct a new array of input shapes excluding the input shapes of the + // erased arguments. + SmallVector updated_input_shapes_attr; + for (const unsigned i : args_to_erase_bit_vector.flip().set_bits()) { + updated_input_shapes_attr.emplace_back(input_shapes_attr[i]); + } + + // Replaces the attribute with the updated "#tf_type.shape" array. + // Builder builder(func_op.getContext()); + func_op->setAttr( + kTfInputShapesAttr, + ArrayAttr::get(func_op.getContext(), updated_input_shapes_attr)); + } + return success(); +} + +// Updates 'while_op' signatures based on which arguments should be removed +// in 'arguments_to_erase'. +template +T GetUpdatedWhileOp(T while_op, const U& argument_types, + const llvm::SmallVector& arguments_to_erase) { + OpBuilder builder(while_op); + llvm::SmallVector new_operand_types; + llvm::SmallVector new_operands; + auto operands = while_op->getOperands(); + const int num_operands = while_op->getNumOperands(); + llvm::BitVector skip_indices(num_operands); + for (int i : arguments_to_erase) skip_indices.set(i); + for (int i = 0; i < num_operands; ++i) { + if (!skip_indices.test(i)) { + new_operand_types.emplace_back(argument_types[i]); + new_operands.emplace_back(operands[i]); + } + } + auto new_while_op = T::create(builder, while_op->getLoc(), new_operand_types, + new_operands, while_op->getAttrs()); + int new_index = 0; + for (int i = 0; i < num_operands; ++i) { + if (!skip_indices.test(i)) { + while_op->getResult(i).replaceAllUsesWith( + new_while_op->getResult(new_index++)); + } + } + return new_while_op; +} + +// Creates a constant op that holds 'tensor_elements'. +TF::ConstOp GetConstOpFromElementsAttr(ElementsAttr tensor_elements, + OpBuilder builder, Location loc) { + return TF::ConstOp::create(builder, loc, tensor_elements); +} + +// Replace usage of 'read_variable_op' with 'value'. +void PropagateUsage(TF::ReadVariableOp read_variable_op, ElementsAttr value) { + OpBuilder builder(read_variable_op); + read_variable_op->getResult(0).replaceAllUsesWith( + GetConstOpFromElementsAttr(value, builder, read_variable_op->getLoc())); +} + +// Propagates a resource usage across the graph where +// 'user_op' uses a resource and is passed to this op at 'argument_index'. +// This resource should be replaced by 'value'. +// Output params: +// - work_list: Is updated with new regions to process that is called +// by 'user_op'; +// - arguments_to_erase: Captures updates to the graph - which arguments +// to remove from the op; +LogicalResult PropagateUsage( + Operation* user_op, int argument_index, ElementsAttr value, + llvm::SmallVector, 4>* work_list, + llvm::MapVector>* + arguments_to_erase) { + if (auto read_variable_op = dyn_cast(user_op)) { + (*arguments_to_erase)[read_variable_op]; + PropagateUsage(read_variable_op, value); + } else if (auto call = dyn_cast(user_op)) { + (*arguments_to_erase)[call].push_back(argument_index); + if (auto func = dyn_cast(call.resolveCallable())) { + (*arguments_to_erase)[func].push_back(argument_index); + work_list->push_back(std::make_pair(&func.getRegion(), argument_index)); + } + } else if (auto if_op = dyn_cast(user_op)) { + (*arguments_to_erase)[if_op].push_back(argument_index); + for (auto callee : {if_op.then_function(), if_op.else_function()}) { + (*arguments_to_erase)[callee].push_back(argument_index - 1); + work_list->push_back( + std::make_pair(&callee.getBody(), argument_index - 1)); + } + } else if (auto if_op = dyn_cast(user_op)) { + (*arguments_to_erase)[if_op].push_back(argument_index); + for (auto callee : {&if_op.getThenBranch(), &if_op.getElseBranch()}) { + work_list->push_back(std::make_pair(callee, argument_index)); + } + } else if (auto while_op = dyn_cast(user_op)) { + (*arguments_to_erase)[while_op].push_back(argument_index); + for (auto callee : {while_op.cond_function(), while_op.body_function()}) { + (*arguments_to_erase)[callee].push_back(argument_index); + work_list->push_back(std::make_pair(&callee.getBody(), argument_index)); + } + } else if (auto while_op = dyn_cast(user_op)) { + (*arguments_to_erase)[while_op].push_back(argument_index); + for (auto callee : {&while_op.getCond(), &while_op.getBody()}) { + work_list->push_back(std::make_pair(callee, argument_index)); + } + } else if (auto batch_func_op = dyn_cast(user_op)) { + (*arguments_to_erase)[batch_func_op].push_back(argument_index); + // Add the called function to the work list. + func::FuncOp func_op = batch_func_op.func(); + (*arguments_to_erase)[func_op].push_back(argument_index); + work_list->push_back({&func_op.getRegion(), argument_index}); + } else { + // Return and yield ops are the only ops that use the resource outside of + // the above ops. + if (!(isa(user_op) || isa(user_op))) { + user_op->emitError() << "could not rewrite use of immutable bound input"; + return failure(); + } + } + + return success(); +} + +// An override that takes region. +LogicalResult PropagateUsage( + Region* region, ElementsAttr value, int argument_index, + llvm::SmallVector, 4>* work_list, + llvm::MapVector>* + arguments_to_erase) { + auto arg = region->getArgument(argument_index); + for (auto& usage : arg.getUses()) { + auto* user_op = usage.getOwner(); + int operand_index = usage.getOperandNumber(); + if (failed(PropagateUsage(user_op, operand_index, value, work_list, + arguments_to_erase))) { + return failure(); + } + } + + return success(); +} +} // namespace + +LogicalResult EraseObsoleteResourceUses( + llvm::MapVector> + arguments_to_erase) { + // All updates to different ops are captured in 'arguments_to_erase'. + // Now loop on them and based on each item type update accordingly. + for (auto& [user_op, args_to_erase] : arguments_to_erase) { + if (auto func = dyn_cast(user_op)) { + // To update a function we will need to: + // 1) Remove the unused arguments from the function itself. + // 1-2) Remove func attributes corresponding to the removed arguments. + // 2) Remove any returns that are not needed from the function terminator + // op in the function. + // 3) Update function result to match the terminator. + llvm::BitVector result_indices_to_erase; + UpdateTerminatorArguments(func, args_to_erase, result_indices_to_erase); + if (failed(EraseFuncOpArguments(func, args_to_erase))) { + return failure(); + } + + if (failed(func.eraseResults(result_indices_to_erase))) { + return failure(); + } + } else if (auto read_var = dyn_cast(user_op)) { + // Read variables was already replaced by constant op. Just remove the op. + read_var->erase(); + } else if (auto while_op = dyn_cast(user_op)) { + GetUpdatedWhileOp( + while_op, while_op.cond_function().getArgumentTypes(), args_to_erase); + while_op->erase(); + } else if (auto while_op = dyn_cast(user_op)) { + auto new_while_op = GetUpdatedWhileOp( + while_op, while_op.getCond().getArgumentTypes(), args_to_erase); + new_while_op.getCond().takeBody(while_op.getCond()); + new_while_op.getBody().takeBody(while_op.getBody()); + llvm::BitVector erase_indices; + UpdateTerminatorArguments(new_while_op.getBody(), args_to_erase, + erase_indices); + llvm::BitVector body_bit_vector( + new_while_op.getBody().front().getNumArguments()); + for (auto i : args_to_erase) body_bit_vector.set(i); + new_while_op.getBody().front().eraseArguments(body_bit_vector); + llvm::BitVector cond_bit_vector( + new_while_op.getCond().front().getNumArguments()); + for (auto i : args_to_erase) cond_bit_vector.set(i); + new_while_op.getCond().front().eraseArguments(cond_bit_vector); + while_op->erase(); + } else if (auto batch_func_op = dyn_cast(user_op)) { + llvm::BitVector erase_indices(user_op->getNumOperands()); + for (auto operand_index : args_to_erase) { + erase_indices.set(operand_index); + } + batch_func_op.eraseArguments(erase_indices); + } else { + llvm::BitVector erase_indices(user_op->getNumOperands()); + for (auto operand_index : args_to_erase) { + erase_indices.set(operand_index); + } + user_op->eraseOperands(erase_indices); + } + } + + return success(); +} + +// Traces usage of 'var_handle_op' or 'resources' and replaces it's usage with +// constant value 'value'. All op operands updates are captured in +// 'arguments_to_erase'. +LogicalResult ReplaceVarWithConstant( + mlir::Value::use_range uses, ElementsAttr value, + llvm::MapVector>* + arguments_to_erase) { + llvm::SmallVector, 4> work_list; + for (auto& usage : uses) { + auto* user_op = usage.getOwner(); + int operand_index = usage.getOperandNumber(); + if (failed(PropagateUsage(user_op, operand_index, value, &work_list, + arguments_to_erase))) { + return failure(); + } + } + + // Container to mark visited regions to avoid infinite loop. + llvm::DenseSet> visited; + while (!work_list.empty()) { + auto work_item = work_list.pop_back_val(); + if (visited.contains(work_item)) continue; + if (failed(PropagateUsage(work_item.first, value, work_item.second, + &work_list, arguments_to_erase))) { + return failure(); + } + visited.insert(work_item); + } + + return success(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..b0afa133ce9ab97b9a784dd7b8e37ad910fc2658 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h @@ -0,0 +1,47 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_UTILS_H_ + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace tf_saved_model { +// Container to hold all update actions on ops. +// Key: Operation to update. +// Value: optional list of argument indices to delete from this op. +// Note that we use MapVector because we want to iterate on the same order +// of insertion. +LogicalResult EraseObsoleteResourceUses( + llvm::MapVector> + arguments_to_erase); + +// Traces usage of 'var_handle_op' or 'resources' and replaces it's usage with +// constant value 'value'. All op operands updates are captured in +// 'arguments_to_erase'. +LogicalResult ReplaceVarWithConstant( + mlir::Value::use_range uses, ElementsAttr value, + llvm::MapVector>* + arguments_to_erase); +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.cc new file mode 100644 index 0000000000000000000000000000000000000000..73f7233b292a711f8bb68e99e0445115359aa489 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.cc @@ -0,0 +1,244 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h" + +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_value_typed_analyzer.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/session_utils.h" +#include "tensorflow/core/framework/resource_var.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +// Attribute name that specifies the input shapes of a function. +constexpr StringRef kTfInputShapesAttr = "tf._input_shapes"; + +// Build and returns ElementsAttr which holds the data in 'tensor'. +ElementsAttr GetTensorValueAsElementsAttr(const tensorflow::Tensor& tensor, + OpBuilder builder) { + absl::StatusOr tensor_attr_or = + tensorflow::ConvertTensor(tensor, &builder); + if (!tensor_attr_or.ok()) return nullptr; + return tensor_attr_or.value(); +} + +// Returns ElementsAttr which has the value held by 'resource_tensor'. +ElementsAttr GetTensorValueAsElementsAttr( + TF::VarHandleOp var_handle_op, const tensorflow::Tensor& resource_tensor, + const tensorflow::DeviceMgr* mgr, OpBuilder builder) { + if (resource_tensor.dtype() != tensorflow::DT_RESOURCE) { + return GetTensorValueAsElementsAttr(resource_tensor, builder); + } + + auto handle = resource_tensor.scalar()(); + auto* var_ptr = tf_saved_model::GetVariableFromSession(var_handle_op, + handle.device(), mgr); + if (!var_ptr) { + return nullptr; + } + tensorflow::core::RefCountPtr var(var_ptr); + auto* tensor = var_ptr->tensor(); + + return GetTensorValueAsElementsAttr(*tensor, builder); +} + +// Returns ID for identifying a resource. +std::tuple GetResourceKey( + Operation* op) { + llvm::StringRef device; + if (auto attr = op->getAttrOfType("device")) { + device = attr.getValue(); + } + + llvm::StringRef container; + if (auto attr = op->getAttrOfType("container")) { + container = attr.getValue(); + } + + llvm::StringRef shared_name; + if (auto attr = op->getAttrOfType("shared_name")) { + shared_name = attr.getValue(); + } + + return std::tuple{ + device, container, shared_name}; +} + +// Remove the initialization of the variables in 'var_handle_ops' from +// the session init function 'session_init_func' +void RemoveVariablesInitializations( + const llvm::SmallVector& var_handle_ops, + func::FuncOp session_init_func) { + // We identify the variables using (device, container, shared_name) of the + // resource. Capture them here and use them to identify the useless + // initializations. + llvm::SetVector> + variables; + for (auto var_handle_op : var_handle_ops) + variables.insert(GetResourceKey(var_handle_op)); + + llvm::SmallVector work_list; + for (auto var_handle_op : session_init_func.getOps()) { + if (variables.count(GetResourceKey(var_handle_op))) + work_list.push_back(var_handle_op); + } + + // Capture list of ops to be erased by traversing usage starting from + // the VarHandle ops. + llvm::SetVector erase_list; + while (!work_list.empty()) { + auto* operation = work_list.pop_back_val(); + erase_list.insert(operation); + for (auto& use : operation->getUses()) { + if (erase_list.count(use.getOwner())) continue; + work_list.push_back(use.getOwner()); + } + } + + for (auto* op : erase_list) { + op->dropAllUses(); + op->erase(); + } +} + +// Validates func ops. Returns `failure` if the function is invalid. +LogicalResult ValidateFuncOp(func::FuncOp func_op) { + auto input_shapes_attr = + func_op->getAttrOfType(kTfInputShapesAttr); + if (!input_shapes_attr) return success(); + + if (input_shapes_attr.size() != func_op.getNumArguments()) { + return func_op->emitError( + "Number of arguments and 'tf._input_shapes' " + "attribute size do not match. ") + << "Num args: " << func_op.getNumArguments() + << ", tf._input_shapes size: " << input_shapes_attr.size(); + } + + return success(); +} + +// Validates ModuleOp. Returns `failure` if the module op is invalid. +LogicalResult ValidateModule(ModuleOp module_op) { + for (auto func_op : module_op.getOps()) { + if (failed(ValidateFuncOp(func_op))) { + return failure(); + } + } + return success(); +} + +} // namespace + +LogicalResult FreezeVariables(ModuleOp module, tensorflow::Session* session) { + if (failed(ValidateModule(module))) return failure(); + + const tensorflow::DeviceMgr* mgr = nullptr; + auto status = session->LocalDeviceManager(&mgr); + if (!status.ok()) { + module->emitError( + absl::StrCat("failed to fetch device manager: ", status.message())); + return failure(); + } + + SmallVector session_init_funcs = + tf_saved_model::GetInitializerFunctions(module); + func::FuncOp session_init_func = + session_init_funcs.empty() ? nullptr : session_init_funcs[0]; + + TF::ResourceAnalyzer analyzer(module, /*skip_session_init=*/true); + llvm::SmallVector variables; + // Capture list of all read only variables. + for (auto func : module.getOps()) { + if (func == session_init_func) continue; + for (auto var_handle_op : func.getOps()) { + if (!analyzer.IsPotentiallyWritten(var_handle_op.getResource())) { + variables.push_back(var_handle_op); + } + } + } + + // Fetch the values to replace the VarHandleOps with. + auto resource_tensors_or = + tf_saved_model::GetResourcesFromSession(variables, session); + if (!resource_tensors_or.ok()) { + module->emitError(resource_tensors_or.status().message().data()); + return failure(); + } + + auto* context = module.getContext(); + OpBuilder builder(context); + // Note: We can't modify the graph while navigating through it, as erasing + // invalidate pointers. + // So instead we capture all the updates in the below map, and then + // process them after. + + // Container to hold all update actions on ops. + // Key: Operation to update. + // Value: optional list of argument indices to delete from this op. + // Note that we use MapVector because we want to iterate on the same order + // of insertion. + llvm::MapVector> + arguments_to_erase; + for (auto [var_handle_op, resource_tensor] : + llvm::zip(variables, resource_tensors_or.value())) { + builder.setInsertionPointAfterValue(var_handle_op); + auto elements_attr = GetTensorValueAsElementsAttr( + var_handle_op, resource_tensor, mgr, builder); + if (failed(ReplaceVarWithConstant(var_handle_op.getResource().getUses(), + elements_attr, &arguments_to_erase))) { + return failure(); + } + } + + if (failed(EraseObsoleteResourceUses(arguments_to_erase))) { + return failure(); + } + + // Remove initialization of unused variables. + if (session_init_func) + RemoveVariablesInitializations(variables, session_init_func); + + // Remove the unused VarHandleOp. + for (auto var_handle_op : variables) { + if (var_handle_op) var_handle_op->erase(); + } + return success(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h new file mode 100644 index 0000000000000000000000000000000000000000..b122b6258de77c8b5c9ece13ed4cecb637fae59f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h @@ -0,0 +1,33 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_VARIABLES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_VARIABLES_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Freezes readonly variables in the graph. +LogicalResult FreezeVariables(ModuleOp module, tensorflow::Session* session); + +} // namespace tf_saved_model + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_FREEZE_VARIABLES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables_test_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables_test_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..8b53878edf34b2bee35071d2933a0de3ece679a9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables_test_pass.cc @@ -0,0 +1,44 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_freeze_variables.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/fake_session.h" + +namespace mlir { +namespace tf_test { +namespace { + +#define GEN_PASS_DEF_FREEZEVARIABLESTESTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h.inc" + +struct FreezeVariableTestPass + : public impl::FreezeVariablesTestPassBase { + void runOnOperation() override { + TF::test_util::FakeSession session; + if (tf_saved_model::FreezeVariables(getOperation(), &session).failed()) + signalPassFailure(); + } +}; +} // namespace + +std::unique_ptr> CreateFreezeVariableTestPass() { + return std::make_unique(); +} +} // namespace tf_test +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..2d0cf38144cdf0860a0cc1844a64abdf5f7a230e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h @@ -0,0 +1,92 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Creates a pass that optimizes tf_saved_model.global_tensor ops. +std::unique_ptr> CreateOptimizeGlobalTensorsPass(); + +// Creates a pass that freezes tf_saved_model.global_tensor ops. +std::unique_ptr> CreateFreezeGlobalTensorsPass( + bool allow_mutable_tensors = false); + +// Creates a pass that freezes tf_saved_model.asset ops. +std::unique_ptr> CreateFreezeAssetsPass( + std::string saved_model_dir = ""); + +// Creates a pass that unfreezes mutable global tensors. +std::unique_ptr> +CreateUnfreezeMutableGlobalTensorsPass(); + +// Creates as pass that removes variables in the session initializer. +// This job is required with lifting variable passes. Originally, the session +// initializer function does assigning variables. However, the read-only +// variable assignments will be done via lifting variables pass by converting +// the read-only variables to constant ops, instead. This pass removes the +// redundant operations. This pass should be located in front of the pass for +// lifting read-only variables. +std::unique_ptr> +CreateRemoveVariablesInSessionInitializerPass(); + +// Creates a pass that removes duplicate 'tf_saved_model.bound_input' bindings. +std::unique_ptr> CreateDedupBoundInputBindingPass(); + +// Create a pass that removes function arguments that map to global tensors. +std::unique_ptr CreateLowerGlobalsToMlProgramPass(); + +// Create a pass that lowers variable read/write ops to ml_program ops. +std::unique_ptr> +CreateLowerVariableOpsToMlProgramPass(); + +// Strips saved_model attributes from a module and its functions. +std::unique_ptr> CreateStripSavedModuleMetadataPass(); + +// Convert the session initializer to a function. +std::unique_ptr> +CreateConvertSessionInitializerToFunctionPass(); + +// Creates forwarding functions for 'exported_names'. +std::unique_ptr> +CreateAddFunctionsForExportedNamesPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_DEDUPBOUNDINPUTBINDINGPASS +#define GEN_PASS_DECL_FREEZEASSETSPASS +#define GEN_PASS_DECL_FREEZEGLOBALTENSORSPASS +#define GEN_PASS_DECL_LOWERGLOBALSTOMLPROGRAMPASS +#define GEN_PASS_DECL_LOWERVARIABLEOPSTOMLPROGRAMPASS +#define GEN_PASS_DECL_OPTIMIZEGLOBALTENSORSPASS +#define GEN_PASS_DECL_REMOVEVARIABLESINSESSIONINITIALIZERPASS +#define GEN_PASS_DECL_STRIPSAVEDMODULEMETADATAPASS +#define GEN_PASS_DECL_ADDFUNCTIONSFOREXPORTEDNAMESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" + +} // namespace tf_saved_model + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_TF_SAVED_MODEL_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..5c11165d873f7e9111d4e9df2d40812625ef02bf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.td @@ -0,0 +1,202 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +// TF SavedModel dialect passes. + +def FreezeGlobalTensorsPass : Pass<"tf-saved-model-freeze-global-tensors", "ModuleOp"> { + let summary = "Freeze tf_saved_model.global_tensor's in func bodies."; + + let description = [{ + This pass will replace a func's bound inputs which are bound to + tf.ReadVariable ops global tensors with tf.Const ops inside the func's body. + If this pass runs successfully, the resultant IR will be guaranteed to: + + 1. Not contain any tf_saved_model.global_tensor ops + 2. Not contain any tf_saved_model.bound_input arg attrs on tf_saved_model + exported functions + Else, the pass fails. + + The reason this pass has this contract is so that once this succeeds, we know + the IR is in correct form for inference backends (like lite) that do not + support resources/variables. Further, this contract also ensures that this + pass lowers from saved model to pure TF. Hence it fails, if it cannot lower. + }]; + + let constructor = "::mlir::tf_saved_model::CreateFreezeGlobalTensorsPass()"; + + let options = [ + Option<"allow_mutable_tensors", "allow-mutable-tensors", "bool", /*default=*/"false", + "Allows mutable tensors to be in the graph."> + ]; +} + +def UnfreezeMutableGlobalTensorsPass : Pass<"tf-saved-model-unfreeze-mutable-global-tensors", "ModuleOp"> { + let summary = "Unfreeze mutable global tensors."; + + let description = [{ + This pass "unfreezes" the use of global constant tensor ops found in the + module and converts them to tf.VarHandleOps. Also, an initialization pattern + tf.AssignVariableOp(tf.VarHandleOp, tf.ConstOp) is inserted to the + initializer function of type "init_op" for each of the unfrozen constants. + }]; + + let constructor = "::mlir::tf_saved_model::CreateUnfreezeMutableGlobalTensorsPass()"; +} + +def OptimizeGlobalTensorsPass : Pass<"tf-saved-model-optimize-global-tensors", "ModuleOp"> { + let summary = "Optimize tf_saved_model.global_tensor's."; + + let constructor = "mlir::tf_saved_model::CreateOptimizeGlobalTensorsPass()"; +} + +def RemoveVariablesInSessionInitializerPass : Pass<"tf-saved-model-remove-vars-in-session-initializer", "ModuleOp"> { + let summary = "Remove variables in tf saved model's session initializer."; + + let constructor = "mlir::tf_saved_model::CreateRemoveVariablesInSessionInitializerPass()"; +} + +def FreezeAssetsPass : Pass<"tf-saved-model-freeze-assets", "ModuleOp"> { + let summary = "Freeze tf_saved_model.asset's in func bodies."; + let constructor = "mlir::tf_saved_model::CreateFreezeAssetsPass()"; +} + +def DedupBoundInputBindingPass : Pass<"tf-saved-model-dedup-bound-input-binding-pass", "mlir::func::FuncOp"> { + let summary = "Remove duplicate 'tf_saved_model.bound_input' bindings."; + let constructor = "mlir::tf_saved_model::CreateDedupBoundInputBindingPass()"; +} + +def LowerGlobalsToMlProgramPass: Pass<"tf-saved-model-lower-globals-to-mlprogram", "mlir::ModuleOp"> { + let summary = "Remove (and remap) function arguments that map to global tensors."; + let description = [{ + This pass will lower AssignVariableOp and ReadVariableOp to their ml_program + equivalents, for those resources which stem from a "bound input" function argument + that maps to a GlobalTensorOp. The arguments in question will be removed. + + So this will lower e.g. + + func @f(%arg0 {tf_saved_model.bound_input = @v}) { + %0 = "tf.ReadVariableOp"(%arg0) + ... + } + + to + + func @f() { + %0 = ml_program.global_load @v + ... + } + }]; + let constructor = "mlir::tf_saved_model::CreateLowerGlobalsToMlProgramPass()"; +} + +def LowerVariableOpsToMlProgramPass: Pass<"tf-saved-model-lower-variable-ops-to-mlprogram", "mlir::ModuleOp"> { + let summary = "Lower tf.ReadVariable and tf.AssignVariable"; + + let description = [{ + This pass will lower AssignVariableOp and ReadVariableOp to their ml_program + equivalents, for those resources which come from a VarHandle op, but also + for those which come from a "bound input" parameter. For the latter, + said parameter will not be removed, unlike LowerGlobalsToMlProgramPass. + + So this will lower e.g. + + %0 = "tf.VarHandleOp" {name = "foo"} + %1 = "tf.ReadVariableOp"(%0) + + to + + %0 = ml_program.global_load @vars.foo + }]; + + let constructor = "::mlir::tf_saved_model::CreateLowerVariableOpsToMlProgramPass()"; +} + +def StripSavedModuleMetadataPass : Pass<"tf-strip-saved-module-metadata", "ModuleOp"> { + let summary = "Removes saved_model attributes."; + let description = [{ + Removes all saved_model attributes from a module and its functions. + This only removes attributes, saved_model ops will stay in place. + + This pass is useful for pass pipelines that have already consumed + all the tf_saved_model information, and now want to do further + transformations without tf_saved_model's verifiers kicking in. + + For example, this would remove the semantics attribute from module: + + module attributes {tf_saved_model.semantics} [...] + + so the result becomes just + + module [...] + }]; + let constructor = "::mlir::tf_saved_model::CreateStripSavedModuleMetadataPass()"; +} + +def ConvertSessionInitializerToFunctionPass : Pass<"tf-saved-model-convert-session-initializer-to-function", "ModuleOp"> { + let summary = "Converts the session initializer to a function."; + let description = [{ + This converts + "tf_saved_model.session_initializer"() {initializers = [@a, @b, @c]} : () -> () + to + func.func @session_initializer() { + call @a() : () -> () + call @b() : () -> () + call @c() : () -> () + return + } + }]; + let constructor = "::mlir::tf_saved_model::CreateConvertSessionInitializerToFunctionPass()"; +} + +def AddFunctionsForExportedNamesPass : Pass<"tf-saved-model-add-functions-for-exported-names", "ModuleOp"> { + let summary = "Create a trampoline function for each exported name."; + let description = [{ + This converts + func.func @foo() attributes {tf_saved_model.exported_names = ["a", "b", "c"]} + to + func.func @a() {return @foo_internal()} + func.func @b() {return @foo_internal()} + func.func @c() {return @foo_internal()} + func.func private @foo_internal() + }]; + let constructor = "::mlir::tf_saved_model::CreateAddFunctionsForExportedNamesPass()"; +} + +def AssetSinkingPass : Pass<"tf-saved-model-asset-sinking", "mlir::ModuleOp"> { + let summary = "Sinks SavedModel asset filenames to constants"; + + let description = [{ + This pass sinks arguments of SavedModel methods that are bounded to + `tf_saved_model.asset` into constants in the methods. After the pass, unused + asset ops are removed from the module. + + This is to convert initialization methods with bound inputs into the same + methods without any arguments, so that program invocation doesn't need to + track and explicitly pass asset filenames. + + This pass accepts an option `saved-model-dir`, which specifies the directory + where SavedModel is stored. This is a required option because all asset + filenames are relative to this directory. + }]; + + let constructor = "::mlir::tf_saved_model::CreateAssetSinkingPass(\"\")"; + + let options = [ + Option<"saved_model_dir_", "saved-model-dir", "std::string", "", + "SavedModel directory, which is prepended to asset file names.">, + ]; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_test_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_test_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..b8c4bf9729b5cc1fc735374ea9a35aa159ad8d4e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tf_test_passes.td @@ -0,0 +1,106 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +// TF SavedModel dialect test passes. + +def FreezeVariablesTestPass : Pass<"tf-freeze-variables-test-pass", + "ModuleOp"> { + let summary = "Freezes read only variables"; + + let description = [{ + Test pass for freezing read only variables in the graph. + }]; + + let constructor = "::mlir::tf_test::CreateFreezeVariableTestPass()"; +} + +def InitTextFileToImportTestPass : Pass<"tf-init-text-file-to-import-test", + "ModuleOp"> { + let summary = "generate a temporary file and invoke InitTextFileToImportPass"; + let constructor = "::mlir::tf_test::CreateInitTextFileToImportTestPass()"; +} + +def InitTextFileToImportSavedModelTestPass : Pass<"tf-init-text-file-to-import-saved-model-test", + "ModuleOp"> { + let summary = "mimick a saved model and invoke InitTextFileToImportPass"; + let constructor = "::mlir::tf_test::CreateInitTextFileToImportSavedModelTestPass()"; +} + +def TestTensorFlowLowerTFPass : Pass<"test-tf-lower-tf", "mlir::func::FuncOp"> { + let summary = "Test pass for TF->TF lowerings"; + let constructor = "::mlir::tf_test::CreateTestTFLowerTFPass()"; + + let options = [ + Option<"default_patterns_", "default-patterns", "bool", /*default=*/"true", + "Activate the default TF->TF patterns">, + Option<"pre_hlo_patterns_", "pre-hlo-patterns", "bool", /*default=*/"true", + "Activate the pre-HLO TF->TF patterns">, + ]; +} + +def TestResourceAliasAnalysis : Pass<"tf-test-resource-alias-analysis", "mlir::func::FuncOp"> { + let summary = "Add remarks based on resource alias analysis result, for " + "testing purpose."; + let constructor = "::mlir::tf_test::CreateTestResourceAliasAnalysisPass()"; +} + +def TestClusteringPolicyPass : Pass<"tf-test-clustering-policy", "mlir::func::FuncOp"> { + let summary = "Test pass for clustering based on the user defined policy"; + let constructor = "::mlir::tf_test::CreateTestClusteringPolicyPass()"; +} + +def TestSideEffectAnalysisPass : Pass<"tf-test-side-effect-analysis", "ModuleOp"> { + let summary = "Test pass for analyzing side-effect analysis result"; + let description = [{ + Adds "Predecessors" and "Successors" remarks for each op based on + the SideEffectAnalysis result. For testing purpose only. + }]; + let constructor = "::mlir::tf_test::CreateTestSideEffectAnalysisPass()"; +} + +def LiftVariablesInvalidSessionTestPass : Pass<"tf-saved-model-lift-variables-invalid-session-test", "ModuleOp"> { + let summary = "Lift variables and save them as global tensors with an invalid " + "session"; + let constructor = "mlir::tf_test::CreateLiftVariablesInvalidSessionTestPass()"; +} + +def LiftVariablesTestPass : Pass<"tf-saved-model-lift-variables-test", "ModuleOp"> { + let summary = "Lift variables and save them as global tensors"; + let constructor = "mlir::tf_test::CreateLiftVariablesTestPass()"; + + let options = [ + Option<"import_variables_as_dense_resources_", "import-variables-as-dense-resources", "bool", /*default=*/"false", + "Import variables as dense resources">, + ]; +} + +def InitializeVariablesInSessionInitializerPass : Pass<"tf-saved-model-initialize-variables-in-session-init", "ModuleOp"> { + let summary = "Initialize variables in session initializer function."; + + let constructor = "mlir::tf_test::CreateInitializeVariablesInSessionInitializerTestPass()"; +} + +def ResourceAnalyzerTestPass : Pass<"tf-resource-analyzer-test", "ModuleOp"> { + let summary = "Test pass for resource analysis."; + + let description = [{ + Test pass for ResourceAnalyzer, which analyzes whether a resource value is + potentially written. + }]; + + let constructor = "::mlir::tf_test::CreateResourceAnalyzerTestPass()"; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tfg-to-tfe.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tfg-to-tfe.cc new file mode 100644 index 0000000000000000000000000000000000000000..0a44b51624e047416822d1a6fa28936bfae25527 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tfg-to-tfe.cc @@ -0,0 +1,581 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/core/transforms/toposort/pass.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TF { +namespace { + +// FIXME: This should be consistent with +// tensorflow::kImportModelDefaultGraphFuncName +static const char kImportModelDefaultGraphFuncName[] = "main"; + +// Please refer to the TFG dialect description for the list of used attributes. +// Belows are the attributes in TFE. +// TFE Arguments and Results (Got from "_Arg", +// "_Retval", .etc) +// NodeDef.device <-> "tf.device" +// NodeDef.attr <-> "tf." +// +// TFE general operations +// NodeDef.device <-> "device" +// +// The following two functions are only used for mapping/excluding attributes +// which are inconsistent between TFG and TFE. +// +static mlir::LogicalResult FilterTfgSpecificArgResultAttributes( + mlir::MLIRContext *context, mlir::ArrayRef types, + mlir::ArrayAttr array_attr, llvm::SmallVector &output_types, + llvm::SmallVector &output_attrs) { + for (auto it : llvm::zip( + types, array_attr.template getAsRange())) { + if (mlir::isa(std::get<0>(it))) continue; + output_types.push_back(std::get<0>(it)); + + mlir::NamedAttrList list; + for (mlir::NamedAttribute attr : std::get<1>(it).getValue()) { + // Skip if the attribute has "tfg" prefix. + if (attr.getName().getValue().starts_with("tfg")) continue; + list.append(attr); + } + output_attrs.push_back(list.getDictionary(context)); + } + return mlir::success(); +} + +static mlir::LogicalResult ReformatOpAttributes( + mlir::MLIRContext *context, llvm::ArrayRef attrs, + llvm::SmallVectorImpl &output) { + for (mlir::NamedAttribute attr : attrs) { + if (attr.getName().strref().contains( + mlir::tfg::TFGraphDialect::getDeviceAttrKey())) { + tensorflow::DeviceNameUtils::ParsedName parsed_name; + if (!tensorflow::DeviceNameUtils::ParseFullName( + mlir::cast(attr.getValue()).getValue().str(), + &parsed_name)) + return mlir::failure(); + if (!parsed_name.has_type) { + parsed_name.type = "CPU"; + parsed_name.has_type = true; + } + if (!parsed_name.has_id) { + parsed_name.id = 0; + parsed_name.has_id = true; + } + output.push_back(mlir::NamedAttribute( + mlir::StringAttr::get(context, "device"), + mlir::StringAttr::get( + context, + tensorflow::DeviceNameUtils::ParsedNameToString(parsed_name)))); + } else { + output.push_back(attr); + } + } + return mlir::success(); +} + +static void FilterOutBlockArgControlDep( + ValueRange operands, llvm::SmallVectorImpl &filtered) { + for (Value value : operands) + if (!mlir::isa(value)) filtered.push_back(value); +} + +// Split the tfg.NextIteration into tf_executor::NextIterationSourceOp and +// tf_executor::NextIterationSinkOp to break the cycle introduced by itself. +static void SplitNextIteration(Block &block) { + // TODO(b/207144333): Supports callback for unregistered ops + block.walk([&](Operation *op) { + if (op->getName().getStringRef() != "tfg.NextIteration") return; + mlir::OpBuilder builder(op); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(op->getOperands().drop_front(), new_operands); + + auto source_op = tf_executor::NextIterationSourceOp::create( + builder, op->getLoc(), op->getOperand(0).getType()); + tf_executor::NextIterationSinkOp::create(builder, op->getLoc(), + source_op.getToken(), + /*input=*/op->getOperand(0), + /*controlInputs=*/new_operands); + op->replaceAllUsesWith( + ValueRange({source_op.getOutput(), source_op.getControl()})); + op->erase(); + }); +} + +class ConvertGraphOp : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + tfg::GraphOp graph, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const final { + Location loc = graph.getLoc(); + // To keep the import-as-graph logic taken by TFG, we create `void func()` + // to contain the ops in the tfg::GraphOp. That means the arguments/results + // will be the operations inside the function body rather than representing + // them in the function signature. + FunctionType func_type = rewriter.getFunctionType({}, {}); + func::FuncOp func = func::FuncOp::create( + rewriter, loc, kImportModelDefaultGraphFuncName, func_type); + rewriter.setInsertionPointToStart(func.addEntryBlock()); + auto executor_graph = + tf_executor::GraphOp::create(rewriter, loc, func_type.getResults()); + rewriter.inlineRegionBefore(graph.getNodes(), executor_graph.getBody(), + executor_graph.getBody().end()); + + // Add terminator of tf_executor::graph + rewriter.setInsertionPointToEnd(&executor_graph.getBody().front()); + tf_executor::FetchOp::create(rewriter, loc); + + // Add terminator of func + rewriter.setInsertionPointToEnd(&func.getBody().front()); + func::ReturnOp::create(rewriter, loc); + + rewriter.replaceOp(graph.getOperation(), func.getOperation()->getResults()); + + return success(); + } +}; + +class ConvertGraphFuncOp : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + tfg::GraphFuncOp graph_func, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const final { + assert(!graph_func.getGeneric()); + Location loc = graph_func.getLoc(); + FunctionType ftype = graph_func.getFunctionType(); + + func::FuncOp func = func::FuncOp::create( + rewriter, graph_func.getLoc(), + graph_func->getAttrOfType(SymbolTable::getSymbolAttrName()) + .getValue(), + ftype); + + func->setAttrs(graph_func->getAttrs()); + + llvm::SmallVector arg_types; + llvm::SmallVector res_types; + llvm::SmallVector arg_attrs; + llvm::SmallVector res_attrs; + if (failed(FilterTfgSpecificArgResultAttributes( + getContext(), ftype.getInputs(), graph_func.getAllArgAttrs(), + arg_types, arg_attrs)) || + failed(FilterTfgSpecificArgResultAttributes( + getContext(), ftype.getResults(), graph_func.getAllResultAttrs(), + res_types, res_attrs))) + return failure(); + + // Update the function type which has excluded the control args. + func->setAttr("function_type", TypeAttr::get(rewriter.getFunctionType( + arg_types, res_types))); + + // Update arg/result attributes. + func.setAllArgAttrs(arg_attrs); + func.setAllResultAttrs(res_attrs); + + rewriter.setInsertionPointToStart(func.addEntryBlock()); + // In TFE, the function body is inlined in a GraphOp. Create a GraphOp + // instance and move the regions from GraphFuncOp to GraphOp. + auto executor_graph = tf_executor::GraphOp::create( + rewriter, loc, func.getFunctionType().getResults()); + + // Replace the uses of block arguments with function arguments. Note that we + // can't erase the arguments here because the operations may still use them + // and these uses will be dropped after legalization of each op. + unsigned idx = 0; + Block &block = graph_func.getBody().front(); + for (auto iter = block.args_begin(), end_iter = block.args_end(); + iter != end_iter; ++iter) { + if (!mlir::isa(iter->getType())) + iter->replaceAllUsesWith(func.getBody().getArgument(idx++)); + } + + rewriter.inlineRegionBefore(graph_func.getBody(), executor_graph.getBody(), + executor_graph.getBody().end()); + + rewriter.setInsertionPointToEnd(&func.getBody().front()); + func::ReturnOp::create(rewriter, loc, + executor_graph.getOperation()->getResults()); + + rewriter.replaceOp(graph_func.getOperation(), + func.getOperation()->getResults()); + + return success(); + } +}; + +class ConvertReturnOp : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite( + tfg::ReturnOp ret, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const final { + rewriter.replaceOpWithNewOp(ret.getOperation(), + adaptor.getOperands()); + return success(); + } +}; + +class ConvertControlTriggerOp : public ConversionPattern { + public: + explicit ConvertControlTriggerOp(MLIRContext *context) + : ConversionPattern("tfg.ControlTrigger", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertEnterOp : public ConversionPattern { + public: + explicit ConvertEnterOp(MLIRContext *context) + : ConversionPattern("tfg.Enter", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertExitOp : public ConversionPattern { + public: + explicit ConvertExitOp(MLIRContext *context) + : ConversionPattern("tfg.Exit", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertLoopCondOp : public ConversionPattern { + public: + explicit ConvertLoopCondOp(MLIRContext *context) + : ConversionPattern("tfg.LoopCond", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertMergeOp : public ConversionPattern { + public: + explicit ConvertMergeOp(MLIRContext *context) + : ConversionPattern("tfg.Merge", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertSwitchOp : public ConversionPattern { + public: + explicit ConvertSwitchOp(MLIRContext *context) + : ConversionPattern("tfg.Switch", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertSwitchNOp : public ConversionPattern { + public: + explicit ConvertSwitchNOp(MLIRContext *context) + : ConversionPattern("tfg.SwitchN", PatternBenefit(1), context) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + llvm::SmallVector new_types(op->getResultTypes()); + new_types.back() = rewriter.getType(); + + llvm::SmallVector new_operands; + FilterOutBlockArgControlDep(operands, new_operands); + + rewriter.replaceOpWithNewOp( + op, new_types, new_operands, op->getAttrs()); + return success(); + } +}; + +class ConvertGeneralOp : public ConversionPattern { + public: + ConvertGeneralOp(MLIRContext *context, + const DenseSet &func_symbols) + : ConversionPattern(MatchAnyOpTypeTag(), PatternBenefit(1), context), + func_symbols_(func_symbols) {} + + LogicalResult matchAndRewrite( + Operation *op, llvm::ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + if (!llvm::isa(op->getDialect())) return failure(); + + Location loc = op->getLoc(); + llvm::SmallVector new_types(op->getResultTypes()); + // Update the control type from tf_type.control to tf_executor.control. + new_types.back() = rewriter.getType(); + + // Control operand is attached on tf_executor::IslandOp. + llvm::SmallVector island_control_operands; + llvm::SmallVector inner_op_operands; + + for (Value value : operands) { + // Because of the property of graph region, the control operands may + // not have been converted to tf_executor::ControlType. + if (mlir::isa(value.getType()) || + mlir::isa(value.getType())) { + if (!mlir::isa(value)) + island_control_operands.push_back(value); + } else { + inner_op_operands.push_back(value); + } + } + + auto island = tf_executor::IslandOp::create(rewriter, loc, new_types, + island_control_operands); + island.getBody().push_back(new mlir::Block); + + rewriter.setInsertionPointToEnd(&island.getBody().front()); + + // Control dependency has been applied on tf_executor.island. Remove it + // while creating the tf operations. + new_types.pop_back(); + + llvm::SmallVector, 1> new_regions; + for (auto ®ion : op->getRegions()) { + new_regions.push_back(std::make_unique()); + new_regions.back()->takeBody(region); + } + + llvm::SmallVector attrs; + if (failed(ReformatOpAttributes(getContext(), op->getAttrs(), attrs))) + return failure(); + + Operation *inner_op; + + StringRef op_name = op->getName().stripDialect(); + if (!func_symbols_.contains(op_name)) { + std::string tf_op_name = llvm::formatv( + "{0}.{1}", TF::TensorFlowDialect::getDialectNamespace(), op_name); + OperationState state = + OperationState(loc, tf_op_name, inner_op_operands, new_types, attrs, + op->getSuccessors(), new_regions); + inner_op = rewriter.create(state); + } else { + bool disable_call_shape_inference = false; + if (op->hasAttr("_disable_call_shape_inference")) { + disable_call_shape_inference = + op->getAttrOfType("_disable_call_shape_inference") + .getValue(); + } + inner_op = LegacyCallOp::create( + rewriter, loc, new_types, inner_op_operands, + /*args_attrs=*/nullptr, + /*res_attrs=*/nullptr, op_name, disable_call_shape_inference); + } + + tf_executor::YieldOp::create(rewriter, loc, inner_op->getResults()); + + rewriter.replaceOp(op, island.getOperation()->getResults()); + + return success(); + } + + private: + const DenseSet &func_symbols_; +}; + +#define GEN_PASS_DEF_LEGALIZETFGTOTFPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class LegalizeTFGToTFE + : public impl::LegalizeTFGToTFPassBase { + void getDependentDialects(DialectRegistry ®istry) const override { + RegisterAllTensorFlowDialects(registry); + } + + void runOnOperation() override; +}; + +} // namespace + +void LegalizeTFGToTFE::runOnOperation() { + MLIRContext &context = getContext(); + ModuleOp module = getOperation(); + + DenseSet func_symbols; + for (auto &op : module.getBodyRegion().getOps()) { + if (auto func = llvm::dyn_cast(op)) { + func_symbols.insert( + func->getAttrOfType(SymbolTable::getSymbolAttrName()) + .getValue()); + } + } + + ConversionTarget target(context); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalOp(); + target.addLegalOp(); + target.addLegalOp(); + + RewritePatternSet patterns(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context, func_symbols); + // Control flow V1 operation conversion patterns. + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + patterns.add(&context); + FrozenRewritePatternSet finalPatterns(std::move(patterns)); + + // Turn the graph region into SSACFG region by applying an order to the + // operations. + for (auto &op : module.getBodyRegion().getOps()) { + for (auto ®ion : op.getRegions()) { + for (auto &block : region) { + // Split tfg.NextIteration to break the cycle. + SplitNextIteration(block); + tfg::SortTopologically(&block); + } + } + } + + // Version information is embedded in graph operation in TFG. In TFE, it's + // embedded in the module operation. + for (auto &op : module.getBodyRegion().getOps()) { + auto graph = dyn_cast(op); + if (!graph) continue; + Builder b(&context); + auto producer = b.getNamedAttr( + "producer", b.getI32IntegerAttr(graph.getVersion().getProducer())); + auto min_consumer = b.getNamedAttr( + "min_consumer", + b.getI32IntegerAttr(graph.getVersion().getMinConsumer())); + auto bad_consumers = + b.getNamedAttr("bad_consumers", + b.getI32ArrayAttr(graph.getVersion().getBadConsumers())); + module->setAttr("tf.versions", + b.getDictionaryAttr(llvm::ArrayRef( + {producer, min_consumer, bad_consumers}))); + break; + } + + if (failed(applyFullConversion(module.getOperation(), target, finalPatterns))) + signalPassFailure(); + + // The uses of arg control dependency has been dropped. We can safely remove + // the block argument here. + module.walk([&](tf_executor::GraphOp graph) { + graph.getBody().front().eraseArguments( + [](BlockArgument arg) { return true; }); + }); +} + +std::unique_ptr CreateLegalizeTFGToTFEPass() { + return std::make_unique(); +} + +} // end namespace TF +} // end namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_annotate_dynamic_shape_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_annotate_dynamic_shape_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..73be65e7c6fb2bd1614ef73036318c86eebcc038 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_annotate_dynamic_shape_inputs.cc @@ -0,0 +1,161 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +#define DEBUG_TYPE "tf-tpu-annotate-dynamic-shape-inputs" + +namespace mlir { +namespace TFTPU { + +namespace { + +#define GEN_PASS_DEF_TPUANNOTATEDYNAMICSHAPEINPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class TPUAnnotateDynamicShapeInputsPass + : public impl::TPUAnnotateDynamicShapeInputsPassBase< + TPUAnnotateDynamicShapeInputsPass> { + void runOnOperation() override; +}; + +// Finds op that created a given value. If the value is a BlockArgument, this +// returns the owner of the Block. +Operation* GetOpOfValue(Value value) { + if (auto block_arg = mlir::dyn_cast(value)) + return block_arg.getOwner()->getParentOp(); + + return value.getDefiningOp(); +} + +void TPUAnnotateDynamicShapeInputsPass::runOnOperation() { + getOperation().walk([&](tf_device::ClusterFuncOp cluster_func_op) { + Builder builder(cluster_func_op->getContext()); + // Skip non-tpu device cluster_func. + auto cluster_id = + cluster_func_op->getAttrOfType(TF::kReplicationInfoAttr); + if (!cluster_id) return WalkResult::advance(); + + llvm::SmallVector dynamic_shape_arg_index; + + // Traverse the operands of the cluster func op and find which operand + // is returned by TPUAnnotateTensorsWithDynamicShapeOp. + for (const auto& cluster_func_operand : + llvm::enumerate(cluster_func_op.getOperands())) { + auto device_launch_op = llvm::dyn_cast( + GetOpOfValue(cluster_func_operand.value())); + if (!device_launch_op) continue; + for (auto result : llvm::zip( + device_launch_op.getResults(), + device_launch_op.GetBody().getTerminator()->getOperands())) { + if (std::get<0>(result) == cluster_func_operand.value() && + llvm::isa( + std::get<1>(result).getDefiningOp())) { + dynamic_shape_arg_index.push_back(cluster_func_operand.index()); + } + } + } + + cluster_func_op->setAttr(TF::kDynamicArgIndexAttr, + builder.getI32ArrayAttr(dynamic_shape_arg_index)); + + FlatSymbolRefAttr func_attr = cluster_func_op.getFuncAttr(); + func::FuncOp func = + cluster_func_op->getParentOfType().lookupSymbol( + func_attr.getValue()); + + // Update the marked argument with dynamic shapes. + for (int index : dynamic_shape_arg_index) { + BlockArgument arg = func.getArgument(index); + auto inputType = mlir::dyn_cast(arg.getType()); + // Only rank 1 tensor is supported for now. + if (!inputType || inputType.getRank() != 1) continue; + auto shape = llvm::to_vector<4>(inputType.getShape()); + llvm::SmallVector bounds(shape.begin(), shape.end()); + // Mark the dim as dynamic dim. + shape[0] = ShapedType::kDynamic; + auto extensions = + mhlo::TypeExtensionsAttr::get(func->getContext(), bounds); + auto resultType = + RankedTensorType::get(shape, inputType.getElementType(), extensions); + arg.setType(resultType); + } + llvm::SmallVector arg_types; + for (auto arg : func.getArguments()) arg_types.push_back(arg.getType()); + func.setType( + FunctionType::get(func.getContext(), arg_types, + func.front().getTerminator()->getOperandTypes())); + return WalkResult::advance(); + }); + + // Remove the annotated op after since it is just a placeholder. + DenseSet launch_ops; + getOperation().walk([&](Operation* op) { + if (llvm::isa(op)) { + for (auto result : llvm::zip(op->getOperands(), op->getResults())) { + std::get<1>(result).replaceAllUsesWith(std::get<0>(result)); + } + launch_ops.insert(op->getParentOfType()); + op->erase(); + } + return WalkResult::advance(); + }); + + for (auto launch_op : launch_ops) { + Block& block = launch_op.GetBody(); + if (&block.front() == &block.back()) { + // The tf_device.launch is empty (except for the return). + // Remove the whole tf_device.launch, since later passes will make it send + // the arguments back and forth between the devices. + Operation* return_op = &block.back(); + assert(llvm::isa(return_op)); + for (auto [inner, outer] : + llvm::zip(return_op->getOperands(), launch_op->getResults())) { + outer.replaceAllUsesWith(inner); + } + launch_op->erase(); + } + } +} + +} // namespace + +std::unique_ptr> +CreateTPUAnnotateDynamicShapeInputsPass() { + return std::make_unique(); +} +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_cleanup_attributes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_cleanup_attributes.cc new file mode 100644 index 0000000000000000000000000000000000000000..39f0676ff7ecb99fb86938953e81b7295710561a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_cleanup_attributes.cc @@ -0,0 +1,80 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDeviceAttr[] = "device"; +constexpr char kClassAttr[] = "_class"; + +#define GEN_PASS_DEF_TPUCLEANUPCLUSTERATTRIBUTESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class TPUCleanupClusterAttributesPass + : public impl::TPUCleanupClusterAttributesPassBase< + TPUCleanupClusterAttributesPass> { + public: + void runOnOperation() override { + auto traverse_op = [&](Operation* op, tf_device::ClusterOp tpu_cluster) { + if (isa(op)) return WalkResult::advance(); + op->removeAttr(TF::kReplicationInfoAttr); + op->removeAttr(TF::kCompileDeviceTypeAttr); + // This attribute is used for op colocation. Since all ops are located + // on a single device cluster, this private attribute is no longer + // needed. + op->removeAttr(kClassAttr); + if (auto attr = op->getAttrOfType(kDeviceAttr)) { + // Preserve device attribute if the op is placed on a replicated core + // device. Device attribute is used to infer the appropriate sharding + // within TPUs for this op. + // TODO(b/183598857): Use explicit sharding ops from the front-end. + // For example, dequeue ops generated by + // tensorflow/python/tpu/tpu_feed.py + if (!tensorflow::IsTPUReplicatedCore(attr.getValue()) && + !isa(op)) { + op->removeAttr(kDeviceAttr); + } + } + return WalkResult::advance(); + }; + + if (failed(TFTPU::WalkReachableFromTpuCluster(getOperation(), traverse_op))) + return signalPassFailure(); + } +}; + +} // namespace + +std::unique_ptr> +CreateTPUClusterCleanupAttributesPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_composite_resource_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_composite_resource_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..faa8803c61b7240926bb4fd072f7a9f1f1ee3d23 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_composite_resource_ops.cc @@ -0,0 +1,140 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +namespace mlir { +namespace TFTPU { +namespace { + +#define GEN_PASS_DEF_TPUCOLOCATECOMPOSITERESOURCEOPSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Pass that co-locates resource ops that use composite device resources +// (packed tensors) with the underlying physical TPU device. +struct TPUColocateCompositeResourceOps + : public impl::TPUColocateCompositeResourceOpsPassBase< + TPUColocateCompositeResourceOps> { + void runOnOperation() override; +}; + +// Wraps single op in `tf_device.launch` for explicit device assignment. +void WrapOpInLaunch(OpBuilder* builder, Location loc, Operation* op, + llvm::StringRef device) { + builder->setInsertionPoint(op); + auto launch = tf_device::LaunchOp::create( + *builder, loc, builder->getStringAttr(device), op->getResultTypes()); + launch.getBody().push_back(new Block); + op->replaceAllUsesWith(launch); + + builder->setInsertionPointToEnd(&launch.GetBody()); + tf_device::ReturnOp::create(*builder, loc, op->getResults()); + + // Move op inside cluster. + op->moveBefore(launch.GetBody().getTerminator()); +} + +llvm::SmallVector GetResourceOpsUsingCompositeArgsInReplicate( + tf_device::ReplicateOp replicate) { + llvm::SmallVector resource_users; + const auto add_resource_op_to_list = [&resource_users](Operation* op) { + if (!llvm::isa(op)) return; + + resource_users.emplace_back(op); + }; + + llvm::SmallVector resource_users_to_visit; + for (auto composite_arguments : replicate.GetPackedBlockArguments()) { + for (auto resource_user : composite_arguments.getUsers()) + resource_users_to_visit.emplace_back(resource_user); + } + + while (!resource_users_to_visit.empty()) { + llvm::SmallVector new_resource_users; + + for (auto resource_user : resource_users_to_visit) { + add_resource_op_to_list(resource_user); + + // Account for pass-through identity ops. + if (auto pass_through_identity = + llvm::dyn_cast(resource_user)) { + for (auto identity_user : + pass_through_identity.getOutput().getUsers()) { + new_resource_users.emplace_back(identity_user); + } + } + } + resource_users_to_visit.swap(new_resource_users); + } + + return resource_users; +} + +void ColocateCompositeResourceOpsInReplicate( + tf_device::ReplicateOp replicate_op, OpBuilder* builder) { + auto devices = replicate_op.getDevices(); + if (!devices) return; + if (!devices.value().get(tensorflow::GetDeviceAliasForLogicalCore(0))) return; + + const auto composite_resource_users = + GetResourceOpsUsingCompositeArgsInReplicate(replicate_op); + for (auto resource_user : composite_resource_users) { + WrapOpInLaunch(builder, resource_user->getLoc(), resource_user, + tensorflow::GetDeviceAliasForLogicalCore(0)); + } +} + +void TPUColocateCompositeResourceOps::runOnOperation() { + // Find all the executes first, since we will mutate the nodes around each + // execute in the same tf_device.replicate op. + llvm::SmallVector execute_launches; + getOperation().walk([&](tf_device::LaunchOp op) { + if (op.WrapsSingleOp() && + llvm::isa( + op.GetBody().front())) + execute_launches.push_back(op); + }); + + OpBuilder builder(&getContext()); + for (auto execute_launch : execute_launches) { + auto replicate = execute_launch->getParentOfType(); + if (!replicate) continue; + + ColocateCompositeResourceOpsInReplicate(replicate, &builder); + } +} + +} // namespace + +std::unique_ptr> +CreateTPUColocateCompositeResourceOps() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_splits.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_splits.cc new file mode 100644 index 0000000000000000000000000000000000000000..c233e3bf7144de7e68d46c0b7a7efc19cca6eb01 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_colocate_splits.cc @@ -0,0 +1,85 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +#define GEN_PASS_DEF_TPUCOLOCATESPLITSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +constexpr char kDeviceAttr[] = "device"; +// Attribute of colocation classes. +constexpr char kClassAttr[] = "_class"; + +bool HasDevice(Operation* op) { + auto attr = op->getAttrOfType(kDeviceAttr); + if (!attr) return false; + return !attr.getValue().empty(); +} + +// Returns the predecessors of `op` when `op`'s predecessors are wrapped by +// islands. +llvm::SmallVector IslandPredecessors(Operation* op) { + llvm::SmallVector predecessors; + for (Value operand : op->getOperands()) { + if (Operation* pred = operand.getDefiningOp()) { + int result_number = llvm::cast(operand).getResultNumber(); + if (auto pred_island = llvm::dyn_cast(pred)) { + Value yield_operand = pred_island.GetYield().getOperand(result_number); + predecessors.push_back(yield_operand.getDefiningOp()); + } + } + } + return predecessors; +} + +struct TPUColocateSplits + : public impl::TPUColocateSplitsPassBase { + void runOnOperation() override; +}; + +void TPUColocateSplits::runOnOperation() { + getOperation().walk([&](Operation* op) { + if (auto split = llvm::dyn_cast(op)) { + if (HasDevice(split) || split->getAttrOfType(kClassAttr)) + return WalkResult::advance(); + for (Operation* pred : IslandPredecessors(split)) { + if (auto colocation_classes = + pred->getAttrOfType(kClassAttr)) { + split->setAttr(kClassAttr, colocation_classes); + return WalkResult::advance(); + } + } + } + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> CreateTPUColocateSplitsPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_device_propagation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_device_propagation.cc new file mode 100644 index 0000000000000000000000000000000000000000..35db64c0bb438bd7e4ae8da851ce1f47951a1540 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_device_propagation.cc @@ -0,0 +1,296 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDeviceAttr[] = "device"; +constexpr char kFuncDeviceAttr[] = "tf.device"; + +// Checks if a function only contains a tf_executor.graph. +bool IsSupportedGraph(func::FuncOp func) { + if (!llvm::hasSingleElement(func)) return false; + + Block& block = func.front(); + if (!llvm::hasSingleElement(block.without_terminator())) return false; + + auto graph = llvm::dyn_cast(block.front()); + if (!graph) return false; + + Operation* terminator = block.getTerminator(); + if (graph.getNumResults() != terminator->getNumOperands()) return false; + for (auto result : llvm::zip(graph.getResults(), terminator->getOperands())) + if (std::get<0>(result) != std::get<1>(result)) return false; + + return true; +} + +// Checks if an operation of the tf_executor dialect can have TPU devices +// propagated through. +bool IsSupportedExecutorOp(Operation& op) { + auto ops_have_same_device = [](Operation* lhs, Operation* rhs) { + auto lhs_device_attr = lhs->getAttrOfType(kDeviceAttr); + auto rhs_device_attr = rhs->getAttrOfType(kDeviceAttr); + return (!lhs_device_attr && !rhs_device_attr) || + (lhs_device_attr && rhs_device_attr && + lhs_device_attr.getValue() == rhs_device_attr.getValue()); + }; + + // Check if tf_executor.NextIteration.Source/tf_executor.NextIteration.Sink + // pair has matching devices or no devices. + if (auto source = llvm::dyn_cast(op)) { + return ops_have_same_device(source, source.GetSink()); + } else if (auto sink = llvm::dyn_cast(op)) { + return ops_have_same_device(sink.GetSource(), sink); + } + + return llvm::isa(op); +} + +// Assigns all data results to a specified device. +void PopulateDeviceForOpResults( + Operation& op, llvm::StringRef device, + llvm::DenseMap& value_to_device) { + Operation* op_to_update = &op; + // Use tf_executor.island op if present as non v1 control flow op results are + // forwarded by a parent tf_executor.island op. + if (llvm::isa(op_to_update->getParentOp())) + op_to_update = op_to_update->getParentOp(); + + for (Value result : op_to_update->getResults()) { + if (mlir::isa(result.getType())) continue; + if (mlir::isa(result.getType())) break; + + value_to_device.insert({result, device}); + } +} + +// Checks if an operation can have TPU devices propagated through. +bool IsSupportedOpToSetDevice(Operation& op) { + return IsSupportedExecutorOp(op) || + isa(op); +} + +// Finds nonconflicting TPU device for an operation from its operands. If an +// operand has no device or a non TPU device, or if there are conflicting +// devices, and empty StringRef will be returned. Control dependencies, +// NextIteration.Source -> NextIteration.Sink token dependencies, and +// LoopCond -> Switch data dependencies are ignored. +llvm::StringRef FindDeviceFromOperands( + Operation& op, + const llvm::DenseMap& value_to_device) { + llvm::StringRef new_device; + const bool is_switch = llvm::isa(op); + for (Value operand : op.getOperands()) { + if (mlir::isa(operand.getType())) continue; + if (mlir::isa(operand.getType())) break; + + if (is_switch && + llvm::isa_and_nonnull(operand.getDefiningOp())) + continue; + + auto it = value_to_device.find(operand); + if (it == value_to_device.end()) return llvm::StringRef(); + + if (new_device.empty()) { + new_device = it->getSecond(); + continue; + } + + if (new_device != it->getSecond()) return llvm::StringRef(); + } + + return new_device; +} + +// Propagates devices from function arguments. +void PropagateDevicesFromArguments( + func::FuncOp func, + llvm::DenseMap& value_to_device) { + for (BlockArgument& arg : func.getArguments()) { + auto arg_device_attr = + func.getArgAttrOfType(arg.getArgNumber(), kFuncDeviceAttr); + if (!arg_device_attr || arg_device_attr.getValue().empty() || + !tensorflow::IsTPUDevice(arg_device_attr.getValue())) + continue; + value_to_device.insert({arg, arg_device_attr.getValue()}); + } +} + +// Propagates devices from operation operands to results. Updating the device of +// a tf_executor.NextIteration.Source/tf_executor.NextIteration.Sink will result +// in multiple passes over the tf_executor.graph to propagate devices in loops. +void PropagateDevicesInGraph( + tf_executor::GraphOp graph, + llvm::DenseMap& value_to_device) { + auto ops = graph.GetBody().without_terminator(); + + bool updated_next_iteration = false; + do { + updated_next_iteration = false; + for (Operation& op : ops) { + if (!IsSupportedExecutorOp(op)) continue; + + Operation* op_to_update = &op; + // Unpack inner op of tf_executor.island. + if (auto island_op = + llvm::dyn_cast(op_to_update)) { + if (!island_op.WrapsSingleOp()) continue; + op_to_update = &island_op.GetBody().front(); + } + + // If op already has a TPU device set, simply propagate its device. + auto device_attr = op_to_update->getAttrOfType(kDeviceAttr); + const bool has_device = device_attr && !device_attr.getValue().empty(); + if (has_device && tensorflow::IsTPUDevice(device_attr.getValue())) { + PopulateDeviceForOpResults(*op_to_update, device_attr.getValue(), + value_to_device); + continue; + } + + // Op has an unsupported device. + if (has_device) continue; + + if (!IsSupportedOpToSetDevice(*op_to_update)) continue; + + llvm::StringRef new_device = + FindDeviceFromOperands(*op_to_update, value_to_device); + if (new_device.empty()) continue; + + auto new_device_attr = + mlir::StringAttr::get(op_to_update->getContext(), new_device); + op_to_update->setAttr(kDeviceAttr, new_device_attr); + PopulateDeviceForOpResults(*op_to_update, new_device_attr.getValue(), + value_to_device); + + if (auto sink = + llvm::dyn_cast(op_to_update)) { + auto source = sink.GetSource(); + source->setAttr(kDeviceAttr, new_device_attr); + PopulateDeviceForOpResults(*source, new_device_attr.getValue(), + value_to_device); + updated_next_iteration = true; + } + } + } while (updated_next_iteration); +} + +// Propagates devices to function results. +void PropagateDevicesToResults( + func::FuncOp func, tf_executor::FetchOp fetch, + const llvm::DenseMap& value_to_device) { + // We apply all result attributes at once to avoid excessive allocations when + // we have many result values. + llvm::SmallVector, 8> result_attrs; + { + llvm::SmallVector tmp; + func.getAllResultAttrs(tmp); + + for (const auto& res : tmp) { + result_attrs.push_back(res.getValue().vec()); + } + } + + mlir::Builder builder(func.getOperation()); + + for (OpOperand& operand : fetch.getOperation()->getOpOperands()) { + if (mlir::isa(operand.get().getType())) break; + auto it = value_to_device.find(operand.get()); + if (it != value_to_device.end()) { + auto device_attr = func.getResultAttrOfType( + operand.getOperandNumber(), kFuncDeviceAttr); + if (device_attr && !device_attr.getValue().empty()) continue; + + // Update the existing attribute named `kFuncDeviceAttr` if found. + // Otherwise introduce a new attribute. + auto& resultAttrForOp = result_attrs[operand.getOperandNumber()]; + bool found = false; + for (int i = 0; i < resultAttrForOp.size(); ++i) { + auto attr = resultAttrForOp[i]; + if (attr.getName() == kFuncDeviceAttr) { + resultAttrForOp[i] = builder.getNamedAttr( + kFuncDeviceAttr, + StringAttr::get(func.getContext(), it->getSecond())); + found = true; + break; + } + } + if (!found) { + resultAttrForOp.push_back(builder.getNamedAttr( + kFuncDeviceAttr, + StringAttr::get(func.getContext(), it->getSecond()))); + } + } + } + + llvm::SmallVector tmp; + for (const auto& res : result_attrs) { + tmp.push_back(builder.getDictionaryAttr(res)); + } + func.setAllResultAttrs(tmp); +} + +#define GEN_PASS_DEF_TPUDEVICEPROPAGATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUDevicePropagation + : public impl::TPUDevicePropagationPassBase { + void runOnOperation() override; +}; + +void TPUDevicePropagation::runOnOperation() { + func::FuncOp func = getOperation(); + if (!IsSupportedGraph(func)) return; + + llvm::DenseMap value_to_device; + PropagateDevicesFromArguments(func, value_to_device); + auto graph = llvm::cast(func.front().front()); + PropagateDevicesInGraph(graph, value_to_device); + PropagateDevicesToResults(func, graph.GetFetch(), value_to_device); +} + +} // namespace + +std::unique_ptr> CreateTPUDevicePropagationPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_dynamic_layout_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_dynamic_layout_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..cf910590734023d4bbea13622baae7d266176c3b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_dynamic_layout_pass.cc @@ -0,0 +1,299 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/UseDefLists.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDeviceAttr[] = "device"; +constexpr char kDeviceCPU[] = "CPU"; +constexpr char kFuncDeviceAttr[] = "tf.device"; + +struct TPUDynamicLayoutPass + : public TF::PerFunctionAggregateAnalysisConsumerPass< + TPUDynamicLayoutPass, TF::ResourceAliasAnalysis> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TPUDynamicLayoutPass) + + void runOnFunction( + func::FuncOp func, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis); + + StringRef getArgument() const final { return "tf-tpu-dynamic-layout-pass"; } + + StringRef getDescription() const final { + return "Inserts TPU layout ops to determine layout at run time."; + } +}; + +// Checks if the input producer op is supported in this transform. Right now, we +// only check if it is a tf.IteratorGetNext where resource input is coming from +// a VarHandle on CPU or a function argument assigned to CPU. +bool IsSupportedInputOp( + Operation* op, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { + TF::IteratorGetNextOp iterator_op = llvm::dyn_cast(op); + if (!iterator_op) return false; + + Value resource_iterator = iterator_op.getIterator(); + + if (resource_alias_analysis.IsUnknownResource(resource_iterator)) + return false; + llvm::SmallSetVector aliases = + resource_alias_analysis.GetResourceAliases(resource_iterator); + + auto is_generator = [](Value val) { + if (mlir::isa(val)) return true; + Operation* definition = val.getDefiningOp(); + return definition->getNumOperands() == 0 && + definition->getNumResults() == 1; + }; + + // Check all generator aliases (ops or function argument) are on CPU. + func::FuncOp func = iterator_op->getParentOfType(); + return llvm::all_of(aliases, [&](Value alias) { + // Ignore non-generator aliases. + if (!is_generator(alias)) return true; + + StringAttr device; + if (auto arg = mlir::dyn_cast(alias)) { + device = func.getArgAttrOfType(arg.getArgNumber(), + kFuncDeviceAttr); + } else { + device = alias.getDefiningOp()->getAttrOfType(kDeviceAttr); + } + + if (!device) return false; + tensorflow::DeviceNameUtils::ParsedName parsed_device; + if (!tensorflow::DeviceNameUtils::ParseFullName(device.getValue().str(), + &parsed_device)) { + return false; + } + return parsed_device.has_type && parsed_device.type == kDeviceCPU; + }); +} + +OpBuilder CreateBuilderAfterOp(Operation* op) { + return OpBuilder(op->getBlock(), ++Block::iterator(op)); +} + +// Builds a TPUGetLayoutOp with the given compile op and input index. +TF::TPUGetLayoutOp BuildGetLayout(const int64_t execute_arg_index, + Value compilation_key, + tf_device::LaunchOp compile_launch, + OpBuilder* builder) { + return TF::TPUGetLayoutOp::create( + *builder, compile_launch.getLoc(), + llvm::ArrayRef{RankedTensorType::get({ShapedType::kDynamic}, + builder->getIntegerType(64))}, + llvm::ArrayRef{compilation_key}, + llvm::ArrayRef{ + builder->getNamedAttr("index", + builder->getI64IntegerAttr(execute_arg_index)), + builder->getNamedAttr("is_output", builder->getBoolAttr(false))}); +} + +// Builds a TPUCopyWithLayoutOp with the given get_layout op and input. +TF::TPUCopyWithLayoutOp BuildCopyWithLayout(tf_device::LaunchOp execute_launch, + tf_device::LaunchOp compile_launch, + TF::TPUGetLayoutOp get_layout, + Value input, OpBuilder* builder) { + return TF::TPUCopyWithLayoutOp::create( + *builder, execute_launch.getLoc(), llvm::ArrayRef{input.getType()}, + llvm::ArrayRef{input, get_layout.getLayout()}); +} + +// Performs transformation for a non-replicated input. +void HandleInput(Value input, const int64_t execute_arg_index, + TF::TPUExecuteOp execute, tf_device::LaunchOp execute_launch, + tf_device::LaunchOp compile_launch) { + OpBuilder builder = CreateBuilderAfterOp(compile_launch); + auto get_layout = BuildGetLayout(execute_arg_index, execute.getKey(), + compile_launch, &builder); + builder.setInsertionPoint(execute_launch); + auto copy_with_layout = BuildCopyWithLayout(execute_launch, compile_launch, + get_layout, input, &builder); + copy_with_layout->setAttr(kDeviceAttr, execute_launch.getDeviceAttr()); + execute.setOperand(execute_arg_index, copy_with_layout); +} + +// Performs transformation for replicated inputs. Returns true if this is a +// supported case (thus transform happened). +bool HandleReplicatedInputs( + const int64_t execute_arg_index, Value compilation_key, + tf_device::LaunchOp execute_launch, tf_device::LaunchOp compile_launch, + mlir::BlockArgument replicate_arg, tf_device::ReplicateOp replicate, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { + // We need to know the devices to copy to. + if (!replicate.getDevices()) return false; + + MutableArrayRef inputs = + replicate.GetOperandsForBlockArgument(replicate_arg); + for (const auto& entry : llvm::enumerate(inputs)) { + auto input_op = entry.value().get().getDefiningOp(); + if (!input_op || !IsSupportedInputOp(input_op, resource_alias_analysis)) + return false; + } + OpBuilder builder = CreateBuilderAfterOp(compile_launch); + auto get_layout = BuildGetLayout(execute_arg_index, compilation_key, + compile_launch, &builder); + builder.setInsertionPoint(replicate); + for (const auto& entry : llvm::enumerate(inputs)) { + auto copy_with_layout = + BuildCopyWithLayout(execute_launch, compile_launch, get_layout, + entry.value().get(), &builder); + + auto device_list = mlir::cast( + replicate.getDevices().value().get(execute_launch.getDevice())); + copy_with_layout->setAttr(kDeviceAttr, + device_list.getValue()[entry.index()]); + + entry.value().set(copy_with_layout); + } + return true; +} + +// Performs transformation on a compile and associated execute(s) ops. The +// compile should not have other uses. +void HandleCompileAndExecutes( + tf_device::LaunchOp compile_launch, + llvm::MutableArrayRef execute_launches, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { + auto compile = + llvm::cast(compile_launch.GetBody().front()); + tensorflow::tpu::TPUCompileMetadataProto metadata; + metadata.ParseFromString(compile.getMetadata().str()); + llvm::SmallVector, 4> input_mappings = + tensorflow::GetMetadataArgumentMapping(metadata); + + bool metadata_updated = false; + auto maybe_replicate = + execute_launches.front()->getParentOfType(); + + for (auto execute_and_input_mapping : + llvm::zip(execute_launches, input_mappings)) { + auto& execute_launch = std::get<0>(execute_and_input_mapping); + auto execute = + llvm::cast(execute_launch.GetBody().front()); + const auto& input_mapping = std::get<1>(execute_and_input_mapping); + + for (const auto& input_and_idx : llvm::enumerate(execute.getArgs())) { + Value input = input_and_idx.value(); + const int64_t execute_arg_index = input_and_idx.index(); + if (auto block_arg = mlir::dyn_cast(input)) { + // For a block argument, consider transforms only when it is a + // replicated input (defining ops will be outside the replicate node). + if (maybe_replicate != block_arg.getParentRegion()->getParentOp() || + !HandleReplicatedInputs(execute_arg_index, execute.getKey(), + execute_launch, compile_launch, block_arg, + maybe_replicate, resource_alias_analysis)) { + continue; + } + } else { + // For an op output, consider transforms only when 1) there is no + // replication or 2) it is outside the replicate node that encloses the + // execute node. (Because if the op is inside replicate, it is probably + // not on the host.) + auto* input_op = input.getDefiningOp(); + if (maybe_replicate && + maybe_replicate.getBody().isAncestor(input_op->getParentRegion())) { + continue; + } + if (!IsSupportedInputOp(input_op, resource_alias_analysis)) continue; + HandleInput(input, execute_arg_index, execute, execute_launch, + compile_launch); + } + + metadata.mutable_args(input_mapping[execute_arg_index]) + ->set_unrestricted_layout(true); + metadata_updated = true; + } + } + + if (metadata_updated) + compile->setAttr("metadata", StringAttr::get(compile.getContext(), + metadata.SerializeAsString())); +} + +void TPUDynamicLayoutPass::runOnFunction( + func::FuncOp func, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { + func.walk([&](TF::_TPUCompileMlirOp compile) { + // Detect tf._TPUCompileMlir -> tf.TPUExecute(s). + auto compile_launch = + llvm::dyn_cast(compile->getParentOp()); + if (!compile_launch || !compile_launch.WrapsSingleOp()) return; + + llvm::SmallVector execute_launches; + execute_launches.reserve(compile_launch.getNumResults() - 1); + for (Value program_result : + llvm::drop_begin(compile_launch.getResults(), 1)) { + if (!program_result.hasOneUse()) return; + Operation* user = *program_result.user_begin(); + auto execute = llvm::dyn_cast(user); + if (!execute) return; + auto execute_launch = + llvm::dyn_cast(execute->getParentOp()); + if (!execute_launch || !execute_launch.WrapsSingleOp()) return; + execute_launches.push_back(execute_launch); + } + + HandleCompileAndExecutes(compile_launch, execute_launches, + resource_alias_analysis); + }); +} + +} // namespace + +std::unique_ptr> CreateTPUDynamicLayoutPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_host_computation_expansion.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_host_computation_expansion.cc new file mode 100644 index 0000000000000000000000000000000000000000..593a8be251d3557699e20478c5c21968bc306a61 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_host_computation_expansion.cc @@ -0,0 +1,139 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; + +bool HasOutsideCompilationAttribute(Operation* op) { + return op->getAttrOfType(kXlaOutsideCompilationAttr) != nullptr; +} + +// Finds op that created a given value. If the value is a BlockArgument, this +// returns the owner of the Block. +Operation* GetOpOfValue(Value value) { + if (auto block_arg = mlir::dyn_cast(value)) + return block_arg.getOwner()->getParentOp(); + + return value.getDefiningOp(); +} + +// TODO(b/158596585): Replace this with a cost model analysis. +bool IsTrivialUnaryOperation(Operation* op) { + return llvm::isa(op); +} + +// Adds outside compilation attributes to unary ops such as Identity/Cast ops +// at the head of TPU computation that is used only by other outside compiled +// ops. Identity ops and Cast ops is commonly added to the start of TPU +// computation. Adding/expanding outside compilation attributes to these ops +// will ensure that head outside compiled ops are correctly located and moved to +// host. +// TODO(b/158691733): Also handle ops inside function calls/control flows. +void ExpandHeadOutsideCompiledOps(tf_device::ClusterOp cluster, + OpBuilder* builder) { + Region* cluster_region = &cluster.getBody(); + llvm::SmallSetVector head_outside_compiled_ops; + + // Traverse the graph in topological order to find all outside compiled ops + // at head of TPU computation or unary ops that are only used by other outside + // compiled ops. + auto cluster_ops = cluster.GetBody().without_terminator(); + for (Operation& cluster_op : cluster_ops) { + if (IsTrivialUnaryOperation(&cluster_op) || + HasOutsideCompilationAttribute(&cluster_op)) { + auto walk_result = cluster_op.walk([&](Operation* op) { + for (Value operand : op->getOperands()) { + Operation* operand_op = GetOpOfValue(operand); + if (head_outside_compiled_ops.count(operand_op)) continue; + + if (operand_op->getParentRegion() == cluster_region) + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + + if (!walk_result.wasInterrupted()) + head_outside_compiled_ops.insert(&cluster_op); + } + } + + for (auto head_outside_compiled_op : + llvm::reverse(head_outside_compiled_ops)) { + auto users = head_outside_compiled_op->getUsers(); + if (users.empty() || + HasOutsideCompilationAttribute(head_outside_compiled_op)) + continue; + + bool should_expand_op_to_host_computation = true; + for (auto consumer_op : users) { + if (should_expand_op_to_host_computation && + !HasOutsideCompilationAttribute(consumer_op)) { + should_expand_op_to_host_computation = false; + continue; + } + } + + if (should_expand_op_to_host_computation) + head_outside_compiled_op->setAttr(kXlaOutsideCompilationAttr, + builder->getStringAttr("")); + } +} + +#define GEN_PASS_DEF_TPUHOSTCOMPUTATIONEXPANSIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUHostComputationExpansionPass + : public impl::TPUHostComputationExpansionPassBase< + TPUHostComputationExpansionPass> { + void runOnOperation() override; +}; + +void TPUHostComputationExpansionPass::runOnOperation() { + OpBuilder builder(&getContext()); + getOperation().walk([&](tf_device::ClusterOp cluster) { + ExpandHeadOutsideCompiledOps(cluster, &builder); + }); +} + +} // anonymous namespace + +std::unique_ptr> +CreateTPUHostComputationExpansionPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_identity_pruning.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_identity_pruning.cc new file mode 100644 index 0000000000000000000000000000000000000000..14b3c8ebc0d505d5ee97074015449e42787f4d43 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_identity_pruning.cc @@ -0,0 +1,112 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +// This pass removes Identity/IdentityN ops from the TPU computation and +// reachable functions. +// TODO(lyandy): Remove this pass once resource op lifting is migrated to use +// resource alias analysis and support region based control flow. Removing +// Identity ops may remove `_XlaSharding` annotation attribute if Identity ops +// are used to propagate such information. + +#define GEN_PASS_DEF_TPUIDENTITYPRUNINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUIdentityPruning + : public impl::TPUIdentityPruningPassBase { + void runOnOperation() override; +}; + +// Collects all reachable functions (via call ops) from a given region. +SmallVector CollectReachableFunctions(Region& region) { + llvm::SmallPtrSet reachable_funcs; + + auto collect_reachable_funcs = + [&reachable_funcs](Region& src, + SmallVectorImpl& funcs_to_visit) { + src.walk([&reachable_funcs, &funcs_to_visit](CallOpInterface call_op) { + auto func = dyn_cast_or_null(call_op.resolveCallable()); + if (func && reachable_funcs.insert(func).second) + funcs_to_visit.push_back(func); + }); + }; + + SmallVector funcs_to_visit; + collect_reachable_funcs(region, funcs_to_visit); + + while (!funcs_to_visit.empty()) { + SmallVector new_funcs_to_visit; + for (func::FuncOp func_to_visit : funcs_to_visit) { + if (!func_to_visit.getCallableRegion()) continue; + collect_reachable_funcs(*func_to_visit.getCallableRegion(), + new_funcs_to_visit); + } + funcs_to_visit.swap(new_funcs_to_visit); + } + + return llvm::to_vector<4>(reachable_funcs); +} + +// Removes Identity/IdentityN ops from a region and forwards its operands to its +// results. +void RemoveIdentityFromRegion(Region& region) { + region.walk([](Operation* op) { + if (isa(op)) { + op->replaceAllUsesWith(op->getOperands()); + op->erase(); + } + }); +} + +void TPUIdentityPruning::runOnOperation() { + SmallVector clusters; + getOperation().walk( + [&](tf_device::ClusterOp cluster) { clusters.push_back(cluster); }); + + for (tf_device::ClusterOp cluster : clusters) { + RemoveIdentityFromRegion(cluster.getBody()); + auto reachable_funcs = CollectReachableFunctions(cluster.getBody()); + for (func::FuncOp reachable_func : reachable_funcs) + RemoveIdentityFromRegion(*reachable_func.getCallableRegion()); + } +} + +} // anonymous namespace + +std::unique_ptr> CreateTPUIdentityPruningPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_parallel_execute_sink_resource_write.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_parallel_execute_sink_resource_write.cc new file mode 100644 index 0000000000000000000000000000000000000000..61520b5a00d554340a77d7fc3843c6d7a807f4f0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_parallel_execute_sink_resource_write.cc @@ -0,0 +1,166 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +#define GEN_PASS_DEF_TPUPARALLELEXECUTESINKRESOURCEWRITEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass that moves `tf.AssignVariableOp` into a `tf_device.parallel_execute` +// region if the `tf.AssignVariableOp` is the only consumer of a +// `tf_device.parallel_execute` result. This will allow +// TPUMergeVariablesWithExecute to merge resource writes without special +// handling for `tf_device.parallel_execute`. +struct TPUParallelExecuteSinkResourceWrite + : public impl::TPUParallelExecuteSinkResourceWritePassBase< + TPUParallelExecuteSinkResourceWrite> { + void runOnOperation() override; +}; + +// Finds an AssignVariableOp that can be moved into the parallel_execute region. +// These AssignVariableOps must be the only consumer of the respective +// parallel_execute result, and the resource handle producer must be from an op +// before or above the parallel_execute. +TF::AssignVariableOp GetSingleUseResourceWrite( + tf_device::ParallelExecuteOp parallel_execute, Value result) { + if (!result.hasOneUse()) return nullptr; + + OpOperand& use = *result.getUses().begin(); + auto assign_var = dyn_cast(use.getOwner()); + if (!assign_var) return nullptr; + + if (use.get() != assign_var.getValue()) return nullptr; + + auto* resource_handle_op = assign_var.getResource().getDefiningOp(); + if (resource_handle_op == parallel_execute) return nullptr; + + if (resource_handle_op && + resource_handle_op->getBlock() == + parallel_execute.getOperation()->getBlock() && + parallel_execute.getOperation()->isBeforeInBlock(resource_handle_op)) + return nullptr; + + return assign_var; +} + +// Finds AssignVariableOps that can be moved into a parallel_execute region and +// moves them. Leftover parallel_execute results that were used by the +// such AssignVariableOp are also pruned. +void SinkResourceWritesIntoParallelExecute( + tf_device::ParallelExecuteOp parallel_execute) { + bool rewrite = false; + const int num_regions = parallel_execute.getNumRegions(); + llvm::SmallVector results_to_remap; + + // Go through each region and find AssignVariableOps that can be moved into + // the parallel_execute region. Result indices by region index are collected, + // so they can be removed afterwards. + llvm::SmallVector, 4> results_to_remove_by_region; + results_to_remove_by_region.resize(num_regions); + for (int i = 0; i < num_regions; ++i) { + Block& block = parallel_execute.GetRegionBlockWithIndex(i); + auto results = parallel_execute.GetRegionOutputs(i); + auto& results_to_remove = results_to_remove_by_region[i]; + results_to_remove.reserve(results.size()); + Operation* terminator = block.getTerminator(); + for (auto result : llvm::enumerate(results)) { + TF::AssignVariableOp assign_var = + GetSingleUseResourceWrite(parallel_execute, result.value()); + if (!assign_var) { + results_to_remap.push_back(result.value()); + continue; + } + + // Move AssignVariableOp and update the value to be written to the + // resource variable to be the non forwarded value from within the + // parallel_execute region. + assign_var.getOperation()->moveBefore(terminator); + assign_var.getValueMutable().assign( + terminator->getOperand(result.index())); + results_to_remove.push_back(result.index()); + } + + rewrite |= !results_to_remove.empty(); + } + + if (!rewrite) return; + + // Remove leftover unused results (terminator operands) from moving + // AssignVariabeOps into the parallel_execute region. + for (auto results_to_remove : llvm::enumerate(results_to_remove_by_region)) { + Block& block = + parallel_execute.GetRegionBlockWithIndex(results_to_remove.index()); + Operation* terminator = block.getTerminator(); + for (int index_to_remove : llvm::reverse(results_to_remove.value())) + terminator->eraseOperand(index_to_remove); + } + + // Replace old parallel_execute with new parallel_execute by moving the + // regions to a new parallel_execute and remapping the results. + llvm::SmallVector new_result_types; + new_result_types.reserve(results_to_remap.size()); + for (Value old_result : results_to_remap) + new_result_types.push_back(old_result.getType()); + + OpBuilder builder(parallel_execute); + auto new_parallel_execute = tf_device::ParallelExecuteOp::create( + builder, parallel_execute.getLoc(), num_regions, new_result_types); + + for (auto region : llvm::zip(new_parallel_execute.getRegions(), + parallel_execute.getRegions())) + std::get<0>(region).takeBody(std::get<1>(region)); + + for (auto result : + llvm::zip(results_to_remap, new_parallel_execute.getResults())) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + parallel_execute.erase(); +} + +void TPUParallelExecuteSinkResourceWrite::runOnOperation() { + llvm::SmallVector parallel_executes; + getOperation().walk([&](tf_device::ParallelExecuteOp parallel_execute) { + parallel_executes.push_back(parallel_execute); + }); + + for (tf_device::ParallelExecuteOp parallel_execute : parallel_executes) + SinkResourceWritesIntoParallelExecute(parallel_execute); +} + +} // anonymous namespace + +std::unique_ptr> +CreateTPUParallelExecuteSinkResourceWritePass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_partitioned_op_conversion.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_partitioned_op_conversion.cc new file mode 100644 index 0000000000000000000000000000000000000000..3345a99b3aa8b4cb449d8c2ea028a4ac16237db6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_partitioned_op_conversion.cc @@ -0,0 +1,147 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFTPU { +namespace { + +#define GEN_PASS_DEF_TPUPARTITIONEDOPCONVERSIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUPartitionedOpConversionPass + : public impl::TPUPartitionedOpConversionPassBase< + TPUPartitionedOpConversionPass> { + void runOnOperation() override; +}; + +template +LogicalResult ReplacePartitionedOp(IntegerAttr num_cores_per_replica, T op) { + constexpr bool is_input = + std::is_same_v, TF::TPUPartitionedInputOp>; + static_assert( + is_input || std::is_same_v, TF::TPUPartitionedOutputOp>, + "operator should either be an input or output"); + + OpBuilder builder(op); + int partition_dim = op.getPartitionDim(); + bool is_replicated = partition_dim == -1; + if (!(is_replicated || num_cores_per_replica)) return failure(); + + Type first_operand_type; + if constexpr (is_input) { + first_operand_type = op.getOperand(0).getType(); + } else { + first_operand_type = op.getOperand().getType(); + } + + auto element_type = getElementTypeOrSelf(first_operand_type); + if (mlir::isa(element_type)) { + first_operand_type = + mlir::cast(element_type).getSubtypes().front(); + } + + auto tensor_type = mlir::dyn_cast_or_null(first_operand_type); + if (!(tensor_type && tensor_type.hasRank())) { + return op->emitError() + << "cannot convert op with unranked or non-tensor input type " + << tensor_type << "."; + } + + int rank = tensor_type.getRank(); + if (rank <= partition_dim) { + return op->emitError() << "cannot partition " << first_operand_type + << " (rank = " << rank << ") along dimension " + << partition_dim << "."; + } + + llvm::SmallVector partition_dims(is_replicated ? 0 : rank, 1); + if (!is_replicated) { + partition_dims[partition_dim] = num_cores_per_replica.getInt(); + } + + if constexpr (is_input) { + auto pi = TF::TPUPartitionedInputV2Op::create( + builder, op.getLoc(), op.getType(), op.getOperands(), + builder.getI64ArrayAttr(partition_dims), builder.getBoolAttr(false), + op.get_XlaShardingAttr(), op.get_XlaShardingV2Attr()); + op->replaceAllUsesWith(pi); + } else { + auto po = TF::TPUPartitionedOutputV2Op::create( + builder, op.getLoc(), op.getResultTypes(), op.getOperand(), + builder.getI64ArrayAttr(partition_dims), op.get_XlaShardingAttr(), + op.get_XlaShardingV2Attr()); + op->replaceAllUsesWith(po); + } + + return success(); +} + +void TPUPartitionedOpConversionPass::runOnOperation() { + llvm::SmallVector metadata; + getOperation()->walk( + [&metadata](TF::TPUReplicateMetadataOp op) { metadata.push_back(op); }); + + IntegerAttr num_cores_per_replica; + if (metadata.size() == 1) { + num_cores_per_replica = metadata.front().getNumCoresPerReplicaAttr(); + } + + auto result = getOperation()->walk([&num_cores_per_replica](Operation* op) { + std::optional status; + if (auto partitioned_input = + llvm::dyn_cast_or_null(op)) { + status = ReplacePartitionedOp(num_cores_per_replica, partitioned_input); + } else if (auto partitioned_output = + llvm::dyn_cast_or_null(op)) { + status = ReplacePartitionedOp(num_cores_per_replica, partitioned_output); + } + + if (status.has_value()) { + if (failed(*status) || !op->use_empty()) return WalkResult::interrupt(); + + op->erase(); + } + + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) { + signalPassFailure(); + return; + } +} + +} // namespace + +std::unique_ptr> +CreateTPUPartitionedOpConversionPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_reorder_replicate_and_partitioned_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_reorder_replicate_and_partitioned_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..99e38539ce91dbc470ee3bc11409269a4662f25a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_reorder_replicate_and_partitioned_inputs.cc @@ -0,0 +1,176 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFTPU { +namespace { + +#define GEN_PASS_DEF_TPUREORDERREPLICATEANDPARTITIONEDINPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUReorderReplicateAndPartitionedInputsPass + : public impl::TPUReorderReplicateAndPartitionedInputsPassBase< + TPUReorderReplicateAndPartitionedInputsPass> { + void runOnOperation() override; +}; + +LogicalResult ReorderReplicateAndPartitionedInputs( + TF::TPUReplicatedInputOp replicated_input) { + if (!llvm::all_of(replicated_input.getInputs(), [](Value input) { + return llvm::isa_and_nonnull( + input.getDefiningOp()); + })) + return replicated_input.emitOpError() + << "expects all inputs from 'tf.TPUPartitionedInputV2' ops"; + + const auto metadata_iter = + replicated_input->getBlock()->getOps(); + TF::TPUReplicateMetadataOp metadata; + if (!metadata_iter.empty()) metadata = *(metadata_iter.begin()); + + auto first_partitioned_input = llvm::cast( + replicated_input.getOperand(0).getDefiningOp()); + auto partition_dims = first_partitioned_input.getPartitionDims(); + const std::optional<::llvm::StringRef> xla_sharding = + first_partitioned_input.get_XlaSharding(); + + size_t num_cores_per_replica = first_partitioned_input.getNumOperands(); + if (metadata) { + num_cores_per_replica = metadata.getNumCoresPerReplica(); + } else if (first_partitioned_input.getIsPacked()) { + return first_partitioned_input->emitOpError() + << "num cores per replica unavailable, metadata missing?"; + } + + const bool packed_input = first_partitioned_input.getIsPacked(); + const size_t num_operands_expected = packed_input ? 1 : num_cores_per_replica; + if (metadata && + num_operands_expected != first_partitioned_input.getNumOperands()) { + return first_partitioned_input->emitOpError() + << "expects " << num_operands_expected << " operands but found " + << first_partitioned_input.getNumOperands(); + } + + for (const auto& operand : replicated_input.getInputs().drop_front()) { + auto partitioned_input = + llvm::cast(operand.getDefiningOp()); + const std::optional<::llvm::StringRef> op_xla_sharding = + partitioned_input.get_XlaSharding(); + const auto op_partition_dims = partitioned_input.getPartitionDims(); + // Abort if TPUPartitionedInputV2(s) do not have the same attributes. + if (!llvm::equal(partition_dims, op_partition_dims)) { + return partitioned_input->emitOpError() + << "expects partition_dims = " << partition_dims << " but found " + << op_partition_dims; + } else if (partitioned_input.getIsPacked() != + first_partitioned_input.getIsPacked()) { + return partitioned_input->emitOpError() + << "packing should match across ops"; + } else if (partitioned_input.getNumOperands() != num_operands_expected) { + return partitioned_input->emitOpError() + << "expects " << num_operands_expected << " operands but found " + << partitioned_input.getNumOperands(); + } else if (xla_sharding != op_xla_sharding) { + return replicated_input.emitOpError() + << "expects all inputs from 'tf.TPUPartitionedInputV2' ops to " + "have identical XLA sharding"; + } + } + + // 2D Matrix to store per core per replica operands. The matrix dimensions are + // num_cores_per_replica x num_replicas. i-th row holds the operands for i-th + // core. j-th column holds the operands for j-th replica. + llvm::SmallVector, 4> + operands_per_replica_per_core(num_cores_per_replica); + + // Collect all operands in the 2D matrix. + for (auto operand : replicated_input.getInputs()) { + Operation* pi = operand.getDefiningOp(); + for (unsigned core_id = 0; core_id < num_cores_per_replica; ++core_id) { + const auto pi_operand = + packed_input ? pi->getOperand(0) : pi->getOperand(core_id); + operands_per_replica_per_core[core_id].push_back(pi_operand); + } + } + + // Create new `tf.TPUReplicatedInput` ops feeding into one + // `tf.TPUPartitionedInputV2` op. + OpBuilder builder(replicated_input); + llvm::SmallVector operands_per_core; + for (auto& operands_per_replica : operands_per_replica_per_core) { + const bool is_packed = + packed_input && llvm::all_equal(operands_per_replica); + if (is_packed) // reduce the duplicates to one input for packed vars + operands_per_replica.erase(operands_per_replica.begin() + 1, + operands_per_replica.end()); + auto replicate_op = TF::TPUReplicatedInputOp::create( + builder, replicated_input.getLoc(), replicated_input.getType(), + operands_per_replica, replicated_input->getAttrs()); + replicate_op.setIsPacked(is_packed); + operands_per_core.push_back(replicate_op); + } + + auto pi = TF::TPUPartitionedInputV2Op::create( + builder, first_partitioned_input.getLoc(), replicated_input.getType(), + operands_per_core, first_partitioned_input->getAttrs()); + pi.setIsPacked(false); // inputs are now ops--not resources + replicated_input.replaceAllUsesWith(pi.getOutput()); + return success(); +} + +void TPUReorderReplicateAndPartitionedInputsPass::runOnOperation() { + auto result = + getOperation()->walk([](TF::TPUReplicatedInputOp replicated_input) { + if (llvm::none_of(replicated_input.getInputs(), [](Value input) { + return llvm::isa_and_nonnull( + input.getDefiningOp()); + })) + return WalkResult::advance(); + if (failed(ReorderReplicateAndPartitionedInputs(replicated_input))) + return WalkResult::interrupt(); + + assert(replicated_input->use_empty()); + replicated_input->erase(); + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) { + signalPassFailure(); + return; + } + + getOperation()->walk([](TF::TPUPartitionedInputV2Op partitioned_input) { + if (partitioned_input->use_empty()) partitioned_input->erase(); + }); +} + +} // namespace + +std::unique_ptr> +CreateTPUReorderReplicateAndPartitionedInputsPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_partitioning.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_partitioning.cc new file mode 100644 index 0000000000000000000000000000000000000000..3d768921cb30f061ae5119d904ff5541c1cf320a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_partitioning.cc @@ -0,0 +1,287 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h" +#include "xla/xla_data.pb.h" + +namespace mlir { +namespace TFTPU { +namespace { + +#define GEN_PASS_DEF_TPURESOURCEREADSWRITESPARTITIONINGPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +constexpr char kUseSpmdAttr[] = "use_spmd_for_xla_partitioning"; +constexpr char kNumCoresPerReplicaAttr[] = "num_cores_per_replica"; + +struct TPUResourceReadsWritesPartitioningPass + : public impl::TPUResourceReadsWritesPartitioningPassBase< + TPUResourceReadsWritesPartitioningPass> { + void runOnOperation() override; +}; + +bool AllResourceTypesHaveSubtypes(TypeRange resources) { + for (Type resource : resources) + if (!llvm::hasSingleElement( + llvm::cast( + llvm::cast(resource).getElementType()) + .getSubtypes())) + return false; + + return true; +} + +Type GetResourceSubtype(Type type) { + return llvm::cast( + llvm::cast(type).getElementType()) + .getSubtypes() + .front(); +} + +Type GetResourceSubtype(Value resource) { + return GetResourceSubtype(resource.getType()); +} + +// Updates uses of `old_read` to `new_partitioned_input` and `new_reads`. +// `old_partitioned_input` is the predecessor of `old_read`. `new_reads` +// contains the predecessors of `new_partitioned_input`. +LogicalResult UpdateReadUses(TF::ReadVariableOp old_read, + TF::TPUPartitionedInputV2Op old_partitioned_input, + TF::TPUPartitionedInputV2Op new_partitioned_input, + llvm::SmallVector new_reads, + bool is_packed) { + xla::OpSharding sharding; + sharding.ParseFromString( + old_partitioned_input.get_XlaShardingAttr().getValue().str()); + for (OpOperand& read_use : + llvm::make_early_inc_range(old_read.getValue().getUses())) { + if (dyn_cast_or_null(read_use.getOwner())) { + // ClusterFunc's use of the Read is replaced with use of the + // TPUPartitionedInputV2. + read_use.set(new_partitioned_input); + } else { + // Outside compiled code's use of the Read after TPUPartitionedInputV2 is + // replaced with use of the first Read before the TPUPartitionedInputV2. + if (sharding.type() != xla::OpSharding::REPLICATED) { + // TODO(b/243077297): Generalize to any sharding. + old_partitioned_input.emitOpError( + "TPUPartitionedInputV2 variable used in outside compiled code is " + "only supported with REPLICATED sharding"); + return failure(); + } else if (is_packed) { + // TODO(b/202047549) Generalize to the packed case. + old_partitioned_input.emitOpError( + "TPUPartitionedInputV2 variable used in outside compiled code is " + "only supported with UNPACKED ops (wrap value in tf.identity or" + "disable packing in TPUStrategy)"); + return failure(); + } + read_use.set(new_reads[0]); + } + } + return success(); +} + +namespace { +constexpr char kDeviceAttr[] = "device"; +constexpr char kFuncDeviceAttr[] = "tf.device"; +mlir::Attribute GetDeviceOfResource(mlir::func::FuncOp func, + mlir::Value resource) { + if (auto* resource_op = resource.getDefiningOp()) { + return resource_op->getAttr(kDeviceAttr); + } else { + const auto resource_arg = dyn_cast_or_null(resource); + if (resource_arg && (resource_arg.getOwner() == &(func.front()))) { + return func.getArgAttrOfType( + resource_arg.getArgNumber(), kFuncDeviceAttr); + } else { + return mlir::Attribute{}; + } + } +} + +bool IsCompositeDevice(mlir::Attribute attr) { + const auto str_attr = llvm::dyn_cast_if_present(attr); + return str_attr && + (str_attr.getValue().find("COMPOSITE") != llvm::StringRef::npos); +} +} // namespace + +// Rewrites unpartitioned resource reads and writes to partitioned resource +// reads and writes. The TPU computation from the frontend is generated in such +// a way that resource operations operate on the unpartitioned resource handle +// (from a `tf.TPUReplicatedInput`). This results in resource reads and writes +// on the unpartitioned resource handle post resource op decomposition/lifting. +// Here the unpartitioned resource read and write is expanded to individual +// resource reads and writes per associated partitioned resource handle. +LogicalResult PartitionResourceReadsWrites( + tf_device::ClusterFuncOp cluster_func) { + bool use_spmd = false; + if (auto use_spmd_attr = cluster_func->getAttrOfType(kUseSpmdAttr)) + use_spmd = use_spmd_attr.getValue(); + + if (!use_spmd) return success(); + + auto num_cores_per_replica_attr = + cluster_func->getAttrOfType(kNumCoresPerReplicaAttr); + + // Wrap the ClusterFunc with a ParallelExecute if it does not already exist. + OpBuilder builder(cluster_func); + tf_device::ParallelExecuteOp parallel_execute = + cluster_func->getParentOfType(); + if (!parallel_execute) + parallel_execute = mlir::TF::BuildParallelExecuteOp(cluster_func, &builder); + + // Rewrite results before rewriting operands as `tf.TPUPartitionedInputV2` + // resource handle results is an indicator for a partitioned resource + // variable. These `tf.TPUPartitionedInputV2` will be removed when rewriting + // the operands. + for (Value result : parallel_execute.getExecuteOutputs()) { + if (!result.hasOneUse()) continue; + auto assign_var = + llvm::dyn_cast(*result.getUsers().begin()); + if (!assign_var || assign_var.getValue() != result) continue; + auto partitioned_input = + llvm::dyn_cast_or_null( + assign_var.getResource().getDefiningOp()); + if (!partitioned_input || + !AllResourceTypesHaveSubtypes(partitioned_input.getInputs().getTypes())) + continue; + + const auto inputs = partitioned_input.getInputs(); + const bool packed_input = partitioned_input.getIsPacked(); + int num_cores_per_replica = partitioned_input.getN(); + if (num_cores_per_replica_attr) { + num_cores_per_replica = num_cores_per_replica_attr.getInt(); + } else if (packed_input) { + return partitioned_input->emitOpError() + << "num cores per replica unavailable"; + } + + const int num_operands_expected = packed_input ? 1 : num_cores_per_replica; + if (num_cores_per_replica_attr && num_operands_expected != inputs.size()) { + return partitioned_input->emitOpError() + << "expects " << num_operands_expected << " operands but found " + << partitioned_input.getNumOperands(); + } + + builder.setInsertionPoint(assign_var); + llvm::SmallVector partitioned_output_types; + partitioned_output_types.reserve(num_cores_per_replica); + for (int i = 0; i < num_cores_per_replica; ++i) { + const auto& input = packed_input ? inputs[0] : inputs[i]; + partitioned_output_types.push_back(GetResourceSubtype(input.getType())); + } + + auto partitioned_output = TF::TPUPartitionedOutputV2Op::create( + builder, cluster_func->getLoc(), partitioned_output_types, result, + partitioned_input.getPartitionDimsAttr(), + partitioned_input.get_XlaShardingAttr(), + partitioned_input.get_XlaShardingV2Attr()); + for (auto [i, value] : llvm::enumerate(partitioned_output.getOutput())) { + const auto& resource = packed_input ? inputs[0] : inputs[i]; + TF::AssignVariableOp::create(builder, assign_var->getLoc(), + /*resource=*/resource, /*value=*/value); + } + assign_var.erase(); + } + + for (OpOperand& operand : cluster_func->getOpOperands()) { + auto read_var = llvm::dyn_cast_or_null( + operand.get().getDefiningOp()); + if (!read_var) continue; + auto partitioned_input = + llvm::dyn_cast_or_null( + read_var.getResource().getDefiningOp()); + if (!partitioned_input || !AllResourceTypesHaveSubtypes( + partitioned_input.getInputs().getTypes())) { + continue; + } + + // we only want to create one read variable op per unique input + // otherwise tpu rewriting will fail to clean up the duplicates + llvm::SmallMapVector read_variable_ops; + llvm::SmallVector partitioned_reads; + builder.setInsertionPoint(partitioned_input); + + bool is_packed = partitioned_input.getIsPacked(); + const auto func = partitioned_input->getParentOfType(); + + for (Value input : partitioned_input.getInputs()) { + // At this stage, the op may've been rewritten to be "unpacked" but + // taking multiple copies of a resource on a COMPOSITE device + is_packed = + is_packed || IsCompositeDevice(GetDeviceOfResource(func, input)); + // If a read variable op already doesn't exist for this input, create it + auto search = read_variable_ops.find(input); + if (search == read_variable_ops.end()) { + auto partitioned_read = TF::ReadVariableOp::create( + builder, read_var->getLoc(), GetResourceSubtype(input), input); + search = read_variable_ops.insert({input, partitioned_read.getValue()}) + .first; + } + partitioned_reads.push_back(search->second); + } + + auto partitioned_read = TF::TPUPartitionedInputV2Op::create( + builder, partitioned_input->getLoc(), read_var.getValue().getType(), + partitioned_reads, partitioned_input.getPartitionDimsAttr(), + partitioned_input.getIsPackedAttr(), + partitioned_input.get_XlaShardingAttr(), + partitioned_input.get_XlaShardingV2Attr()); + if (failed(UpdateReadUses(read_var, partitioned_input, partitioned_read, + partitioned_reads, is_packed))) + return failure(); + read_var->erase(); + if (partitioned_input->use_empty()) partitioned_input->erase(); + } + return mlir::TF::RemoveSingletonParallelExecuteOp(parallel_execute, &builder); +} + +void TPUResourceReadsWritesPartitioningPass::runOnOperation() { + llvm::SmallVector cluster_funcs; + getOperation()->walk([&cluster_funcs](tf_device::ClusterFuncOp cluster_func) { + cluster_funcs.push_back(cluster_func); + }); + for (tf_device::ClusterFuncOp cluster_func : cluster_funcs) + if (failed(PartitionResourceReadsWrites(cluster_func))) + return signalPassFailure(); +} + +} // namespace + +std::unique_ptr> +CreateTPUResourceReadsWritesPartitioningPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_read_for_write.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_read_for_write.cc new file mode 100644 index 0000000000000000000000000000000000000000..60e36fc68f76d132ccb9dc4747c11fa1f1799b57 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_resource_read_for_write.cc @@ -0,0 +1,144 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TFTPU { + +#define GEN_PASS_DEF_TPURESOURCEREADFORWRITEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// A pass that finds TPU clusters with write only resource access and adds an +// associated resource read, so the resource can later be fused into TPUExecute. +namespace { +struct TPUResourceReadForWritePass + : public impl::TPUResourceReadForWritePassBase< + TPUResourceReadForWritePass> { + void runOnOperation() override; +}; + +// Helper struct holding a resource value and its associated type. +struct ResourceValueAndSubtype { + Value resource; + Type subtype; +}; + +// Finds resource handle and type for result if result writes to a resource. +ResourceValueAndSubtype GetResourceWriteResult( + tf_device::ClusterFuncOp cluster_func, Value result) { + ResourceValueAndSubtype resource; + if (!result.hasOneUse()) return resource; + Operation* result_user = *result.getUsers().begin(); + auto assign_var = dyn_cast(result_user); + if (!assign_var) return resource; + + auto handle = assign_var.getResource(); + // Skip result if cluster writes to the same variable via multiple results. + for (Operation* handle_user : handle.getUsers()) { + if (handle_user == assign_var) continue; + auto assign_var_user = dyn_cast(handle_user); + if (!assign_var_user) continue; + if (assign_var_user.getValue().getDefiningOp() == cluster_func) + return resource; + } + + resource.resource = assign_var.getResource(); + resource.subtype = assign_var.getValue().getType(); + return resource; +} + +// Checks if resource is read by TPU cluster. +bool ClusterFuncHasResourceRead(tf_device::ClusterFuncOp cluster_func, + Value resource) { + for (Operation* resource_user : resource.getUsers()) { + if (auto read = dyn_cast(resource_user)) { + for (Operation* read_user : read.getValue().getUsers()) { + if (read_user == cluster_func) return true; + if (isa(read_user)) return true; + } + } + } + + return false; +} + +void TPUResourceReadForWritePass::runOnOperation() { + SmallVector cluster_funcs; + getOperation().walk([&](tf_device::ClusterFuncOp cluster_func) { + cluster_funcs.push_back(cluster_func); + }); + + OpBuilder builder(&getContext()); + // Add resource reads for resource writes from TPU cluster where for such + // resources the TPU cluster does not read from. + for (tf_device::ClusterFuncOp cluster_func : cluster_funcs) { + builder.setInsertionPoint(cluster_func); + + SmallVector read_operands; + for (Value result : cluster_func.getResults()) { + // TODO(lyandy): Update pass to use resource alias analysis. + auto resource_and_type = GetResourceWriteResult(cluster_func, result); + if (!resource_and_type.resource) continue; + if (ClusterFuncHasResourceRead(cluster_func, resource_and_type.resource)) + continue; + auto new_read = TF::ReadVariableOp::create( + builder, resource_and_type.resource.getLoc(), + resource_and_type.subtype, resource_and_type.resource); + read_operands.push_back(new_read.getValue()); + } + + if (read_operands.empty()) continue; + + // Update caller and function types with new read operands. + auto operands = llvm::to_vector<4>(cluster_func.getOperands()); + operands.append(read_operands.begin(), read_operands.end()); + + auto loc = cluster_func.getLoc(); + auto new_cluster_func = tf_device::ClusterFuncOp::create( + builder, loc, cluster_func.getResultTypes(), operands, + cluster_func->getAttrs()); + cluster_func.replaceAllUsesWith(new_cluster_func); + func::FuncOp func = cluster_func.getFuncOp(); + Block& block = func.front(); + for (Value read_operand : read_operands) + block.addArgument(read_operand.getType(), loc); + + func.setType(FunctionType::get(&getContext(), block.getArgumentTypes(), + func.getResultTypes())); + cluster_func.erase(); + } +} + +} // namespace + +std::unique_ptr> CreateTPUResourceReadForWritePass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_space_to_depth_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_space_to_depth_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..33bd3938cdd253b05e94dc243f8ae1cb901a087c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_space_to_depth_pass.cc @@ -0,0 +1,717 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace mlir { +namespace TFTPU { + +namespace { + +constexpr char kDeviceAttr[] = "device"; +typedef std::pair Conv2DWithBlockSize; + +struct BlockArgumentInfo { + unsigned arg_num; + unsigned num_users; +}; + +#define GEN_PASS_DEF_TPUSPACETODEPTHPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// TODO(wangtao): add a pass to check if it is profitable to space to depth +// transform and invoke the transform if it is needed. +struct TPUSpaceToDepthPass + : public impl::TPUSpaceToDepthPassBase { + void runOnOperation() override; +}; + +// Updates func argument type to have the updated input shape. +void UpdateFuncType(func::FuncOp func) { + auto arg_types = func.front().getArgumentTypes(); + auto result_types = func.front().getTerminator()->getOperandTypes(); + func.setType(FunctionType::get(func.getContext(), arg_types, result_types)); +} + +void HandleFuncOp(Operation* op) { + auto func = llvm::cast(op); + UpdateFuncType(func); +} + +// Handles cast op between the first convolution and the block argument. +LogicalResult HandleCast(TF::CastOp cast_op, ArrayRef new_shape) { + auto cast_input = cast_op.getX(); + // Update input type. + auto transform_result_type = + RankedTensorType::get(new_shape, getElementTypeOrSelf(cast_input)); + cast_input.setType(transform_result_type); + auto block_arg = mlir::dyn_cast(cast_input); + auto cast_op_input = dyn_cast_or_null(cast_input.getDefiningOp()); + while (block_arg || cast_op_input) { + if (block_arg) { + // Change on device function type/shape. + HandleFuncOp(block_arg.getOwner()->getParentOp()); + block_arg = nullptr; + cast_op_input = nullptr; + } else { + auto cast_input = cast_op_input.getX(); + // Update input type. + auto transform_result_type = + RankedTensorType::get(new_shape, getElementTypeOrSelf(cast_input)); + cast_input.setType(transform_result_type); + // Update block arg and cast_op_input. + block_arg = mlir::dyn_cast(cast_input); + cast_op_input = dyn_cast_or_null(cast_input.getDefiningOp()); + } + } + return success(); +} + +// Handles padding before convolution for space to depth transform. +LogicalResult HandlePad(TF::PadOp op, int32_t kernel_size, int32_t block_size) { + auto ranked_type = mlir::dyn_cast(op.getInput().getType()); + if (!ranked_type) return failure(); + auto pad_input_shape = ranked_type.getShape(); + Location loc = op.getLoc(); + OpBuilder builder(op); + builder.setInsertionPoint(op); + auto padding_type = RankedTensorType::get({4, 2}, builder.getIntegerType(32)); + + // Calculate paddings. + int32_t pad_total = kernel_size - 1; + int32_t pad_beg = (pad_total / 2 + 1) / block_size; + int32_t pad_end = (pad_total / 2) / block_size; + SmallVector values = {0, 0, pad_beg, pad_end, + pad_beg, pad_end, 0, 0}; + auto paddings = DenseIntElementsAttr::get(padding_type, values); + // Update pad_op paddings. + op.setOperand(1, TF::ConstOp::create(builder, loc, paddings)); + + // Set input type. + auto input = op.getOperand(0); + SmallVector transform_shape = { + pad_input_shape[0], pad_input_shape[1] / block_size, + pad_input_shape[2] / block_size, + pad_input_shape[3] * block_size * block_size}; + // Input of the pad op could be a cast op. + if (auto cast_op = dyn_cast_or_null(input.getDefiningOp())) + if (failed(HandleCast(cast_op, transform_shape))) return failure(); + + auto transform_result_type = + RankedTensorType::get(transform_shape, getElementTypeOrSelf(input)); + input.setType(transform_result_type); + op.setOperand(0, input); + return success(); +} + +// Handles stride for the first convolution for the transform. +void HandleConv2DStride(TF::Conv2DOp conv2d) { + MLIRContext* context = conv2d.getContext(); + SmallVector values = {1, 1, 1, 1}; + auto attrs = llvm::map_range(values, [context](int64_t v) -> Attribute { + return IntegerAttr::get(IntegerType::get(context, 64), v); + }); + // TODO(b/157276506): change type of strides to DenseElementsAttr + auto strides = ArrayAttr::get(context, llvm::to_vector<4>(attrs)); + conv2d->setAttr("strides", strides); +} + +// Transforms input shape for the first convolution. +void HandleConv2DInput(TF::Conv2DOp conv2d, int64_t block_size) { + auto input = conv2d.getInput(); + auto input_shape = mlir::cast(input.getType()).getShape(); + SmallVector transform_shape = { + input_shape[0], input_shape[1] / block_size, input_shape[2] / block_size, + input_shape[3] * block_size * block_size}; + auto transform_result_type = + RankedTensorType::get(transform_shape, getElementTypeOrSelf(input)); + input.setType(transform_result_type); +} + +// Adds padding for convolution filter for space to depth transform. +TF::PadOp GetPadOpForConv2DFilter(ArrayRef filter_shape, Value filter, + OpBuilder* builder, int32_t pad_h, + int32_t pad_w) { + SmallVector values = {pad_h, 0, pad_w, 0, 0, 0, 0, 0}; + auto padding_type = + RankedTensorType::get({4, 2}, builder->getIntegerType(32)); + auto paddings = DenseIntElementsAttr::get(padding_type, values); + auto paddings_value = + TF::ConstOp::create(*builder, filter.getLoc(), paddings); + std::vector pad_shape = {filter_shape[0] + pad_h, + filter_shape[1] + pad_w, filter_shape[2], + filter_shape[3]}; + SmallVector expand_shape(pad_shape.begin(), pad_shape.end()); + + auto expand_result_type = + RankedTensorType::get(expand_shape, getElementTypeOrSelf(filter)); + return TF::PadOp::create(*builder, filter.getLoc(), expand_result_type, + filter, paddings_value); +} + +// Creates reshape op for space to depth transform. +TF::ReshapeOp GetReshapeOpForConv2DFilter(ArrayRef new_shape, + Value input, OpBuilder* builder) { + auto reshape_result_type = + RankedTensorType::get(new_shape, getElementTypeOrSelf(input)); + auto reshape_type = RankedTensorType::get( + {static_cast(new_shape.size())}, builder->getIntegerType(64)); + auto reshape_sizes = DenseIntElementsAttr::get(reshape_type, new_shape); + auto reshape_value = + TF::ConstOp::create(*builder, input.getLoc(), reshape_sizes); + return TF::ReshapeOp::create(*builder, input.getLoc(), reshape_result_type, + input, reshape_value); +} + +// Creates transpose op for shape to depth transform. +TF::TransposeOp GetTransposeOpForConv2DFilter(OpBuilder* builder, Value input) { + SmallVector permutation = {0, 2, 1, 3, 4, 5}; + auto permute_type = RankedTensorType::get({6}, builder->getIntegerType(32)); + auto permute_attr = DenseIntElementsAttr::get(permute_type, permutation); + auto permute_value = + TF::ConstOp::create(*builder, input.getLoc(), permute_attr); + return TF::TransposeOp::create(*builder, input.getLoc(), input, + permute_value); +} + +void HandleConv2DFilter(TF::Conv2DOp conv2d, int64_t block_size) { + // For example, if filter shape is [7, 7, 3, 64] with block_size 2, + // will apply below transforms to the filter: + // 1. Pad the filter to [8, 8, 3, 64] + // 2. Reshape to [4, 2, 4, 2, 3, 64] + // 3. Transpose to [4, 4, 2, 2, 3, 64] + // 4. Reshape to [4, 4, 12, 64] + auto filter = conv2d.getFilter(); + OpBuilder builder(conv2d); + builder.setInsertionPoint(conv2d); + // Book keeping filter information. + auto filter_shape = mlir::cast(filter.getType()).getShape(); + int64_t height = filter_shape[0]; + int64_t width = filter_shape[1]; + int64_t channel = filter_shape[2]; + int64_t out_channel = filter_shape[3]; + // Value/Op before reshape op. + Value before_reshape_value = filter; + if (height % block_size != 0 || width % block_size != 0) { + // Calculate paddings for height and width. + int32_t pad_h = block_size - height % block_size; + int32_t pad_w = block_size - width % block_size; + auto pad_op = + GetPadOpForConv2DFilter(filter_shape, filter, &builder, pad_h, pad_w); + // Update op, height and width before reshape. + before_reshape_value = pad_op; + height = height + pad_h; + width = width + pad_w; + } + + // Reshape. + SmallVector new_shape = { + height / block_size, block_size, width / block_size, + block_size, channel, out_channel}; + auto reshape_op = + GetReshapeOpForConv2DFilter(new_shape, before_reshape_value, &builder); + + // Transpose. + auto transpose_op = GetTransposeOpForConv2DFilter(&builder, reshape_op); + + // Reshape Back. + SmallVector final_shape = { + height / block_size, width / block_size, + channel * block_size * block_size, out_channel}; + auto final_reshape_op = + GetReshapeOpForConv2DFilter(final_shape, transpose_op, &builder); + // Update filter of Conv2D. + conv2d.setOperand(1, final_reshape_op); +} + +// Creates slice op for filter in back prop pass. +TF::SliceOp GetSliceOpForConv2DBackPropFilter( + ArrayRef old_filter_shape, Value input, OpBuilder* builder) { + SmallVector slice_size(old_filter_shape.begin(), + old_filter_shape.end()); + auto slice_result_type = + RankedTensorType::get(slice_size, getElementTypeOrSelf(input)); + auto slice_size_op = TF::ConstOp::create( + *builder, input.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get({4}, builder->getIntegerType(32)), + old_filter_shape)); + SmallVector slice_start_position = {0, 0, 0, 0}; + auto start_position_type = + RankedTensorType::get({4}, builder->getIntegerType(64)); + auto start_position = TF::ConstOp::create( + *builder, input.getLoc(), + DenseIntElementsAttr::get(start_position_type, slice_start_position)); + return TF::SliceOp::create(*builder, input.getLoc(), slice_result_type, input, + start_position, slice_size_op); +} + +// Transforms Conv2DBackPropFilter for space to depth. +void HandleConv2DBackPropFilter(TF::Conv2DBackpropFilterOp backprop, + ArrayRef old_filter_shape, + ArrayRef new_filter_shape, + int64_t block_size) { + OpBuilder builder(backprop); + builder.setInsertionPoint(backprop); + + auto input = backprop.getInput(); + // Get new filter size from new_filter_shape. + auto new_filter_sizes = TF::ConstOp::create( + builder, backprop.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get({4}, builder.getIntegerType(32)), + new_filter_shape)); + + // Set stride to [1, 1, 1, 1]. + MLIRContext* context = backprop.getContext(); + SmallVector values = {1, 1, 1, 1}; + auto attrs = llvm::map_range(values, [context](int64_t v) -> Attribute { + return IntegerAttr::get(IntegerType::get(context, 64), APInt(64, v)); + }); + auto strides = ArrayAttr::get(context, llvm::to_vector<4>(attrs)); + + // new result type. + SmallVector new_shape(new_filter_shape.begin(), + new_filter_shape.end()); + auto new_result_type = + RankedTensorType::get(new_shape, getElementTypeOrSelf(input)); + + // Build new BackPropFilterOp. + auto loc = backprop.getLoc(); + auto new_backprop = TF::Conv2DBackpropFilterOp::create( + builder, loc, new_result_type, input, new_filter_sizes, + backprop.getOutBackprop(), strides, backprop.getUseCudnnOnGpu(), + backprop.getPadding(), backprop.getExplicitPaddings(), + backprop.getDataFormat(), backprop.getDilations()); + + // For example, if new filter shape is [4, 4, 12, 64], old filter shape + // is [7, 7, 3, 64] with block_size 2. + // Below transforms will be applied to the filter: + // 1. Reshape to [4, 4, 2, 2, 3, 64]; + // 2. Transpose to [4, 2, 4, 2, 3, 64]; + // 3. Reshape to [8, 8, 3, 64]; + // 4. Slice to [7, 7, 3, 64]. + SmallVector first_reshape_shape = { + new_filter_shape[0], + new_filter_shape[1], + block_size, + block_size, + new_filter_shape[2] / (block_size * block_size), + new_filter_shape[3]}; + auto first_reshape_op = + GetReshapeOpForConv2DFilter(first_reshape_shape, new_backprop, &builder); + + // Transpose. + auto transpose_op = GetTransposeOpForConv2DFilter(&builder, first_reshape_op); + + // Last Reshape op. + SmallVector last_reshape_shape = { + new_filter_shape[0] * block_size, new_filter_shape[1] * block_size, + new_filter_shape[2] / (block_size * block_size), new_filter_shape[3]}; + auto final_reshape_op = + GetReshapeOpForConv2DFilter(last_reshape_shape, transpose_op, &builder); + + // create slice op. + auto slice_op = GetSliceOpForConv2DBackPropFilter(old_filter_shape, + final_reshape_op, &builder); + + // Update backprop's user with the slice op. + backprop.replaceAllUsesWith(slice_op.getResult()); +} + +// Checks if the input producer op is supported in this transform. Right now, we +// only check if it is a host tf.IteratorGetNext. +bool IsSupportedHostInputOp(Operation* op) { + TF::IteratorGetNextOp iter = llvm::dyn_cast(op); + if (!iter) return false; + auto device = op->getAttrOfType(kDeviceAttr); + if (!device) return false; + tensorflow::DeviceNameUtils::ParsedName parsed_device; + if (!tensorflow::DeviceNameUtils::ParseFullName(device.getValue().str(), + &parsed_device)) { + return false; + } + return parsed_device.type == "CPU"; +} + +// Builds a SpaceToDepthOp with the given get_layout op and input. +TF::SpaceToDepthOp BuildSpaceToDepth(tf_device::ClusterFuncOp cluster_func, + Value input, int32_t block_size, + ArrayRef input_shape) { + auto input_op = input.getDefiningOp(); + OpBuilder builder(input_op); + builder.setInsertionPointAfter(input_op); + SmallVector transform_shape = { + input_shape[0], input_shape[1] / block_size, input_shape[2] / block_size, + input_shape[3] * block_size * block_size}; + auto transform_result_type = + RankedTensorType::get(transform_shape, getElementTypeOrSelf(input)); + return TF::SpaceToDepthOp::create(builder, cluster_func.getLoc(), + transform_result_type, input, block_size); +} + +// Performs transformation for a non-replicated input. +TF::SpaceToDepthOp HandleHostInput(Value input, int64_t index, + tf_device::ClusterFuncOp cluster_func, + int32_t block_size, + ArrayRef input_shape) { + auto space_to_depth = + BuildSpaceToDepth(cluster_func, input, block_size, input_shape); + cluster_func.setOperand(index, space_to_depth); + return space_to_depth; +} + +// Performs transformation for replicated inputs. Returns true if this is a +// supported case (thus transform happened). +bool HandleHostReplicatedInputs(int64_t index, + tf_device::ClusterFuncOp cluster_func, + BlockArgument block_arg, + tf_device::ReplicateOp replicate, + int32_t block_size) { + // We need to know the devices to copy to. + if (!replicate.getDevices()) return false; + + MutableArrayRef inputs = + replicate.GetOperandsForBlockArgument(block_arg); + for (auto& input : inputs) { + auto input_op = input.get().getDefiningOp(); + if (!input_op || !IsSupportedHostInputOp(input_op)) return false; + } + for (auto entry : llvm::enumerate(inputs)) { + Value input = entry.value().get(); + auto ranked_type = mlir::dyn_cast(input.getType()); + if (!ranked_type) return false; + auto input_shape = ranked_type.getShape(); + auto space_to_depth = + BuildSpaceToDepth(cluster_func, input, block_size, input_shape); + entry.value().set(space_to_depth); + block_arg.setType(space_to_depth.getType()); + } + return true; +} + +// Performs transformation on a pair of execute and compile ops. The compile +// should not have other uses. +void HandleCluster(tf_device::ClusterFuncOp cluster_func, int32_t block_size, + unsigned arg_num) { + auto maybe_replicate = + llvm::dyn_cast(cluster_func->getParentOp()); + + llvm::SmallVector transform_input_indices; + for (const auto& input : llvm::enumerate(cluster_func.getOperands())) { + if (auto block_arg = mlir::dyn_cast(input.value())) { + if (block_arg.getArgNumber() != arg_num) continue; + // For a block argument, consider transforms only when it is a replicated + // input (defining ops will be outside the replicate node). + if (maybe_replicate == block_arg.getParentRegion()->getParentOp()) { + HandleHostReplicatedInputs(input.index(), cluster_func, block_arg, + maybe_replicate, block_size); + } + } else { + // For an op output, consider transforms only when 1) there is no + // replicateion or 2) it is outside the replicate node that encloses the + // execute node. (Because if the op is inside replicate, it is probably + // not on the host.) + if (input.index() != arg_num) continue; + auto input_op = input.value().getDefiningOp(); + if (maybe_replicate && + maybe_replicate.getBody().isAncestor(input_op->getParentRegion())) { + continue; + } + if (!IsSupportedHostInputOp(input_op)) continue; + auto ranked_type = + mlir::dyn_cast(input.value().getType()); + if (!ranked_type) continue; + auto input_shape = ranked_type.getShape(); + HandleHostInput(input.value(), input.index(), cluster_func, block_size, + input_shape); + } + } +} + +// Checks if input shape of convolution is good for space to depth transform. +bool Conv2DInputShapeCanTransform(Value input) { + auto ranked_type = mlir::dyn_cast(input.getType()); + if (!ranked_type) return false; + auto input_shape = ranked_type.getShape(); + int32_t batch_size = input_shape[0]; + int32_t channel = input_shape[3]; + if (batch_size > 8 || channel > 8) { + return false; + } + return true; +} + +// Get block argument id and number of users for the input arg. +std::optional GetBlockArgNum(Value arg) { + if (auto block_arg = mlir::dyn_cast(arg)) { + if (!Conv2DInputShapeCanTransform(arg)) return std::nullopt; + unsigned num_users = + std::distance(block_arg.getUsers().begin(), block_arg.getUsers().end()); + BlockArgumentInfo block_arg_info = {block_arg.getArgNumber(), num_users}; + return block_arg_info; + } + return std::nullopt; +} + +// Gets input block argument id and number of users for the input recursively. +// Current supported ops between convolution input and the block arguments are +// PadOp and CastOp. +std::optional GetInputBlockArgNum(Value input) { + auto block_arg_num = GetBlockArgNum(input); + if (block_arg_num.has_value()) return block_arg_num; + + Value next_input = input; + auto pad_op = dyn_cast_or_null(next_input.getDefiningOp()); + auto cast_op = dyn_cast_or_null(next_input.getDefiningOp()); + + while (pad_op || cast_op) { + if (pad_op) { + auto block_arg_num = GetBlockArgNum(pad_op.getInput()); + if (block_arg_num.has_value()) return block_arg_num; + next_input = pad_op.getInput(); + } else { + auto block_arg_num = GetBlockArgNum(cast_op.getX()); + if (block_arg_num.has_value()) return block_arg_num; + next_input = cast_op.getX(); + } + pad_op = dyn_cast_or_null(next_input.getDefiningOp()); + cast_op = dyn_cast_or_null(next_input.getDefiningOp()); + } + + return std::nullopt; +} + +// Checks if a convoluton can apply SpaceToDepth transform. +// Only the first convolution in the graph whose batch size smaller than 8 +// and its input feature size smaller than 8 can be transformed. +std::optional GetConv2DInputArgNum(TF::Conv2DOp conv2d) { + if (conv2d.getDataFormat() != "NHWC" || conv2d.getStrides().size() != 4) { + return std::nullopt; + } + // Current supported ops between convolution input and the block arguments are + // PadOp and CastOp. + return GetInputBlockArgNum(conv2d.getInput()); +} + +// Applies space to depth transform for the first convolution on TPU device. +void HandleFirstConvolution(TF::Conv2DOp conv2d, int64_t block_size) { + // Check if input and filter type are RankedTensorType. + auto input_tensor_type = + mlir::dyn_cast(conv2d.getInput().getType()); + auto filter_tensor_type = + mlir::dyn_cast(conv2d.getFilter().getType()); + if (!input_tensor_type || !filter_tensor_type) return; + // Book keeping filter shape for padding and backprop filter rewrite. + auto filter_shape = filter_tensor_type.getShape(); + SmallVector old_filter_shape(filter_shape.begin(), + filter_shape.end()); + // Handles input. + auto conv2d_input = conv2d.getInput(); + if (auto block_arg = mlir::dyn_cast(conv2d_input)) { + // Change on device function type/shape. + HandleFuncOp(block_arg.getOwner()->getParentOp()); + } + + if (auto pad_op = dyn_cast_or_null(conv2d_input.getDefiningOp())) { + // Rewrite pad_op before Convolutioin. + if (failed(HandlePad(pad_op, filter_shape[0], block_size))) return; + auto pad_input = pad_op.getInput(); + if (auto block_arg = mlir::dyn_cast(pad_input)) { + // Change on device function type/shape. + HandleFuncOp(block_arg.getOwner()->getParentOp()); + } + } + + // Handle Conv2D input, stride and filter. + HandleConv2DInput(conv2d, block_size); + HandleConv2DStride(conv2d); + HandleConv2DFilter(conv2d, block_size); + + // Book keeping new filter shape for backprop filter rewrite. + // Filter shape is defined in HandleConv2DFilter, thus it is RankedTensorType. + filter_shape = + mlir::cast(conv2d.getFilter().getType()).getShape(); + SmallVector new_filter_shape(filter_shape.begin(), + filter_shape.end()); + + // Rewrite Conv2DBackPropFilter that is the user of first convolution's input. + if (!conv2d_input.getDefiningOp()) return; + for (Operation* user : conv2d_input.getDefiningOp()->getUsers()) { + if (auto backprop = dyn_cast(user)) { + HandleConv2DBackPropFilter(backprop, old_filter_shape, new_filter_shape, + block_size); + } + } +} + +// Gets block size that is equal to stride from spatial dimension +// from convolution. +// Space to depth transform won't be triggered if block size <= 1. +int32_t GetConv2DBlockSize(TF::Conv2DOp conv2d) { + SmallVector strides(4, 1); + for (int i = 0; i < 3; ++i) { + strides[i] = mlir::cast(conv2d.getStrides()[i]).getInt(); + } + + // Space to depth only supports striding at spatial dimension. + if (strides[0] != 1 || strides[3] != 1) return 1; + + // Space to depth only supports height_stride == width_stride case. + if (strides[1] != strides[2]) return 1; + + return strides[1]; +} + +void TPUSpaceToDepthPass::runOnOperation() { + std::optional cluster_func; + // Space to depth only supports training loop. + auto func_result = getOperation().walk([&](tf_device::ClusterFuncOp cluster) { + cluster_func = cluster; + return WalkResult::interrupt(); + }); + + // Return if there is no tf_device::ClusterFuncOp in training loop. + if (!func_result.wasInterrupted() || !cluster_func.has_value()) { + return; + } + + // Get the function on device. + auto device_func = cluster_func->getFuncOp(); + if (!device_func) return; + + TF::Conv2DOp first_conv; + // A map maps block argument id to the convolutions consumes them. + llvm::SmallDenseMap> + argnum_and_convolutions; + // A map maps block argument id to the number of users. + llvm::SmallDenseMap argnum_num_users; + + // Find out the qualified convolutions and its block argument ids. + auto conv2d_result = device_func.walk([&](TF::Conv2DOp conv2d) { + std::optional arg_num_and_num_users = + GetConv2DInputArgNum(conv2d); + if (arg_num_and_num_users.has_value()) { + // Get block size for the first convolution. + int64_t block_size = GetConv2DBlockSize(conv2d); + auto arg_num = arg_num_and_num_users.value().arg_num; + auto num_users = arg_num_and_num_users.value().num_users; + argnum_and_convolutions[arg_num].emplace_back(conv2d, block_size); + argnum_num_users[arg_num] = num_users; + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (!conv2d_result.wasInterrupted()) { + return; + } + + // Iterate through block argument and its convolution users. Space to depth + // transform will be applied only if all the below conditions are satisfied: + // 1. All the users of the block argument will lead to convolutions; + // 2. block_size of for the space to depth transform for these convolutions + // are the same; + // 3. block_size of for the space to depth transform for these convolutions + // are larger than 1. + for (auto argnum_and_convolution : argnum_and_convolutions) { + auto arg_num = argnum_and_convolution.getFirst(); + auto conv2d_and_block_sizes = argnum_and_convolution.getSecond(); + // Continue if number of users of the block argment doesn't equal to number + // of transformable convolutions and there is no qualified convolution + // for transform or block size is smaller than 2. + if (argnum_num_users[arg_num] != conv2d_and_block_sizes.size() || + conv2d_and_block_sizes.empty()) { + argnum_and_convolutions.erase(arg_num); + continue; + } + int64_t block_size = conv2d_and_block_sizes[0].second; + if (block_size < 2) { + argnum_and_convolutions.erase(arg_num); + continue; + } + // Continue if not all the block sizes for space to depth transform are the + // same. + for (auto conv2d_and_block_size : conv2d_and_block_sizes) { + if (conv2d_and_block_size.second != block_size) { + argnum_and_convolutions.erase(arg_num); + break; + } + } + } + + // If there is no qualified space to depth transform. + if (argnum_and_convolutions.empty()) { + return; + } + + // Apply space to depth transform. + for (auto argnum_and_convolution : argnum_and_convolutions) { + auto conv2d_and_block_sizes = argnum_and_convolution.getSecond(); + int64_t block_size = conv2d_and_block_sizes[0].second; + // Apply space to depth transform to the input on the host. + HandleCluster(cluster_func.value(), block_size, + argnum_and_convolution.getFirst()); + // Transform the convolution. + for (auto conv2d_and_block_size : conv2d_and_block_sizes) { + HandleFirstConvolution(conv2d_and_block_size.first, + conv2d_and_block_size.second); + } + } +} + +} // namespace + +std::unique_ptr> CreateTPUSpaceToDepthPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_update_embedding_enqueue_op_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_update_embedding_enqueue_op_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..3a48f4adb418053ebc34267cbc230ecb6165ab73 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/tpu_update_embedding_enqueue_op_inputs.cc @@ -0,0 +1,185 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { +namespace TFTPU { +namespace { + +constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; +constexpr char kTPUEmbeddingAttr[] = "_tpu_embedding_layer"; + +#define GEN_PASS_DEF_TPUUPDATEEMBEDDINGENQUEUEOPINPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +struct TPUUpdateEmbeddingEnqueueOpInputsPass + : public impl::TPUUpdateEmbeddingEnqueueOpInputsPassBase< + TPUUpdateEmbeddingEnqueueOpInputsPass> { + void runOnOperation() override; +}; + +// Extracts `_tpu_embedding_layer` attribute from TPU embedding ops and +// clear the attribute from the operation. This ensures that future optimization +// passes does not trigger additional logic due to presence of this attribute. +LogicalResult ExtractEmbeddingAttribute( + Operation* op, llvm::StringMap* embedding_op_map) { + auto embedding_attr = op->getAttrOfType(kTPUEmbeddingAttr); + if (!embedding_attr) return mlir::success(); + + if (!embedding_op_map->insert({embedding_attr.getValue(), op}).second) + return op->emitOpError( + "found duplicate TPU embedding ops potentially from multiple " + "TPUEmbedding layers"); + + op->removeAttr(kTPUEmbeddingAttr); + return success(); +} + +LogicalResult FindTPUEmbeddingOps( + func::FuncOp func_op, llvm::StringMap* enqueue_op_map, + llvm::StringMap* recv_activation_op_map, + llvm::StringMap* send_gradient_op_map) { + auto walk_result = func_op.walk([&](Operation* op) { + if (llvm::isa(op)) + if (failed(ExtractEmbeddingAttribute(op, recv_activation_op_map))) + return WalkResult::interrupt(); + + if (llvm::isa(op)) + if (failed(ExtractEmbeddingAttribute(op, send_gradient_op_map))) + return WalkResult::interrupt(); + + if (llvm::isa(op)) + if (failed(ExtractEmbeddingAttribute(op, enqueue_op_map))) + return WalkResult::interrupt(); + + return WalkResult::advance(); + }); + return failure(walk_result.wasInterrupted()); +} + +// Updates the operand of TPU embedding enqueue ops depending on whether +// the graph is in training mode or in non-training mode. +// If SendTPUEmbeddingGradients op is present, this means that graph is in +// training mode. As so, correctly feed in `then` branch value of SelectV2 +// operand as inputs to the TPU embedding enqueue ops. +LogicalResult UpdateEmbeddingEnqueueOpInput( + const llvm::StringMap& enqueue_op_map, + const llvm::StringMap& recv_activation_op_map, + const llvm::StringMap& send_gradient_op_map, + OpBuilder* builder) { + for (const auto& it : enqueue_op_map) { + const auto& embedding_attr = it.getKey(); + Operation* embedding_op = it.second; + if (!recv_activation_op_map.count(embedding_attr)) + return embedding_op->emitOpError() + << "must have a corresponding '" + << TF::RecvTPUEmbeddingActivationsOp::getOperationName() << "' op"; + + // TPU Embedding enqueue ops take different inputs depending on whether + // graph is in training mode or in eval/prediction mode. During training, + // the mode parameter for TPUEmbeddingEnqueue op must be `train` and for + // evaluation or prediction, mode must be set to `inference`. + // If SendTPUEmbeddingGradients op exists in the graph, then graph is + // in training mode, so create a const op with value `train` use the + // output value of the constant as an operand to the TPU embedding + // enqueue op. + bool is_training = send_gradient_op_map.count(embedding_attr); + + // The last operand of TPUEmbeddingEnqueue ops is the mode which + // represents whether graph is in training mode or in evaluation mode. + auto& mode_enqueue_operand = + embedding_op->getOpOperand(embedding_op->getNumOperands() - 1); + + llvm::SmallVector mode_string_value; + mode_string_value.emplace_back(is_training ? "train" : "inference"); + builder->setInsertionPoint(embedding_op); + auto enqueue_mode = TF::ConstOp::create( + *builder, embedding_op->getLoc(), + DenseStringElementsAttr::get( + RankedTensorType::get({}, builder->getType()), + mode_string_value)); + + auto outside_compilation_attr = + embedding_op->getAttrOfType(kXlaOutsideCompilationAttr); + if (outside_compilation_attr) + enqueue_mode->setAttr(kXlaOutsideCompilationAttr, + outside_compilation_attr); + + mode_enqueue_operand.set(enqueue_mode); + } + + return success(); +} + +void TPUUpdateEmbeddingEnqueueOpInputsPass::runOnOperation() { + OpBuilder builder(&getContext()); + auto func_op = getOperation(); + + // All TPU embedding layer related ops are annotated with + // `_tpu_embedding_layer` attribute along with corresponding string attribute. + // Store all tpu embedding layer related ops with value of + // `_tpu_embedding_layer` attribute as map key. + llvm::StringMap enqueue_op_map; + llvm::StringMap recv_activation_op_map; + llvm::StringMap send_gradient_op_map; + if (failed(FindTPUEmbeddingOps(func_op, &enqueue_op_map, + &recv_activation_op_map, + &send_gradient_op_map))) + return signalPassFailure(); + + if (enqueue_op_map.size() != recv_activation_op_map.size()) { + func_op.emitError() << "expects the number of embedding enqueue ops to " + "match the number of '" + << TF::RecvTPUEmbeddingActivationsOp::getOperationName() + << "' ops"; + return signalPassFailure(); + } + + if (failed(UpdateEmbeddingEnqueueOpInput(enqueue_op_map, + recv_activation_op_map, + send_gradient_op_map, &builder))) + return signalPassFailure(); +} + +} // anonymous namespace + +std::unique_ptr> +CreateTPUUpdateEmbeddingEnqueueOpInputsPass() { + return std::make_unique(); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unfreeze_mutable_global_tensors.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unfreeze_mutable_global_tensors.cc new file mode 100644 index 0000000000000000000000000000000000000000..8b6580979664e5fad7980818528c0118ad70f9d7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unfreeze_mutable_global_tensors.cc @@ -0,0 +1,282 @@ +/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +#define DEBUG_TYPE "unfreeze-mutable-global-tensor" + +namespace mlir { +namespace tf_saved_model { + +namespace { + +#define GEN_PASS_DEF_UNFREEZEMUTABLEGLOBALTENSORSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_savedmodel_passes.h.inc" +struct UnfreezeMutableGlobalTensorsPass + : public impl::UnfreezeMutableGlobalTensorsPassBase< + UnfreezeMutableGlobalTensorsPass> { + UnfreezeMutableGlobalTensorsPass() = default; + UnfreezeMutableGlobalTensorsPass(const UnfreezeMutableGlobalTensorsPass&) {}; + void runOnOperation() override; +}; + +struct GlobalTensorAndUseIndex { + GlobalTensorOp global_tensor; + size_t arg_idx; +}; + +using GlobalTensorUsesMap = + llvm::DenseMap>; + +GlobalTensorUsesMap CreateGlobalTensorUsesMap(ModuleOp module) { + GlobalTensorUsesMap global_tensor_uses; + + SymbolTable symbol_table(module); + for (auto func : module.getOps()) { + global_tensor_uses[func] = {}; + for (size_t i = 0, e = func.getNumArguments(); i < e; i++) { + mlir::SymbolRefAttr sym = + func.getArgAttrOfType(i, "tf_saved_model.bound_input"); + if (!sym) { + continue; + } + GlobalTensorOp global_tensor = symbol_table.lookup( + mlir::cast(sym).getValue()); + if (!global_tensor || !global_tensor.getIsMutable()) { + continue; + } + global_tensor_uses[func].push_back({global_tensor, i}); + } + } + + return global_tensor_uses; +} + +// Adds the symbol to the "initializers" attribute of the session_initializer +// op. +void AddSymbolToInitializersAttr(SessionInitializerOp session_init_op, + FlatSymbolRefAttr symbol) { + const auto prev_initializers = session_init_op.getInitializersAttr(); + llvm::SmallVector initializers_attrs(prev_initializers.begin(), + prev_initializers.end()); + initializers_attrs.emplace_back(symbol); + + session_init_op.setInitializersAttr( + ArrayAttr::get(session_init_op.getContext(), initializers_attrs)); +} + +// Returns the session_initializer op in the module if exists. Otherwise, +// creates a new session_initializer op and returns it. +SessionInitializerOp GetOrCreateSessionInitializerOp(ModuleOp module_op) { + SessionInitializerOp session_init_op = GetSessionInitializerOp(module_op); + + // Create one if it doesn't exist. + if (!session_init_op) { + OpBuilder builder(&module_op.getBodyRegion()); + + session_init_op = SessionInitializerOp::create( + builder, module_op.getLoc(), /*initializers=*/builder.getArrayAttr({})); + } + + return session_init_op; +} + +// Create the initializer function right after the SessionInitializer op. +// Returns the newly created initializer function. The initializer function's +// initializer_type is set to "init_op" since it essentially serves as a +// variable initialization function. +func::FuncOp CreateInitializerFunc(ModuleOp module_op) { + SessionInitializerOp session_init_op = + GetOrCreateSessionInitializerOp(module_op); + + func::FuncOp init_func = GetInitializerFunction( + module_op, /*initializer_type=*/kTfSavedModelInitializerInitType); + if (init_func) { + return init_func; + } + + OpBuilder builder(module_op.getContext()); + builder.setInsertionPointAfter(session_init_op); + + const Location loc = builder.getUnknownLoc(); + const auto func_type = builder.getFunctionType(/*inputs=*/{}, /*results=*/{}); + + init_func = + func::FuncOp::create(builder, loc, /*sym_name=*/"NoOp", func_type); + builder.createBlock(&init_func.getBody(), /*insertPt=*/init_func.begin(), + /*arg_types=*/{}, /*arg_locs=*/{}); + + init_func->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr({"__tf_saved_model_session_initializer_NoOp"})); + init_func->setAttr(kTfSavedModelInitializerTypeAttr, + builder.getStringAttr(kTfSavedModelInitializerInitType)); + + builder.setInsertionPointToStart(&init_func.front()); + func::ReturnOp::create(builder, loc, /*operands=*/ValueRange()); + + SymbolTable symbol_table(module_op); + symbol_table.insert(init_func); + + AddSymbolToInitializersAttr( + session_init_op, FlatSymbolRefAttr::get(init_func.getSymNameAttr())); + + return init_func; +} + +// Returns the initializer function whose tf_saved_model.initializer_type +// is "init_op". Creates and returns a new initializer function iff such +// `FuncOp` is not found. The newly created initializer function's +// initializer_type is "init_op" and its symbol will be added to the symbol +// table and session_initializer op's "intializer" attribute. +func::FuncOp GetOrCreateInitializerFunc(ModuleOp module_op) { + if (auto init_func_op = GetInitializerFunction( + module_op, /*initializer_type=*/kTfSavedModelInitializerInitType); + init_func_op) { + return init_func_op; + } else { + // Create a new initializer function if the init function is not found. + return CreateInitializerFunc(module_op); + } +} + +// Collects the mutable GlobalTensorOps to unfreeze. +std::vector GetTargetGlobalTensorOps(ModuleOp module_op) { + std::vector target_global_tensor_ops; + for (auto global_tensor : module_op.getOps()) { + if (global_tensor.getIsMutable()) { + target_global_tensor_ops.push_back(global_tensor); + } + } + + return target_global_tensor_ops; +} + +// Inside `session_init_func`, creates AssignVariableOps(VarHandleOp, ConstOp) +// for each VarHandleOp that replaces a ConstOp. The `session_init_func` will +// essentially behave like init_op for the newly created VarHandleOps whose +// shared names are the values of `const_op_name_map`. +void CreateAssignVariableOps( + const std::vector& target_global_tensor_ops, + func::FuncOp session_init_func) { + OpBuilder builder(&session_init_func.getBody()); + + for (auto global_tensor : target_global_tensor_ops) { + llvm::StringRef shared_name = global_tensor.getSymName(); + const auto element_type = TF::ResourceType::get( + {mlir::cast(global_tensor.getType())}, + builder.getContext()); + + const auto ranked_tensor_type = RankedTensorType::get( + /*shape=*/{}, /*elementType=*/element_type); + auto var_handle_op = + TF::VarHandleOp::create(builder, global_tensor.getLoc(), + /*resource=*/ranked_tensor_type, + /*container=*/"", shared_name); + + // Assign the ConstOp to each VarHandleOp. These will be used to save the + // variable values to the checkpoint. + auto const_op_copy = TF::ConstOp::create(builder, global_tensor.getLoc(), + *global_tensor.getValue()); + + TF::AssignVariableOp::create(builder, global_tensor.getLoc(), + /*resource=*/var_handle_op, + /*value=*/const_op_copy.getOutput()); + } +} + +void UnfreezeMutableGlobalTensorsPass::runOnOperation() { + ModuleOp module_op = getOperation(); + + // Find the mutable GlobalTensorOps to "unfreeze" into VarHandleOps. + const std::vector target_global_tensor_ops = + GetTargetGlobalTensorOps(module_op); + + if (target_global_tensor_ops.empty()) { + return; + } + + func::FuncOp session_init_func = GetOrCreateInitializerFunc(module_op); + + // In the session initializer function, assign the const op's values to the + // corresponding VarHandleOps. + CreateAssignVariableOps(target_global_tensor_ops, session_init_func); + + // Remove the references to the mutable GlobalTensorOps. + GlobalTensorUsesMap global_tensor_uses = CreateGlobalTensorUsesMap(module_op); + for (auto& [func, global_tensor_and_use_indices] : global_tensor_uses) { + llvm::BitVector args_to_erase(func.getNumArguments()); + + for (auto& [global_tensor, arg_idx] : global_tensor_and_use_indices) { + args_to_erase.set(arg_idx); + + OpBuilder builder(func.getBody()); + builder.setInsertionPointToStart(&func.getBody().front()); + + llvm::StringRef shared_name = global_tensor.getSymName(); + const auto element_type = TF::ResourceType::get( + {mlir::cast(global_tensor.getType())}, + global_tensor.getContext()); + const auto ranked_tensor_type = RankedTensorType::get( + /*shape=*/{}, /*elementType=*/element_type); + + auto var_handle_op = + TF::VarHandleOp::create(builder, global_tensor.getLoc(), + /*resource=*/ranked_tensor_type, + /*container=*/"", shared_name); + + auto arg = func.getArguments()[arg_idx]; + arg.replaceAllUsesWith(var_handle_op->getResults()[0]); + } + + if (failed(func.eraseArguments(args_to_erase))) { + return signalPassFailure(); + } + } + + // Erase the mutable GlobalTensorOps that are replaced by VarHandleOps. + for (auto global_tensor_op : target_global_tensor_ops) { + global_tensor_op.erase(); + } +} + +} // namespace + +std::unique_ptr> +CreateUnfreezeMutableGlobalTensorsPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.cc new file mode 100644 index 0000000000000000000000000000000000000000..a2e114a09625f1603630d93dfa6321e4020a5aee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.cc @@ -0,0 +1,302 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.h" + +#include +#include +#include +#include + +#include "absl/container/inlined_vector.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/matmul_bcast.h" + +namespace mlir { +namespace TF { + +namespace { + +template +class ConvertTFBatchMatMulOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + static TF::ReshapeOp createReshapeOp(Value value, ArrayRef shape, + Type element_type, Location loc, + PatternRewriter& rewriter); + + static std::vector sliceInput(Value value, int batch_size, + Location loc, PatternRewriter& rewriter); + + LogicalResult matchAndRewrite(BatchMatMulOpType op, + PatternRewriter& rewriter) const override; +}; + +#define GEN_PASS_DEF_UNROLLBATCHMATMULPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// Unrolls a BatchMatMul on the batch dimension. We need to slice each batch out +// of the inputs, matmul them individually, then stack them all back together at +// the end. +struct UnrollBatchMatMulPass + : public impl::UnrollBatchMatMulPassBase { + void runOnOperation() override; +}; + +void UnrollBatchMatMulPass::runOnOperation() { + RewritePatternSet patterns(&getContext()); + auto func = getOperation(); + PopulateUnrollTfBatchMatMul(&getContext(), patterns); + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +} // namespace + +template +TF::ReshapeOp ConvertTFBatchMatMulOp::createReshapeOp( + Value value, ArrayRef shape, Type element_type, Location loc, + PatternRewriter& rewriter) { + int64_t shape_rank = shape.size(); + auto shape_spec_type = + RankedTensorType::get({shape_rank}, rewriter.getIntegerType(64)); + Type resultType = RankedTensorType::get(shape, element_type); + auto constant_attr = DenseElementsAttr::get(shape_spec_type, shape); + auto shape_tensor = + TF::ConstOp::create(rewriter, loc, shape_spec_type, constant_attr); + return TF::ReshapeOp::create(rewriter, loc, resultType, /*tensor=*/value, + /*shape=*/shape_tensor); +} + +template +std::vector ConvertTFBatchMatMulOp::sliceInput( + Value value, int batch_size, Location loc, PatternRewriter& rewriter) { + RankedTensorType tensorType = mlir::cast(value.getType()); + Type element_type = tensorType.getElementType(); + + int rank = tensorType.getShape().size(); + int num_rows = tensorType.getShape()[rank - 2]; + int num_cols = tensorType.getShape()[rank - 1]; + + std::vector sliced; + + if (batch_size == 1) { + // Batch size is 1, no splitting is required + // Squeeze the batch dimension, i.e. reshape + // [1, num_rows, num_cols] -> [num_rows, num_cols] + auto reshape_op = createReshapeOp(value, {num_rows, num_cols}, element_type, + loc, rewriter); + sliced.emplace_back(reshape_op.getOutput()); + } else { + // Reshape to rank-3 tensor with first dimension as the batch size. + auto reshape_op = createReshapeOp(value, {batch_size, num_rows, num_cols}, + element_type, loc, rewriter); + + // Create a constant op for the split axis (=0) + auto split_dimension_type = + RankedTensorType::get({}, rewriter.getIntegerType(32)); + auto split_dimension_attr = DenseElementsAttr::get(split_dimension_type, 0); + auto split_dimension_op = TF::ConstOp::create( + rewriter, loc, split_dimension_type, split_dimension_attr); + + // Split along each batch. + SmallVector slice_size = {1, num_rows, num_cols}; + Type slice_result_type = RankedTensorType::get(slice_size, element_type); + llvm::SmallVector output_types(batch_size, slice_result_type); + auto split_op = TF::SplitOp::create(rewriter, loc, output_types, + split_dimension_op.getOutput(), + reshape_op.getOutput()); + + // Squeeze each batch, i.e. reshape + // [1, num_rows, num_cols] -> [num_rows, num_cols] + for (const auto& split_value : split_op.getOutput()) { + auto reshape_op = createReshapeOp(split_value, {num_rows, num_cols}, + element_type, loc, rewriter); + + sliced.emplace_back(reshape_op.getOutput()); + } + } + return sliced; +} + +template +LogicalResult ConvertTFBatchMatMulOp::matchAndRewrite( + BatchMatMulOpType op, PatternRewriter& rewriter) const { + Value input_lhs = op.getX(); + Value input_rhs = op.getY(); + + if (!mlir::isa(input_lhs.getType())) { + // LHS must be a ranked tensor type + return failure(); + } + if (!mlir::isa(input_rhs.getType())) { + // RHS must be a ranked tensor type + return failure(); + } + + auto lhs_type = mlir::cast(input_lhs.getType()); + auto rhs_type = mlir::cast(input_rhs.getType()); + + // Skip int8 x int8 => int32. + if (lhs_type.getElementType().isInteger(8) && + rhs_type.getElementType().isInteger(8)) { + return rewriter.notifyMatchFailure(op, + "skip unrolling for int8 BatchMatMulV3"); + } + + auto element_type = lhs_type.getElementType(); + + if (element_type != rhs_type.getElementType()) { + // The element type of LHS must be the same with element type of RHS + return failure(); + } + + std::vector lhs_shape = lhs_type.getShape(); + std::vector rhs_shape = rhs_type.getShape(); + + Location loc = op.getLoc(); + + // Ensure that input ranks are at least 2. + const int lhs_dims = lhs_shape.size(); + const int rhs_dims = rhs_shape.size(); + if (lhs_dims < 2 || rhs_dims < 2) { + // Both inputs must have rank >= 2 + return failure(); + } + + // Replace the last 2 dimensions of LHS and RHS if necessary. + // The actual transpose is done by MatMulOp. + if (op.getAdjX()) { + std::swap(lhs_shape[lhs_dims - 1], lhs_shape[lhs_dims - 2]); + } + if (op.getAdjY()) { + std::swap(rhs_shape[rhs_dims - 1], rhs_shape[rhs_dims - 2]); + } + + const int64_t rows = lhs_shape[lhs_dims - 2]; + const int64_t cols = rhs_shape[rhs_dims - 1]; + + if (lhs_shape[lhs_dims - 1] != rhs_shape[rhs_dims - 2]) { + // Input dimensions must be compatible for multiplication. + return failure(); + } + + const auto matmul_type = RankedTensorType::get({rows, cols}, element_type); + + if (lhs_dims == 2 && rhs_dims == 2) { + // When both inputs are matrices, just replace the op with a matmul op. + rewriter.replaceOpWithNewOp(op, matmul_type, + /*a=*/input_lhs, + /*b=*/input_rhs, + /*transpose_a=*/op.getAdjX(), + /*transpose_b=*/op.getAdjY()); + return success(); + } + + // Input dimensions must be defined. MatMulBCast does not support partial + // shapes. + for (auto dim : lhs_shape) { + if (dim == mlir::ShapedType::kDynamic) { + return failure(); + } + } + for (auto dim : rhs_shape) { + if (dim == mlir::ShapedType::kDynamic) { + return failure(); + } + } + // Ensure that batch shapes are broadcastable. + tensorflow::MatMulBCast bcast( + absl::InlinedVector(lhs_shape.begin(), lhs_shape.end()), + absl::InlinedVector(rhs_shape.begin(), rhs_shape.end())); + + if (!bcast.IsValid()) { + // Input batch dimensions must be broadcastable + return failure(); + } + + // Compute slices for each batch in the LHS and RHS. + std::vector sliced_lhs = + sliceInput(input_lhs, bcast.x_batch_size(), loc, rewriter); + std::vector sliced_rhs = + sliceInput(input_rhs, bcast.y_batch_size(), loc, rewriter); + + // Compute (single batch) MatMul for each output batch. + std::vector matmuls; + matmuls.reserve(bcast.output_batch_size()); + for (int batch_idx : llvm::seq(0, bcast.output_batch_size())) { + int lhs_batch_idx, rhs_batch_idx; + if (bcast.IsBroadcastingRequired()) { + lhs_batch_idx = bcast.x_batch_indices()[batch_idx]; + rhs_batch_idx = bcast.y_batch_indices()[batch_idx]; + } else { + lhs_batch_idx = batch_idx; + rhs_batch_idx = batch_idx; + } + auto matmul = TF::MatMulOp::create(rewriter, loc, matmul_type, + /*a=*/sliced_lhs[lhs_batch_idx], + /*b=*/sliced_rhs[rhs_batch_idx], + /*transpose_a=*/op.getAdjX(), + /*transpose_b=*/op.getAdjY()); + matmuls.emplace_back(matmul.getProduct()); + } + + // Combine the result of each individual MatMul into a rank-3 tensor. + Type packed_type = RankedTensorType::get( + {bcast.output_batch_size(), rows, cols}, element_type); + const auto axis = rewriter.getI64IntegerAttr(0); + auto pack_op = + TF::PackOp::create(rewriter, loc, packed_type, /*values=*/matmuls, axis); + + // Reshape the rank-3 tensor into the correct output shape. + const auto& result_batch_shape = bcast.output_batch_shape().dim_sizes(); + std::vector result_shape(result_batch_shape.begin(), + result_batch_shape.end()); + result_shape.push_back(rows); + result_shape.push_back(cols); + + auto reshape_op = createReshapeOp(pack_op.getOutput(), result_shape, + element_type, loc, rewriter); + rewriter.replaceOp(op, reshape_op.getOutput()); + return success(); +} + +std::unique_ptr> CreateUnrollBatchMatMulPassPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir + +void mlir::TF::PopulateUnrollTfBatchMatMul(MLIRContext* context, + RewritePatternSet& patterns) { + patterns.add, + ConvertTFBatchMatMulOp, + ConvertTFBatchMatMulOp>(context); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.h new file mode 100644 index 0000000000000000000000000000000000000000..167361c09eae09586d032eccaffb3b52b34efa34 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/unroll_batch_matmul.h @@ -0,0 +1,39 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_UNROLL_BATCH_MATMUL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_UNROLL_BATCH_MATMUL_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/matmul_bcast.h" + +namespace mlir { +namespace TF { + +// Populate patterns to unroll tf.BatchMatMulV2 op into a sequence of TF ops. +// Since TFLite does not support BatchMatMul operation, it unrolls a BatchMatMul +// op into tf.Reshape, tf.Slice, tf.MatMul, tf.Pack, and tf.Reshape ops. +void PopulateUnrollTfBatchMatMul(MLIRContext* context, + RewritePatternSet& patterns); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_UNROLL_BATCH_MATMUL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/update_control_dependencies.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/update_control_dependencies.cc new file mode 100644 index 0000000000000000000000000000000000000000..cc54efdd19584aed0ef7377e642478566a39b21c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/update_control_dependencies.cc @@ -0,0 +1,212 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/log/log.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_remaining_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" + +namespace mlir { +namespace tf_executor { +namespace { + +// Comparator for `OpsInReverseProgramOrder`. +struct IsAfterInBlock { + bool operator()(Operation* op, Operation* other_op) const { + // This function has an average complexity of O(1). + return other_op->isBeforeInBlock(op); + } +}; + +#define GEN_PASS_DEF_EXECUTORUPDATECONTROLDEPENDENCIESPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class UpdateControlDependenciesPass + : public impl::ExecutorUpdateControlDependenciesPassBase< + UpdateControlDependenciesPass> { + public: + void runOnOperation() override; +}; + +void ClearControlInputs(Operation* op, int& num_control_inputs_removed) { + // We only call this function for island or fetch ops. The second pair of + // parentheses is needed for successful compilation. + assert((isa(op))); + if (auto island = dyn_cast(op)) { + num_control_inputs_removed += island.getControlInputs().size(); + island.getControlInputsMutable().clear(); + } else if (auto fetch = dyn_cast(op)) { + GraphOp graph = fetch->getParentOfType(); + int num_control_fetches = fetch.getNumOperands() - graph.getNumResults(); + if (num_control_fetches > 0) { + fetch.getFetchesMutable().erase(graph.getNumResults(), + num_control_fetches); + num_control_inputs_removed += num_control_fetches; + } + } +} + +void SetControlInputs( + Operation* op, + const llvm::SmallVector& preds_in_reverse_program_order, + int& num_control_inputs_added) { + // We only call this function for island or fetch ops. The second pair of + // parentheses is needed for successful compilation. + assert((isa(op))); + mlir::MutableOperandRange mutable_control_inputs = + isa(op) ? cast(op).getControlInputsMutable() + : cast(op).getFetchesMutable(); + // Add control inputs in program order of the defining ops. + for (auto iter = preds_in_reverse_program_order.rbegin(); + iter != preds_in_reverse_program_order.rend(); + ++iter) { + Operation* pred = *iter; + if (auto pred_island = dyn_cast(pred)) { + mutable_control_inputs.append(pred_island.getControl()); + } + } + num_control_inputs_added += preds_in_reverse_program_order.size(); +} + +// Fills `op_to_parallel_ids_map` from parallel execution attributes in `graph`. +// Returns `failure` iff any attribute is malformed. +LogicalResult FillOpToParallelIdsMap( + GraphOp graph, TF::OpToParallelIdsMap& op_to_parallel_ids_map) { + for (Operation& op : graph.GetBody()) { + auto island = dyn_cast(&op); + if (!island) continue; + + // We call `VerifyExportSuitable` in the beginning of the pass, so every + // island wraps a single op. + Operation& wrapped_op = island.GetBody().front(); + TF::ParallelExecutionIdPairs id_pairs; + if (failed(TF::ParseParallelExecutionIds(&wrapped_op, id_pairs))) { + wrapped_op.emitError() + << "Malformed " << TF::kParallelExecAnnotation << " attribute"; + return failure(); + } + if (id_pairs.empty()) continue; + + TF::ParallelIdsMap& ids_map = op_to_parallel_ids_map[island]; + for (const auto& [group_id, branch_id] : id_pairs) + ids_map[group_id] = branch_id; + } + return success(); +} + +// Fills `op_to_parallel_ids_map` from parallel execution attributes in +// `module_op`. Returns `failure` iff any attribute is malformed. +LogicalResult FillOpToParallelIdsMap( + ModuleOp module_op, TF::OpToParallelIdsMap& op_to_parallel_ids_map) { + auto result = module_op->walk([&](GraphOp graph) { + if (failed(FillOpToParallelIdsMap(graph, op_to_parallel_ids_map))) + return WalkResult::interrupt(); + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return failure(); + return success(); +} + +// Computes and sets direct control inputs for `op`. +void UpdateControlDependenciesForOp( + Operation* op, const TF::SideEffectAnalysis::Info& analysis_for_func, + int& num_control_inputs_removed, int& num_control_inputs_added) { + llvm::SmallVector control_deps = + analysis_for_func.DirectControlPredecessors(op); + ClearControlInputs(op, num_control_inputs_removed); + llvm::SmallVector preds_in_reverse_program_order( + control_deps.begin(), control_deps.end()); + std::sort(preds_in_reverse_program_order.begin(), + preds_in_reverse_program_order.end(), IsAfterInBlock()); + SetControlInputs(op, preds_in_reverse_program_order, + num_control_inputs_added); +} + +// This function updates all control dependencies in `func`, represented as +// control inputs for island and fetch ops of the graph body in `func`. +// We rely on side effect analysis and propagate the queried dependencies to +// the island and fetch ops. +LogicalResult UpdateAllControlDependencies( + func::FuncOp func, const TF::SideEffectAnalysis::Info& analysis_for_func) { + int num_control_inputs_removed = 0; + int num_control_inputs_added = 0; + + // We call `VerifyExportSuitable` in the beginning of the pass, so every + // function has a single graph op. + auto graph = cast(func.front().front()); + for (Operation& op_ref : graph.GetBody()) { + Operation* op = &op_ref; + // We only represent control dependencies between island and fetch ops. + if (!isa(op)) continue; + UpdateControlDependenciesForOp(op, analysis_for_func, + num_control_inputs_removed, + num_control_inputs_added); + } + VLOG(2) << "Number of control inputs removed: " << num_control_inputs_removed; + VLOG(2) << "Number of control inputs added: " << num_control_inputs_added; + return success(); +} + +void UpdateControlDependenciesPass::runOnOperation() { + ModuleOp module_op = getOperation(); + // This pass assumes that all functions are suitable for export, i.e., each + // function has a single tf_executor.graph op and all islands wrap single + // ops. + if (failed(tensorflow::VerifyExportSuitable(module_op))) { + signalPassFailure(); + return; + } + TF::OpToParallelIdsMap op_to_parallel_ids_map; + if (failed(FillOpToParallelIdsMap(module_op, op_to_parallel_ids_map))) { + signalPassFailure(); + return; + } + TF::SideEffectAnalysis side_effect_analysis(module_op, + op_to_parallel_ids_map); + for (auto func : module_op.getOps()) { + if (func.isExternal()) continue; + const auto& analysis_for_func = + side_effect_analysis.GetAnalysisForFunc(func); + if (failed(UpdateAllControlDependencies(func, analysis_for_func))) { + signalPassFailure(); + return; + } + } +} + +} // namespace + +std::unique_ptr> +CreateTFExecutorUpdateControlDependenciesPass() { + return std::make_unique(); +} + +} // namespace tf_executor +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..44f2a4fa15b5ebca54267f6e94fab507157797b8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass.cc @@ -0,0 +1,113 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Pass/Pass.h" +#include "absl/log/log.h" +#include "absl/strings/str_cat.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" + +namespace mlir { +namespace TFDevice { + +namespace { + +constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; + +#define GEN_PASS_DEF_VERIFYNOOUTSIDECOMPILATIONMARKERSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +class VerifyNoOutsideCompilationMarkersPass + : public impl::VerifyNoOutsideCompilationMarkersPassBase< + VerifyNoOutsideCompilationMarkersPass> { + public: + void runOnOperation() override; +}; + +bool IsLaunchOp(Operation& op) { + return dyn_cast(op) != nullptr; +} + +bool IsDeviceClusterOp(Operation& op) { + return dyn_cast(op) != nullptr; +} + +bool HasChildLaunchDeviceOp(Operation& op) { + auto cluster_op = dyn_cast(op); + if (cluster_op == nullptr) return false; + + auto walk_result = cluster_op->walk([&](Operation* op) { + if (IsLaunchOp(*op)) return WalkResult::interrupt(); + + return WalkResult::advance(); + }); + + return walk_result.wasInterrupted(); +} + +bool HasXlaOutsideCompilationMarker(Operation& op) { + return op.getAttrOfType(kXlaOutsideCompilationAttr) != nullptr; +} + +void VerifyNoOutsideCompilationMarkersPass::runOnOperation() { + Operation* func_op = getOperation(); + + auto walk_result = func_op->walk([&](Operation* op) { + if (IsDeviceClusterOp(*op) && HasChildLaunchDeviceOp(*op)) { + std::string launch_error = + absl::StrCat("Node `", op->getName().getStringRef().str(), "` ", + "is a launch op which should have been removed by " + "outside compilation"); + + op->emitError() << launch_error; + LOG(ERROR) << launch_error; + return WalkResult::interrupt(); + } + + if (HasXlaOutsideCompilationMarker(*op)) { + std::string outside_compilation_error = absl::StrCat( + "Node `", op->getName().getStringRef().str(), "` ", + "has _xla_outside_compilation set which should have been removed by " + "outside compilation"); + + op->emitError() << outside_compilation_error; + LOG(ERROR) << outside_compilation_error; + return WalkResult::interrupt(); + } + + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> +CreateVerifyNoOutsideCompilationMarkersPass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..06296aa229995061ba7d344f5ef8e021338e4b40 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_no_outside_compilation_markers_pass_test.cc @@ -0,0 +1,129 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "xla/tsl/platform/statusor.h" + +namespace mlir { +namespace TFDevice { + +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::mlir::hlo::test::GetMlirModuleFromString; + +class VerifyNoOutsideCompilationMarkersPassTest : public ::testing::Test { + protected: + void CreateModule(const char* module_string) { + TF_ASSERT_OK_AND_ASSIGN(module_, + GetMlirModuleFromString(module_string, &context_)); + + pm_ = std::make_unique(&context_); + pm_->addNestedPass( + CreateVerifyNoOutsideCompilationMarkersPass()); + } + + mlir::LogicalResult Run() { return pm_->run(module_.get()); } + + private: + MLIRContext context_; + OwningOpRef module_; + std::unique_ptr pm_; +}; + +TEST_F(VerifyNoOutsideCompilationMarkersPassTest, PassesValidOps) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); +} + +TEST_F(VerifyNoOutsideCompilationMarkersPassTest, + FailsXlaOutsideCompilationMarkers) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + "tf.B"() {_xla_outside_compilation = "cluster1"} : () -> () + func.return + } + })"; + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.failed()); +} + +TEST_F(VerifyNoOutsideCompilationMarkersPassTest, + FailsWithLaunchOpsInsideCluster) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %0 = "tf_device.cluster"() ({ + "tf_device.launch"() ({ + "tf.B"() : () -> () + tf_device.return // end device + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + + tf_device.return // end cluster + }) {cluster_attr = "cluster_attr"} : () -> tensor<*xi32> + func.return + } + })"; + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.failed()); +} + +TEST_F(VerifyNoOutsideCompilationMarkersPassTest, + PassesWithLaunchOpsOutsideCluster) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + "tf_device.launch"() ({ + "tf.B"() : () -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + func.return + } + })"; + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_suitable_for_graph_export_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_suitable_for_graph_export_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..07628cc6d9fd082b17c09f3921e909fe8656b820 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/verify_suitable_for_graph_export_pass.cc @@ -0,0 +1,48 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_VERIFYSUITABLEFOREXPORTPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class VerifySuitableForExportPass + : public impl::VerifySuitableForExportPassBase< + VerifySuitableForExportPass> { + public: + void runOnOperation() override { + if (failed(tensorflow::VerifyExportSuitable(getOperation()))) + return signalPassFailure(); + } +}; + +} // namespace + +std::unique_ptr> CreateVerifySuitableForExportPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_deserialization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_deserialization.cc new file mode 100644 index 0000000000000000000000000000000000000000..3e4fe62df4b4bce03a0654e2488deb545f65a34f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_deserialization.cc @@ -0,0 +1,290 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/Quant.h" // from @llvm-project // IWYU pragma: keep +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo // IWYU pragma: keep +#include "stablehlo/dialect/Serialization.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo // IWYU pragma: keep +#include "stablehlo/dialect/VhloOps.h" // from @stablehlo // IWYU pragma: keep +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_call_module_attrs.h" +#include "tensorflow/compiler/tf2xla/kernels/xla_call_module_loader.h" +#include "xla/tsl/platform/statusor.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_XLACALLMODULEDESERIALIZATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +// `tf.backend_config` is a DictionaryAttr, JAX2TF sets the value of its +// i64 attribute `called_index` to the TF function's name. +constexpr llvm::StringRef kTfBackendConfigAttrName = "tf.backend_config"; +constexpr llvm::StringRef kCalledIndexAttrName = "called_index"; +constexpr llvm::StringRef kCalledFuncAttrName = "called_func"; + +// Deserialize the StableHLO module embedded in XlaCallModuleOp's module +// attribute. +absl::StatusOr> DeserializeStablehlo(MLIRContext *context, + XlaCallModuleOp op) { + std::vector disabled_checks; + for (auto attr : op.getDisabledChecks().getAsRange()) { + disabled_checks.push_back(attr.getValue().str()); + } + std::vector platforms; + for (auto attr : op.getPlatforms().getAsRange()) { + platforms.push_back(attr.getValue().str()); + } + + // Set the deserialized StableHLO version to an attribute of the XlaCallModule + // op, this is used if when the module is re-serialized. + auto version = stablehlo::getPortableArtifactVersion(op.getModule()); + if (failed(version)) { + return absl::InvalidArgumentError( + "Failed to get the deserialized StableHLO version, XlaCallModuleOp " + "must have a valid StableHLO module serialized using " + "stablehlo::serializePortableArtifact APIs."); + } + Builder builder(context); + op->setAttr(kStablehloVersionAttrName, + builder.getStringAttr(version.value().toString())); + + TF_ASSIGN_OR_RETURN( + auto loader, + tensorflow::XlaCallModuleLoader::Create( + context, static_cast(op.getVersion()), op.getModule(), + std::move(disabled_checks), std::move(platforms), + /*num_invocation_args=*/op.getArgs().size(), + op.getHasTokenInputOutput(), op.getUseShardyPartitioner())); + return std::move(*loader).module(); +} + +// Renames functions in the stablehlo module to avoid naming conflicts with +// existing functions in the tf module. +// Sets _from_xla_call_module attribute for each stablehlo function. +// Returns the new stablehlo main function's name or error. +// +// If we directly insert stablehlo functions into tf module, MLIR will rename +// the stablehlo functions themselves in the tf module automatically to avoid +// naming conflicts. But we need to rename the function calls inside the +// stablehlo functions as well. So we first do this renaming in the stablehlo +// module itself without inserting into the tf module. +FailureOr RenameStablehloFunctions( + MLIRContext *context, SymbolTableCollection &symbol_tables, + ModuleOp tf_module, ModuleOp stablehlo_module) { + SymbolTable &tf_symbol_table = symbol_tables.getSymbolTable(tf_module); + // `stablehlo_module` is deleted right after the deserialization, so no need + // to store its `SymbolTable` to `SymbolTableCollection`. + SymbolTable stablehlo_symbol_table(stablehlo_module); + + Builder builder(context); + StringAttr main_func_name; + for (auto func : stablehlo_module.getOps()) { + const bool is_main_func = func.getSymName() == kStablehloMainFunctionName; + if (tf_symbol_table.lookup(func.getSymName())) { + if (failed(stablehlo_symbol_table.renameToUnique( + func, {&tf_symbol_table, &stablehlo_symbol_table}))) { + return func.emitError() + << "failed to rename StableHLO function " << func.getSymName(); + } + } + if (is_main_func) { + main_func_name = func.getSymNameAttr(); + } + func->setAttr(kFromXlaCallModuleAttrName, builder.getUnitAttr()); + } + if (!main_func_name) { + return stablehlo_module.emitError() + << "StableHLO module does not have an entry function"; + } + return main_func_name; +} + +// Moves functions from one module to another. +// The moved functions are set to private. +void MoveFunctions(SymbolTableCollection &symbol_tables, ModuleOp from, + ModuleOp to) { + SymbolTable &to_symbol_table = symbol_tables.getSymbolTable(to); + for (auto func : llvm::make_early_inc_range(from.getOps())) { + func->remove(); + func.setPrivate(); + to_symbol_table.insert(func); + } +} + +void CopyStablehloModuleAttrs(ModuleOp stablehlo_module, XlaCallModuleOp op) { + op->setAttr(kStablehloModuleAttrsAttrName, + stablehlo_module->getAttrDictionary()); +} + +// Symbolizes `called_index` attributes in custom all ops to `called_func`. +LogicalResult SymbolizeCustomCallCalledIndex( + ModuleOp module, llvm::ArrayRef function_list) { + WalkResult result = + module.walk([&](stablehlo::CustomCallOp op) { + if (!IsTfFuncCustomCall(op)) { + return WalkResult::advance(); + } + + auto backend_config = + op->getAttrOfType(kTfBackendConfigAttrName); + if (!backend_config) { + op->emitOpError() + << "is missing attribute '" << kTfBackendConfigAttrName << "'"; + return WalkResult::interrupt(); + } + + auto called_index_attr = mlir::dyn_cast_or_null( + backend_config.get(kCalledIndexAttrName)); + if (!called_index_attr) { + op->emitOpError() + << "is missing attribute '" << kCalledIndexAttrName << "'"; + return WalkResult::interrupt(); + } + int called_index = called_index_attr.getInt(); + if (called_index < 0 || called_index >= function_list.size()) { + op->emitOpError() + << "references function #" << called_index + << " but enclosing XlaCallModule has a function list of size " + << function_list.size(); + return WalkResult::interrupt(); + } + + llvm::SmallVector new_config; + // Copy the attributes in the current config except `called_index`. + for (auto attr : backend_config) { + if (attr.getName() != kCalledIndexAttrName) { + new_config.push_back(attr); + } + } + + Builder builder(op.getContext()); + // Sets the `called_index` attribute to the TF function's name. + new_config.push_back(builder.getNamedAttr(kCalledFuncAttrName, + function_list[called_index])); + + // Sets the `tf.backend_config` attribute to the `new_config`. + op->setAttr(kTfBackendConfigAttrName, + builder.getDictionaryAttr(new_config)); + + return WalkResult::advance(); + }); + return result.wasInterrupted() ? failure() : success(); +} + +LogicalResult DeserializeXlaCallModule(MLIRContext *context, + SymbolTableCollection &symbol_tables, + ModuleOp module, XlaCallModuleOp op) { + auto deserialized = DeserializeStablehlo(context, op); + if (!deserialized.ok()) { + return op.emitOpError() + << "failed to deserialize StableHLO module from XlaCallModule: " + << deserialized.status().ToString(); + } + OwningOpRef stablehlo_module = *std::move(deserialized); + + CopyStablehloModuleAttrs(*stablehlo_module, op); + + auto main_func = RenameStablehloFunctions(context, symbol_tables, module, + stablehlo_module.get()); + if (failed(main_func)) { + return failure(); + } + + // Translate `called_index` in TF function custom calls into symbol + // references. `function_list` attribute is needed after that. + llvm::SmallVector function_list( + op.getFunctionList().getAsRange()); + if (failed( + SymbolizeCustomCallCalledIndex(*stablehlo_module, function_list))) { + return failure(); + } + op.removeFunctionListAttr(); + + MoveFunctions(symbol_tables, *stablehlo_module, module); + + // Module is deserialized, we set an empty string to it instead removing + // it because it's a required attribute. + op.setModule(""); + // Set the stablehlo main function as a symbol attribute. This is required + // because we not only need this to look up the StableHLO function called by + // XlaCallModule, but also need the symbol reference to prevent DCE from + // removing the stablehlo functions from the top-level module. + op->setAttr(kStablehloEntryFunctionAttrName, SymbolRefAttr::get(*main_func)); + + return success(); +} + +class XlaCallModuleDeserializationPass + : public impl::XlaCallModuleDeserializationPassBase< + XlaCallModuleDeserializationPass> { + public: + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + XlaCallModuleDeserializationPassBase::getDependentDialects(registry); + mlir::func::registerAllExtensions(registry); + } + + void runOnOperation() override { + ModuleOp module = getOperation(); + SymbolTableCollection symbol_tables; + WalkResult result = module.walk([&](XlaCallModuleOp op) { + if (failed(DeserializeXlaCallModule(&getContext(), symbol_tables, module, + op))) { + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) { + return signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr> +CreateXlaCallModuleDeserializationPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_serialization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_serialization.cc new file mode 100644 index 0000000000000000000000000000000000000000..450859b21fa1c4c09cd401d4b66c67bbfb9d1cdd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_call_module_serialization.cc @@ -0,0 +1,284 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "stablehlo/dialect/Serialization.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo // IWYU pragma: keep +#include "stablehlo/dialect/Version.h" // from @stablehlo +#include "stablehlo/dialect/VhloOps.h" // from @stablehlo // IWYU pragma: keep +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/visitor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_call_module_attrs.h" + +namespace mlir { +namespace TF { +namespace { + +#define GEN_PASS_DEF_XLACALLMODULESERIALIZATIONPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" // IWYU pragma: keep + +// `tf.backend_config` is a DictionaryAttr, JAX2TF sets the value of its +// i64 attribute `called_index` to the TF function's name. +constexpr llvm::StringRef kTfBackendConfigAttrName = "tf.backend_config"; +constexpr llvm::StringRef kCalledIndexAttrName = "called_index"; +constexpr llvm::StringRef kCalledFuncAttrName = "called_func"; + +// Converts `called_func` attributes in custom call ops back to `called_index`. +FailureOr DesymbolizeCustomCallCalledIndex(ModuleOp module) { + Builder builder(module.getContext()); + + SmallVector function_list; + llvm::DenseMap called_indexes; + + WalkResult result = module.walk([&](stablehlo::CustomCallOp op) { + if (!IsTfFuncCustomCall(op)) { + return WalkResult::advance(); + } + + auto backend_config = + op->getAttrOfType(kTfBackendConfigAttrName); + if (!backend_config) { + op->emitOpError() << "is missing attribute '" << kTfBackendConfigAttrName + << "'"; + return WalkResult::interrupt(); + } + auto called_func = mlir::dyn_cast_or_null( + backend_config.get(kCalledFuncAttrName)); + if (!called_func) { + op->emitOpError() << "is missing attribute '" << kCalledFuncAttrName + << "'"; + return WalkResult::interrupt(); + } + + llvm::SmallVector new_config; + // Copy the attributes in the current config except `called_func`. + for (auto attr : backend_config) { + if (attr.getName() != kCalledFuncAttrName) { + new_config.push_back(attr); + } + } + + auto [it, inserted] = + called_indexes.insert({called_func, called_indexes.size()}); + if (inserted) { + function_list.push_back(called_func); + } + + // Set the `called_index` attribute to the TF function's name. + new_config.push_back(builder.getNamedAttr( + kCalledIndexAttrName, builder.getI64IntegerAttr(it->second))); + + // Set the `tf.backend_config` attribute to the `new_config`. + op->setAttr(kTfBackendConfigAttrName, + builder.getDictionaryAttr(new_config)); + + return WalkResult::advance(); + }); + if (result.wasInterrupted()) { + return failure(); + } + + return builder.getArrayAttr(function_list); +} + +// Creates a pruned module containing the XlaCallModule's entry function and +// other functions transitively called by the entry function. +FailureOr> PruneStablehloModule( + SymbolTableCollection& symbol_table, ModuleOp module, XlaCallModuleOp op) { + auto entry_func_symbol = + op->getAttrOfType(kStablehloEntryFunctionAttrName); + if (!entry_func_symbol) { + return op.emitOpError() << "does not have " + << kStablehloEntryFunctionAttrName << " attribute"; + } + auto entry_func = + symbol_table.lookupSymbolIn(module, entry_func_symbol); + if (!entry_func) { + return op.emitOpError() + << "references an unknown entry function " << entry_func_symbol; + } + + OpBuilder builder(module.getContext()); + + OwningOpRef stablehlo_module = + ModuleOp::create(builder, op.getLoc()); + builder.setInsertionPointToEnd(stablehlo_module->getBody()); + + // Copy all referenced StableHLO functions to the new module. + WalkResult result = WalkReachableFunctions( + entry_func, + [&](func::FuncOp f) -> WalkResult { + if (!f->hasAttr(kFromXlaCallModuleAttrName)) { + return WalkResult::advance(); + } + + auto cloned = llvm::cast(builder.clone(*f)); + cloned->removeAttr(kFromXlaCallModuleAttrName); + + if (f == entry_func) { + // Entry function must be public and has symbol name "@main". + cloned.setPublic(); + cloned.setName(kStablehloMainFunctionName); + } else { + cloned.setPrivate(); + } + + return WalkResult::advance(); + }, + &symbol_table); + if (result.wasInterrupted()) { + return failure(); + } + + // Rewrite `custom_call`'s `called_func` attribute to `called_index`. + auto function_list = DesymbolizeCustomCallCalledIndex(*stablehlo_module); + if (failed(function_list)) return failure(); + op.setFunctionListAttr(*function_list); + + // Restore the deserialized stablehlo module's attributes to the reconstructed + // stablehlo module. The stablehlo module's attributes can contain important + // information such as SPMD num_replicas and num_partitions. + auto original_stablehlo_module_attrs = + op->getAttrOfType(kStablehloModuleAttrsAttrName); + if (original_stablehlo_module_attrs) { + (*stablehlo_module)->setAttrs(original_stablehlo_module_attrs); + // Now, remove the attribute because later passes may not know how to handle + // it, we may encounter errors such as: + // "Unhandled attribute kind for attribute '_stablehlo_module_attrs'". + op->removeAttr(kStablehloModuleAttrsAttrName); + } + + return stablehlo_module; +} + +// Serializes the stablehlo module into bytecode. +FailureOr SerializeStablehlo(ModuleOp stablehlo_module, + StringRef target_version) { + std::string bytecode; + llvm::raw_string_ostream os(bytecode); + // We need to pass `allowOtherDialects=true` if + // `stablehlo_version >= 1.11.0`, since the lowered module from JAX can + // have a mix of StableHLO and Shardy dialects. + vhlo::Version mixed_serialization_ok = vhlo::Version(1, 11, 0); + bool allow_other_dialects = + mixed_serialization_ok <= vhlo::Version::fromString(target_version); + if (mlir::failed(stablehlo::serializePortableArtifact( + stablehlo_module, target_version, os, allow_other_dialects))) { + return stablehlo_module.emitError() + << "failed to serialize the pruned stablehlo module"; + } + return bytecode; +} + +// Serializes the stablehlo functions called by XlaCallModuleOp to bytecode +// and embeds the bytecode in XlaCallModuleOp's `module` attribute. +// +// The stablehlo functions include the function referred by XlaCallModuleOp's +// `_entry_function` attribute, and any stablehlo functions called transitively +// from the entry function. +LogicalResult SerializeXlaCallModule(SymbolTableCollection& symbol_table, + ModuleOp module, XlaCallModuleOp op) { + auto stablehlo_module = PruneStablehloModule(symbol_table, module, op); + if (failed(stablehlo_module)) { + return failure(); + } + + // Use the StableHLO version set during deserialization. + auto stablehlo_version = + op->getAttrOfType(kStablehloVersionAttrName); + if (!stablehlo_version) { + return op->emitError() << "does not have " << kStablehloVersionAttrName + << " attribute"; + } + + StringRef target_version = stablehlo_version.getValue(); + auto bytecode = SerializeStablehlo(**stablehlo_module, target_version); + if (failed(bytecode)) { + return failure(); + } + + op.setModule(*bytecode); + op->removeAttr(kStablehloEntryFunctionAttrName); + op->removeAttr(kStablehloVersionAttrName); + + return success(); +} + +// Removes the serialized stablehlo functions, because `XlaCallModuleOp` no +// longer has `_entry_function` attribute referencing the stablehlo main +// function, so all stablehlo functions are of no use in the top-level module. +// +// Walk the module to find functions with `_from_xla_call_module` attribute, +// and remove them. +void RemoveSerializedStablehloFunctions(ModuleOp module) { + module.walk([&](func::FuncOp f) { + if (f->hasAttr(kFromXlaCallModuleAttrName)) { + f->erase(); + } + }); +} + +class XlaCallModuleSerializationPass + : public impl::XlaCallModuleSerializationPassBase< + XlaCallModuleSerializationPass> { + public: + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::SymbolTableCollection symbol_table; + + mlir::WalkResult result = + module.walk([&](mlir::TF::XlaCallModuleOp xla_call_module) { + if (failed(SerializeXlaCallModule(symbol_table, module, + xla_call_module))) { + return mlir::WalkResult::interrupt(); + } + return mlir::WalkResult::advance(); + }); + if (result.wasInterrupted()) { + return signalPassFailure(); + } + + RemoveSerializedStablehloFunctions(module); + } +}; + +} // namespace + +std::unique_ptr> +CreateXlaCallModuleSerializationPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_inline_device_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_inline_device_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..f8ffd0c8b30e6965d6b398b8dcb53d9b37517c92 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_inline_device_ops.cc @@ -0,0 +1,85 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass remove Cluster ops by inlining Cluster ops. + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +namespace mlir { + +namespace { + +#define GEN_PASS_DEF_XLAINLINEDEVICEOPSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +// XlaInlineDeviceOps Pass will inline cluster op based in the parent region. +struct XlaInlineDeviceOpsPass + : public impl::XlaInlineDeviceOpsPassBase { + void runOnOperation() override; +}; + +void InlineDeviceOp(tf_device::ClusterOp cluster_op) { + auto& block = cluster_op.GetBody(); + + llvm::SmallVector non_terminator_ops; + for (Operation& op : block.without_terminator()) { + non_terminator_ops.push_back(&op); + } + for (auto op : non_terminator_ops) { + op->moveBefore(cluster_op); + } + + auto& return_op = cluster_op.GetBody().front(); + // This is the last op, should be tf_device::ReturnOp. + assert(mlir::isa(return_op)); + + Value old_val, new_val; + for (auto it : llvm::zip(cluster_op.getResults(), return_op.getOperands())) { + std::tie(old_val, new_val) = it; + old_val.replaceAllUsesExcept(new_val, &return_op); + } + + cluster_op.erase(); +} + +void XlaInlineDeviceOpsPass::runOnOperation() { + ModuleOp module = getOperation(); + + llvm::SmallVector ops; + module.walk( + [&](tf_device::ClusterOp cluster_op) { ops.push_back(cluster_op); }); + + for (auto cluster_op : ops) { + InlineDeviceOp(cluster_op); + } +} + +} // namespace + +namespace TFDevice { +std::unique_ptr> CreateXlaInlineDeviceOpsPass() { + return std::make_unique(); +} +} // namespace TFDevice + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_rewrite.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_rewrite.cc new file mode 100644 index 0000000000000000000000000000000000000000..2e39dad7da270dbf221e06bfc901e4b5ecb31242 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_rewrite.cc @@ -0,0 +1,111 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass converts stateful and stateless partitioned calls +// with _xla_compile_device_type attribute to XLA launch ops. + +#include + +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h" + +#define DEBUG_TYPE "tf-xla-rewrite" + +namespace mlir { +namespace { + +#define GEN_PASS_DEF_XLAREWRITEPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +struct XlaRewritePass : public impl::XlaRewritePassBase { + void runOnOperation() override; +}; + +void MoveResourceArgsToEnd(func::FuncOp callee) { + llvm::DenseMap mapping; + unsigned num_params = callee.getNumArguments(); + llvm::BitVector removed_params(num_params); + // Copy the resource-type parameters to the end. + for (unsigned i = 0; i < num_params; ++i) { + BlockArgument param = callee.getArgument(i); + if (mlir::isa(getElementTypeOrSelf(param.getType()))) { + removed_params.set(i); + callee.getBody().addArgument(param.getType(), param.getLoc()); + param.replaceAllUsesWith(callee.getArguments().back()); + removed_params.push_back(false); + } + } + // Remove old resource-type parameters. + callee.getBody().front().eraseArguments(removed_params); + // Update function type. + callee.setFunctionType(FunctionType::get(callee.getContext(), + callee.getBody().getArgumentTypes(), + callee.getResultTypes())); +} + +void RewriteCall(tf_device::ClusterFuncOp cluster_func_op, SymbolTable &symtab, + OpBuilder &builder) { + llvm::SmallVector non_resource_args, resource_args; + bool has_resources = false, in_order = true; + for (const Value &arg : cluster_func_op.getOperands()) { + if (!mlir::isa(getElementTypeOrSelf(arg.getType()))) { + non_resource_args.push_back(arg); + if (has_resources) in_order = false; + } else { + resource_args.push_back(arg); + has_resources = true; + } + } + + if (!in_order) { + // Functions do not get reused in practice, so skip the check for if the + // callee has been updated. + StringAttr callee_sym = cluster_func_op.getFuncAttr().getAttr(); + MoveResourceArgsToEnd(symtab.lookup(callee_sym)); + } + builder.setInsertionPoint(cluster_func_op); + auto xla_launch_op = TF::XlaLaunchOp::create( + builder, cluster_func_op.getLoc(), cluster_func_op.getResultTypes(), + /*constants=*/ValueRange({}), ValueRange(non_resource_args), + ValueRange(resource_args), cluster_func_op.getFuncAttr()); + + CopyDeviceAndUnderscoredAttributes(cluster_func_op, xla_launch_op); + cluster_func_op.replaceAllUsesWith(xla_launch_op.getResults()); + cluster_func_op.erase(); +} + +void XlaRewritePass::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTable symtab(module); + OpBuilder builder(&getContext()); + module.walk([&](tf_device::ClusterFuncOp cluster_func_op) { + RewriteCall(cluster_func_op, symtab, builder); + }); +} + +} // namespace + +namespace TFDevice { +std::unique_ptr> CreateXlaRewritePass() { + return std::make_unique(); +} + +} // namespace TFDevice +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_validate_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_validate_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..8965d0bdfc18ca7b5c0b13fe5f0fb70b207396ee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/transforms/xla_validate_inputs.cc @@ -0,0 +1,117 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/log/log.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h" + +namespace mlir { + +namespace { + +#define GEN_PASS_DEF_XLAVALIDATEINPUTSPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_device_passes.h.inc" + +// Validate input graph. +struct XlaValidateInputsPass + : public impl::XlaValidateInputsPassBase { + void runOnOperation() override; +}; + +LogicalResult HasNoNestedEntryFunctions( + const llvm::SmallVector &entry_funcs, SymbolTable &symtab) { + auto calls_entry_functions = [&](SymbolUserOpInterface op) { + llvm::SmallVector callees; + if (GetCallees(op, symtab, callees).failed()) { + return false; + } + for (auto &callee : callees) { + if (IsEntryFunction(callee)) { + return true; + } + } + return false; + }; + + for (auto &entry_func : entry_funcs) { + llvm::SmallVector calls; + if (GetFirstOpsOfType( + entry_func, symtab, /*predicate*/ calls_entry_functions, calls) + .failed()) { + return failure(); + } + if (!calls.empty()) { + // This is not expected to happen in practice. We can add a pass after + // GuaranteeAllFuncsOneUsePass to remove "tf.entry_function" or + // "tf_saved_model.initializer_type" attribute from the callee of the + // inner calls if the problem ever arises. + entry_func->emitError() + << "TF2XLA MLIR Non-replicated Phase 1 Bridge expects no nested calls" + " of entry functions as they prevent graph traversal in some " + "passes from " + "working correctly"; + return failure(); + } + } + return success(); +} + +LogicalResult HasTopLevelCompilationMarker( + llvm::SmallVector &entry_funcs) { + for (auto &entry_func : entry_funcs) { + if (entry_func->hasAttr(mlir::TF::kCompileDeviceTypeAttr)) { + entry_func->emitError() << "TF2XLA MLIR Non-replicated Phase 1 Bridge " + "does not support top-level compilation " + "marker."; + return failure(); + } + } + return success(); +} + +void XlaValidateInputsPass::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTable symtab(module); + llvm::SmallVector entry_funcs = GetEntryFunctions(module); + if (entry_funcs.empty()) { + LOG(WARNING) << "missing entry functions"; + } + + if (HasNoNestedEntryFunctions(entry_funcs, symtab).failed()) { + return signalPassFailure(); + } + + if (HasTopLevelCompilationMarker(entry_funcs).failed()) { + return signalPassFailure(); + } +} + +} // namespace + +namespace TFDevice { +std::unique_ptr> CreateXlaValidateInputsPass() { + return std::make_unique(); +} +} // namespace TFDevice + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..8e34ac02513d5a926453f8ba5062cc1d661637ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/BUILD @@ -0,0 +1,101 @@ +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +cc_library( + name = "import_model", + srcs = [ + "import_model.cc", + ], + hdrs = [ + "import_model.h", + ], + deps = [ + ":mlir_roundtrip_flags", + "//tensorflow/cc/saved_model:bundle_v2", + "//tensorflow/cc/saved_model:constants", + "//tensorflow/cc/saved_model:loader_lite", + "//tensorflow/cc/saved_model:loader_util", + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mlir_import_options", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:initialize_variables_in_session_init", + "//tensorflow/compiler/mlir/tensorflow/transforms:lift_variables_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:mark_initialized_variables_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_saved_model_passes", + "//tensorflow/compiler/mlir/tf2xla/api/v2:graph_to_tf_executor", + "//tensorflow/compiler/tf2xla:functionalize_control_flow", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/common_runtime:function_body", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla:status_macros", + ], +) + +cc_library( + name = "mlir_roundtrip_flags", + hdrs = ["mlir_roundtrip_flags.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/mlir/tf2xla/api/v2:mlir_roundtrip_flags", + ], +) + +cc_library( + name = "mlir_import_options", + hdrs = ["mlir_import_options.h"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "translate_lib", + srcs = ["tf_mlir_translate.cc"], + hdrs = ["tf_mlir_translate.h"], + visibility = ["//visibility:public"], + deps = [ + ":import_model", + "//tensorflow/cc/saved_model:bundle_v2", + "//tensorflow/cc/saved_model:loader_lite", + "//tensorflow/cc/saved_model:reader", + "//tensorflow/compiler/mlir/tensorflow:mlir_import_options", + "//tensorflow/core:core_cpu", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:ops", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@llvm-project//mlir:IR", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc new file mode 100644 index 0000000000000000000000000000000000000000..c84ed57b0be19959bb5782bcc073914c543a7422 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc @@ -0,0 +1,1724 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/translate/import_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/span.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/cc/saved_model/bundle_v2.h" +#include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/loader_util.h" +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/initialize_variables_in_session_init.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/lift_variables.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/mark_initialized_variables.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" +#include "tensorflow/compiler/tf2xla/functionalize_control_flow.h" +#include "xla/status_macros.h" +#include "tensorflow/core/common_runtime/function_body.h" +#include "tensorflow/core/common_runtime/function_def_utils.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/common_runtime/graph_runner.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_def_builder.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/graph_node_util.h" +#include "tensorflow/core/graph/tensor_id.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/fingerprint.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/path.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/threadpool.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow/core/protobuf/saved_object_graph.pb.h" +#include "tensorflow/core/protobuf/saver.pb.h" +#include "tensorflow/core/protobuf/struct.pb.h" +#include "tensorflow/core/protobuf/trackable_object_graph.pb.h" +#include "tsl/platform/errors.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { + +constexpr size_t kNumThreadToConvertSignatures = 10; + +using ::mlir::tf_saved_model::AssetOp; +using ::mlir::tf_saved_model::GlobalTensorOp; +using ::mlir::tf_saved_model::kTfSavedModelExportedNamesAttr; +using ::mlir::tf_saved_model::kTfSavedModelIndexPathAttr; +using ::mlir::tf_saved_model::kTfSavedModelInitializerInitType; +using ::mlir::tf_saved_model::kTfSavedModelInitializerRestoreType; +using ::mlir::tf_saved_model::kTfSavedModelInitializerTypeAttr; +using ::mlir::tf_saved_model::SessionInitializerOp; +using ::tsl::StatusOr; + +namespace { + +void LoadImporterDialects(mlir::MLIRContext& context) { + // Load dialects involved in the conversion + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialectsImpl(registry, false); + context.appendDialectRegistry(registry); + for (llvm::StringRef name : registry.getRegisteredDialectNames()) { + context.getOrLoadDialect(name); + } +} + +absl::StatusOr GetDenseTensorNameFromTensorInfo( + const TensorInfo& tensor_info) { + // TODO(b/184675681): Support other encoding cases. + // + // TODO(b/184679394): Add unit test for this check. + TF_RET_CHECK(tensor_info.encoding_case() == tensorflow::TensorInfo::kName) + << "Only dense tensor is supported, but got encoding case " + << tensor_info.encoding_case(); + return tensor_info.name(); +} + +// This class is used to generate new MLIR function name strings that are both +// unique in the TF function library `flib_` and unique among the name strings +// generated by the class object during its lifetime. +// +// In theory, this class is not necessary because we should simply take +// the TF function name and use it as MLIR function name. However, for some +// unknown reasons (callout for investigation in b/142268695), keeping the +// function names unchanged in an MLIR roundtrip causes test failures. +// TODO(b/142268695) Re-evaluate whether we need this class v.s. directly using +// and TF function name as MLIR function name after b/142268695 is root caused. +class NameUniquifier : public OpOrArgNameMapper { + public: + explicit NameUniquifier(const FunctionLibraryDefinition& flib) + : flib_(flib) {} + + private: + bool IsUnique(llvm::StringRef name) override { + return !flib_.Contains(std::string(name)); + } + + std::string GetName(OpOrVal op_or_val) override { + DCHECK(false) << "Unimplemented"; + return ""; + } + + const FunctionLibraryDefinition& flib_; +}; + +// Determines the names used to reference objects in the SavedObjectGraph. +class ObjectNames { + public: + explicit ObjectNames(const SavedObjectGraph& object_graph, + absl::Span exported_names); + + // Gets the names that external users of the SavedModel can use to refer to + // this node. + llvm::ArrayRef GetExportedNames(int node_id) const; + + // Gets the name in the module symbol table for this node. + // This name is only used for internal IR references. + llvm::StringRef GetSymbolTableName(int node_id) const; + + private: + // In the absence of any other information, use this name as the symbol table + // name for this node. + std::string GetDefaultSymbolTableName(int node_id) const; + // Determines if a name is exported. + bool IsExported(const std::string& name); + // Main object graph traversal function. + void RecursivelyVisitObjectGraph(int node_id); + // Gets a stable StringRef from a std::string. + llvm::StringRef SaveString(const std::string& s) const; + + // The object graph we are traversing. + const SavedObjectGraph& object_graph_; + // The set of names to export. Empty means "export all". + std::unordered_set names_to_export_; + + // When we recursively follow the object graph tree structure from the root, + // we track its path in the object graph by pushing and popping from here + // during traversal. + llvm::SmallVector path_segments_; + // The set of node IDs that are on the current DFS stack. + // For cyclic object graphs, this prevents infinite recursion. + absl::flat_hash_set on_stack_nodes_; + + // Key: node_id. + // Value: all object names that node_id appears as. + // Each object name corresponds to a unique path from the root of the object + // graph. + // The common intuitive case is when there is only one name for a given + // object, which corresponds to the object graph being a tree. + // + // But, there cases where the object graph is a general graph. For + // example, this happens commonly in Keras models, where `foo.bar` is + // also reachable via the name `keras_api.foo.bar`. + // Cycles are possible too. + absl::flat_hash_map> object_names_; + + // Key: node_id + // Value: all names that this object is exported as + absl::flat_hash_map> + exported_names_; + // Key: node_id + // Value: pretty symbol table name to use for internal references to this + // object. + absl::flat_hash_map pretty_symbol_table_name_; + + // Stable strings we can take StringRef's into. Used only by the SaveString + // method. + mutable std::unordered_set saved_strings_; +}; + +ObjectNames::ObjectNames(const SavedObjectGraph& object_graph, + absl::Span exported_names) + : object_graph_(object_graph), + names_to_export_(exported_names.begin(), exported_names.end()) { + // Visit all reachable nodes from the root of the object graph. + // This builds up object_names_ to contain all names like `foo.bar` that a + // particular node in the graph can be reached from. + RecursivelyVisitObjectGraph(/*node_id=*/0); + + // Populate the exported_names_ map. + // TODO(silvasean): Diagnose typos in exported names? + for (auto& kv : object_names_) { + // Make object names map independent of our particular choice of object + // graph traversal. + std::sort( + kv.second.begin(), kv.second.end(), + [](absl::string_view a, absl::string_view b) { + // The sort order here influences the "pretty name" we assign + // below. We want the most debuggable name to be first. + // + // Debuggability heuristics: + // 1. Names that end in digits are likely to be internal aliases + // to the "real" names. + // 2. Longer names are more likely to be internal aliases. + // + // Example set of object names created by Keras for the weight + // matrix of a fully connected layer on a trivial FC mnist + // model: + // - `model.layer-1.kernel` (this is the "best" name) + // - `model.keras_api.layers.1.kernel` + // - `model.variables.0` + // - `model.keras_api.layers.1.keras_api.trainable_variables.0` + // - ... 10 more long aliases ending in digits ... + return std::make_tuple(absl::ascii_isdigit(a.back()), a.size(), a) < + std::make_tuple(absl::ascii_isdigit(b.back()), b.size(), b); + }); + for (const std::string& name : kv.second) { + if (IsExported(name)) { + exported_names_[kv.first].push_back(SaveString(name)); + } + } + } + // Create "pretty" symbol table names for nodes where that is applicable. + // We could make all symbol table names use the default, which is basically + // just the node id. But for debugging purposes, it's nicer if we can mix in + // a recognizable object name if we have the information to do so. + for (auto& kv : object_names_) { + int node_id = kv.first; + std::string internal_name = + absl::StrCat(GetDefaultSymbolTableName(node_id), "__"); + // If the object has an exported name, we prefer that since it is probably + // the most recognizable. Otherwise, we grab some non-exported name of the + // object. + if (exported_names_.find(node_id) != exported_names_.end()) { + internal_name += exported_names_[node_id][0].str(); + } else { + internal_name += object_names_[node_id][0]; + } + pretty_symbol_table_name_[node_id] = SaveString(internal_name); + } +} + +llvm::ArrayRef ObjectNames::GetExportedNames( + int node_id) const { + auto it = exported_names_.find(node_id); + if (it != exported_names_.end()) { + return it->second; + } + return {}; +} + +llvm::StringRef ObjectNames::GetSymbolTableName(int node_id) const { + auto it = pretty_symbol_table_name_.find(node_id); + if (it != pretty_symbol_table_name_.end()) { + return it->second; + } + return SaveString(GetDefaultSymbolTableName(node_id)); +} + +std::string ObjectNames::GetDefaultSymbolTableName(int node_id) const { + return absl::StrCat("__sm_node", node_id); +} + +bool ObjectNames::IsExported(const std::string& name) { + if (names_to_export_.empty()) { + return true; + } + return names_to_export_.find(name) != names_to_export_.end(); +} + +void ObjectNames::RecursivelyVisitObjectGraph(int node_id) { + const SavedObject& object = object_graph_.nodes(node_id); + + switch (object.kind_case()) { + case SavedObject::kConstant: + case SavedObject::kFunction: + case SavedObject::kVariable: { + object_names_[node_id].push_back(absl::StrJoin(path_segments_, ".")); + break; + } + default: + break; + } + + for (const auto& child_ref : object.children()) { + bool on_stack = !on_stack_nodes_.insert(child_ref.node_id()).second; + if (on_stack) { + // This is a backedge. Don't traverse it. + continue; + } + + path_segments_.push_back(child_ref.local_name()); + RecursivelyVisitObjectGraph(child_ref.node_id()); + path_segments_.pop_back(); + + on_stack_nodes_.erase(child_ref.node_id()); + } +} + +llvm::StringRef ObjectNames::SaveString(const std::string& s) const { + return llvm::StringRef(*saved_strings_.insert(s).first); +} + +// Extracts a TensorProto for a Const op from a GraphDef, given an op_name. +// Returns nullptr on not found or other mismatch. +// This returns a pointer to the actual node within the graph_def so as to +// avoid expensive copies. +const TensorProto* ExtractConstTensorFromGraph(const GraphDef& graph_def, + const std::string& op_name) { + const NodeDef* match_node = nullptr; + for (const auto& node : graph_def.node()) { + if (node.name() == op_name) { + match_node = &node; + } + } + + if (!match_node) { + return nullptr; + } + + auto value_it = match_node->attr().find("value"); + if (value_it == match_node->attr().end()) { + return nullptr; + } + + if (!value_it->second.has_tensor()) { + return nullptr; + } + + return &value_it->second.tensor(); +} + +const TrackableObjectGraph::TrackableObject::SerializedTensor* +FindSerializedTensorInTrackable( + const TrackableObjectGraph::TrackableObject& trackable_object, + absl::string_view name) { + for (const auto& maybe_serialized_tensor : trackable_object.attributes()) { + if (maybe_serialized_tensor.name() == name) { + return &maybe_serialized_tensor; + } + } + return nullptr; +} + +absl::Status DiagnoseMultipleConcreteFunctions( + const SavedObjectGraph& object_graph, const ObjectNames& object_names) { + for (int node_id = 0; node_id < object_graph.nodes_size(); node_id++) { + const SavedObject& object = object_graph.nodes(node_id); + if (object_names.GetExportedNames(node_id).empty()) { + continue; + } + if (object.kind_case() == SavedObject::kFunction) { + // We only allow a single input signature to each SavedFunction. + // This assumption means we have a 1:1 correspondence between + // tf.function <=> SavedFunction <=> SavedConcreteFunction <=> FunctionDef + // This makes defining the ABI easier (or even well-defined at all). + // TODO(silvasean): How to detect a function that doesn't have an + // explicitly user-provided input signature, but happens to have been + // traced exactly once? + if (object.function().concrete_functions_size() != 1) { + llvm::SmallVector names; + for (llvm::StringRef s : object_names.GetExportedNames(node_id)) { + names.push_back("'" + s.str() + "'"); + } + return absl::InvalidArgumentError(absl::StrCat( + "Exported function with exported name(s) ", + absl::StrJoin(names, ", "), + " with multiple concrete functions. Add " + "@tf.function(input_signature=[...]) on this function, or use a " + "narrower list of exported names that excludes this function.")); + } + } + } + return absl::OkStatus(); +} + +// Recursively traverses a StructuredValue, linearizing all the leaves. +// +// This currently only handles the subset of StructuredValue that is needed for +// signatures. +// +// Given a StructuredValue with structure [{"x": leaf0}], the "index path" +// needed to reach leaf0 is `[0, "x"]`, as it would be if you were operating on +// a Python object (`obj[0]["x"] is leaf0`). Each leaf corresponds to a +// linearized function argument or return on a FunctionDef, and hence to an +// mlir::func::FuncOp argument / return. +// +// This must match the linearization that happens in `tf.nest.flatten`. +// In particular, dict values should be linearized in sorted key order. +// +// The linearized index paths can be returned back to a structured +// representation (e.g. to emit C structs matching a signature) with a simple +// algorithm that recurses on each run of index paths with identical first +// elements. +class StructuredValueLinearizer { + public: + StructuredValueLinearizer(const StructuredValue& value, + mlir::MLIRContext* context); + + // Returns the list of index paths to each leaf of the StructuredValue, + // in a linearized order matching `tf.nest.flatten`. + // + // If an error occurred during the linearization process, an error message + // with `error_context` prepended will be included in the returned status. + absl::StatusOr> GetLeafIndexPaths( + llvm::StringRef error_context) const; + + private: + // Main function that recursively traverses the StructuredValue. + void RecursivelyFindLeaves(const StructuredValue& value); + + mlir::Builder builder_; + // The current index path. We push/pop this during recursive traversal of the + // StructuredValue. + llvm::SmallVector current_index_path_; + // The list of leaf index paths we have discovered so far. + llvm::SmallVector leaf_index_paths_; + // If non-empty, an error message to report. + std::string error_message_; +}; + +StructuredValueLinearizer::StructuredValueLinearizer( + const StructuredValue& value, mlir::MLIRContext* context) + : builder_(context) { + RecursivelyFindLeaves(value); +} + +absl::StatusOr> +StructuredValueLinearizer::GetLeafIndexPaths( + llvm::StringRef error_context) const { + if (error_message_.empty()) { + return llvm::ArrayRef(leaf_index_paths_); + } + return absl::InvalidArgumentError(absl::StrCat( + error_context.str(), error_message_, + "This likely means that you have @tf.function " + "on an exported function instead of " + "@tf.function(input_signature=[...]). Consider annotating an " + "input_signature or narrowing your set of " + "exported names to not include this function.")); +} + +void StructuredValueLinearizer::RecursivelyFindLeaves( + const StructuredValue& value) { + switch (value.kind_case()) { + case StructuredValue::kDictValue: { + // Dict values must be linearized in sorted order of keys. + const DictValue& dict = value.dict_value(); + using FieldTy = protobuf::MapPair; + llvm::SmallVector fields; + for (auto& field : dict.fields()) { + fields.push_back(&field); + } + llvm::sort(fields, [](const FieldTy* a, const FieldTy* b) { + return a->first < b->first; + }); + for (auto& field : fields) { + current_index_path_.push_back(builder_.getStringAttr(field->first)); + RecursivelyFindLeaves(field->second); + current_index_path_.pop_back(); + } + return; + } + case StructuredValue::kTupleValue: { + const TupleValue& tuple = value.tuple_value(); + for (int i = 0, e = tuple.values_size(); i < e; i++) { + current_index_path_.push_back(builder_.getI64IntegerAttr(i)); + RecursivelyFindLeaves(tuple.values(i)); + current_index_path_.pop_back(); + } + return; + } + // We don't differentiate between tuples and lists. + case StructuredValue::kListValue: { + const ListValue& list = value.list_value(); + for (int i = 0, e = list.values_size(); i < e; i++) { + current_index_path_.push_back(builder_.getI64IntegerAttr(i)); + RecursivelyFindLeaves(list.values(i)); + current_index_path_.pop_back(); + } + return; + } + case StructuredValue::kTensorSpecValue: { + // Base case: record the current path stack as the index path needed to + // get to this leaf. + leaf_index_paths_.push_back(builder_.getArrayAttr(current_index_path_)); + return; + } + case StructuredValue::kNoneValue: { + // Base case: do nothing. + // This arises, for example, as the top-level object of an output + // signature when there are no return values. + return; + } + default: { + llvm::raw_string_ostream os(error_message_); + // TODO(silvasean): Use an enumerant name string instead of a number. + os << "Unhandled structured value kind " << value.kind_case() + << " at index path: "; + for (auto path_element : current_index_path_) { + os << "."; + if (auto integer = mlir::dyn_cast(path_element)) { + os << integer.getValue(); + } else { + auto str = mlir::cast(path_element); + os << str.getValue(); + } + } + os << "\n"; + } + } +} + +// For exported functions with bound inputs, rewrite the function +// signature to match the requirements of tf_saved_model bound input args. +// +// The raw imported functions have `tensor<*x!tf_type.resource>` as the type for +// mutable bound inputs and `tensor<...>` as the type for immutable +// bound inputs. Here we canonicalize both of them into +// `tensor>>`. +void AdjustBoundInputArgTypes(mlir::ModuleOp module) { + mlir::SymbolTable symbol_table(module); + for (auto func : module.getOps()) { + if (!mlir::tf_saved_model::IsExported(func)) continue; + mlir::OpBuilder builder(func.getBody()); + llvm::SmallVector new_input_types; + for (int i = 0, e = func.getNumArguments(); i < e; i++) { + auto arg = func.getArgument(i); + auto global_tensor = mlir::tf_saved_model::LookupBoundInputOfType< + mlir::tf_saved_model::GlobalTensorOp>(func, i, symbol_table); + if (global_tensor) { + auto old_type = arg.getType(); + auto new_type = + mlir::tf_saved_model::GetBoundInputArgTypeFor(global_tensor); + arg.setType(new_type); + if (global_tensor.getIsMutable()) { + auto arg_with_original_type = mlir::TF::CastOp::create( + builder, global_tensor.getLoc(), old_type, arg, + /*Truncate=*/builder.getBoolAttr(false)); + arg.replaceAllUsesWith(arg_with_original_type); + // The RAUW replaces the arg with itself, so we need to set it back. + arg_with_original_type.setOperand(arg); + } else { + auto arg_with_original_type = mlir::TF::ReadVariableOp::create( + builder, global_tensor.getLoc(), old_type, arg); + arg.replaceAllUsesWith(arg_with_original_type); + // The RAUW replaces the arg with itself, so we need to set it back. + arg_with_original_type.setOperand(arg); + } + } + new_input_types.push_back(arg.getType()); + } + func.setType(mlir::FunctionType::get(module.getContext(), new_input_types, + func.getFunctionType().getResults())); + } +} + +// Marks the visibility of functions in the saved model module. +void MarkSavedModelFunctionVisibility(mlir::ModuleOp module) { + for (auto func : module.getOps()) { + auto visibility = mlir::tf_saved_model::IsExported(func) + ? mlir::func::FuncOp::Visibility::Public + : mlir::func::FuncOp::Visibility::Private; + func.setVisibility(visibility); + } +} + +// Reorder the ops in the module to make testing easier and less dependent +// on implementation details such as the order of functions in the +// FunctionDefLibrary. +// +// The order this ensures is: +// 1. GlobalTensorOp's +// 2. FuncOps's. +// +// Within each of 1. and 2., ops are sorted by exported name (if +// available, and only the first exported name is considered), followed by +// non-exported ops. +void SortSavedModelModule(mlir::ModuleOp module) { + struct NamedGlobalTensor { + llvm::StringRef name; + GlobalTensorOp global_tensor; + }; + llvm::SmallVector named_global_tensors; + for (auto global_tensor : module.getOps()) { + auto exported_names = mlir::tf_saved_model::GetExportedNames(global_tensor); + // We use stable_sort, so duplicate empty names are fine here. + named_global_tensors.push_back( + {exported_names.empty() ? "" : exported_names.front(), global_tensor}); + } + llvm::stable_sort(named_global_tensors, + [](const NamedGlobalTensor& a, const NamedGlobalTensor& b) { + return std::make_tuple(a.name.empty(), a.name) < + std::make_tuple(b.name.empty(), b.name); + }); + + struct NamedFunc { + llvm::StringRef name; + mlir::func::FuncOp func; + }; + llvm::SmallVector named_funcs; + llvm::SmallVector private_funcs; + for (auto func : module.getOps()) { + auto exported_names = mlir::tf_saved_model::GetExportedNames(func); + if (!exported_names.empty()) + named_funcs.push_back({exported_names.front(), func}); + else + private_funcs.push_back(func); + } + llvm::stable_sort(named_funcs, [](const NamedFunc& a, const NamedFunc& b) { + return a.name < b.name; + }); + llvm::stable_sort(private_funcs, + [](mlir::func::FuncOp a, mlir::func::FuncOp b) { + return a.getName() < b.getName(); + }); + + struct NamedAsset { + llvm::StringRef name; + AssetOp asset; + }; + llvm::SmallVector assets; + for (auto asset : module.getOps()) { + assets.push_back({asset.getName(), asset}); + } + llvm::stable_sort(assets, [](const NamedAsset& a, const NamedAsset& b) { + return a.name < b.name; + }); + + // Move onto the front of the module in reverse of the final desired order. + for (auto func : llvm::reverse(private_funcs)) { + func.getOperation()->moveBefore(&module.getBody()->front()); + } + for (auto named_func : llvm::reverse(named_funcs)) { + named_func.func.getOperation()->moveBefore(&module.getBody()->front()); + } + for (auto named_global_tensor : llvm::reverse(named_global_tensors)) { + named_global_tensor.global_tensor.getOperation()->moveBefore( + &module.getBody()->front()); + } + + for (auto asset : assets) { + asset.asset.getOperation()->moveBefore(&module.getBody()->front()); + } + + auto initializers = module.getOps(); + if (!initializers.empty()) { + (*initializers.begin()) + .getOperation() + ->moveBefore(&module.getBody()->front()); + } +} + +absl::Status CreateSavedModelIR( + const ObjectNames& object_names, mlir::ModuleOp module, + const SavedObjectGraph& object_graph, + const std::unordered_map& tf_name_to_mlir_name, + SavedModelV2Bundle* saved_model, MLIRImportOptions import_options) { + mlir::OpBuilder builder(module.getBodyRegion()); + mlir::SymbolTable symbol_table(module); + + // Create a side data-structure, indexed by the object_graph node_id to + // a TrackableObject that is restorable. + absl::flat_hash_map + restored_objects; + TF_RETURN_IF_ERROR(saved_model->VisitObjectsToRestore( + [&](int saved_node_id, + const TrackableObjectGraph::TrackableObject& trackable_object) { + restored_objects.insert( + std::make_pair(saved_node_id, &trackable_object)); + return absl::OkStatus(); + })); + + for (int node_id = 0; node_id < object_graph.nodes_size(); node_id++) { + const SavedObject& object = object_graph.nodes(node_id); + // For correctness, we cannot import functions that don't have exported + // names, since they don't necessarily have a well-defined ABI (diagnosed + // earlier). + // + // For variables/constants, pruning them is purely an optimization, + // and more complicated since it requires use-def analysis of which + // functions use which variables/constants, so we don't do anything + // special for them here as part of our initial IR construction. + if (object.kind_case() == SavedObject::kFunction) { + if (object_names.GetExportedNames(node_id).empty()) { + continue; + } + std::string error_context = + "While importing SavedModel function '" + + object_names.GetExportedNames(node_id)[0].str() + "': "; + const SavedFunction& function = object.function(); + auto orig_func = symbol_table.lookup( + tf_name_to_mlir_name.find(function.concrete_functions(0))->second); + mlir::func::FuncOp func = orig_func; + // If there are potentially references to this func from within the + // module, create a wrapper around it and decorate the wrapper with the + // tf_saved_model attributes instead. + if (!mlir::SymbolTable::symbolKnownUseEmpty(orig_func.getSymNameAttr(), + &module.getBodyRegion())) { + func = orig_func.cloneWithoutRegions(); + module.insert(module.getBody()->begin(), func); + func.addEntryBlock(); + func.setName(builder.getStringAttr("__sm_exported_" + + orig_func.getName().str())); + llvm::SmallVector args_as_values; + for (auto block_argument : func.getArguments()) { + args_as_values.push_back(block_argument); + } + mlir::OpBuilder body_builder(&func.getBody()); + auto call = mlir::TF::StatefulPartitionedCallOp::create( + body_builder, func.getLoc(), + orig_func.getFunctionType().getResults(), args_as_values, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, + mlir::SymbolRefAttr::get(builder.getContext(), orig_func.getName()), + /*config=*/builder.getStringAttr(""), + /*config_proto=*/builder.getStringAttr(""), + /*executor_type=*/builder.getStringAttr("")); + mlir::func::ReturnOp::create(body_builder, func.getLoc(), + call.getResults()); + } + func->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr(object_names.GetExportedNames(node_id))); + const SavedConcreteFunction& concrete_function = + object_graph.concrete_functions().at(function.concrete_functions(0)); + + // We do not handle the other element of this tuple, which corresponds to + // Python kwonlyargs, since currently TensorFlow prohibits this in + // combination with input_signature: + // https://github.com/tensorflow/tensorflow/blob/8cb8627abb5ef83a6fba34f8fd0e4ee430562eb1/tensorflow/python/eager/function.py#L2027-L2030 + // Our SavedModel import requires input_signature on the tf.function, so + // we never need to handle the kwonlyargs. + auto positional_arg_structure = + concrete_function.canonicalized_input_signature() + .tuple_value() + .values(0); + StructuredValueLinearizer input_linearizer(positional_arg_structure, + builder.getContext()); + + int bound_input_base = + func.getNumArguments() - concrete_function.bound_inputs_size(); + TF_ASSIGN_OR_RETURN(auto input_index_paths, + input_linearizer.GetLeafIndexPaths( + error_context + "in input signature: ")); + const int input_index_paths_size = input_index_paths.size(); + if (bound_input_base != input_index_paths_size) { + return absl::InvalidArgumentError(absl::StrCat( + error_context, + "Argument mismatch between concrete function input signature " + "vs underlying FunctionDef for concrete function '", + function.concrete_functions(0), "' (", input_index_paths.size(), + " vs ", bound_input_base, ")")); + } + for (const auto& index_path : llvm::enumerate(input_index_paths)) { + func.setArgAttr(index_path.index(), kTfSavedModelIndexPathAttr, + index_path.value()); + } + + for (const auto& bound_input : + llvm::enumerate(concrete_function.bound_inputs())) { + int arg_index = bound_input_base + bound_input.index(); + auto symbol_ref = mlir::SymbolRefAttr::get( + builder.getContext(), + object_names.GetSymbolTableName(bound_input.value())); + func.setArgAttr(arg_index, "tf_saved_model.bound_input", symbol_ref); + } + + StructuredValueLinearizer output_linearizer( + concrete_function.output_signature(), builder.getContext()); + TF_ASSIGN_OR_RETURN(auto output_index_paths, + output_linearizer.GetLeafIndexPaths( + error_context + "in output signature: ")); + if (func.getNumResults() != output_index_paths.size()) { + return absl::InvalidArgumentError(absl::StrCat( + error_context, + "Result mismatch between concrete function output signature " + "vs underlying FunctionDef for concrete function '", + function.concrete_functions(0), "' (", output_index_paths.size(), + " vs ", func.getNumResults(), ")")); + } + for (const auto& index_path : llvm::enumerate(output_index_paths)) { + func.setResultAttr(index_path.index(), kTfSavedModelIndexPathAttr, + index_path.value()); + } + } else if (object.kind_case() == SavedObject::kVariable) { + const SavedVariable& variable = object.variable(); + // Find the trackable in the side data structure. + auto variable_trackable_it = restored_objects.find(node_id); + + TF_ASSIGN_OR_RETURN( + auto type, ConvertToMlirTensorType(variable.shape(), variable.dtype(), + &builder)); + + if (variable_trackable_it == restored_objects.end()) { + if (!import_options.allow_uninitialized_variables) { + return absl::FailedPreconditionError(absl::StrCat( + "Could not restore saved variable: ", variable.name())); + } + + // The user indicated we should allow loading the model with + // uninitialized variables, use the type information to construct a + // dummy uninitialized variable operation. + auto op = mlir::tf_saved_model::GlobalTensorOp::create( + builder, builder.getUnknownLoc(), + builder.getStringAttr(object_names.GetSymbolTableName(node_id)), + mlir::ElementsAttr(), + /*type=*/mlir::TypeAttr::get(type), + /*is_mutable=*/builder.getUnitAttr()); + op->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr(object_names.GetExportedNames(node_id))); + } else { + const auto* serialized_tensor_attr = FindSerializedTensorInTrackable( + *variable_trackable_it->second, "VARIABLE_VALUE"); + if (!serialized_tensor_attr) { + return absl::FailedPreconditionError(absl::StrCat( + "Could not find serialized tensor for saved variable: ", + variable.name())); + } + const auto& checkpoint_key = serialized_tensor_attr->checkpoint_key(); + + // Load it from the reader. + Tensor value; + TF_RETURN_WITH_CONTEXT_IF_ERROR( + saved_model->variable_reader()->Lookup(checkpoint_key, &value), + "Could not read checkpoint key from variables bundle: ", + checkpoint_key); + TF_ASSIGN_OR_RETURN( + auto value_attr, + ConvertTensor(value, &builder, + /*convert_to_dense_resource=*/ + import_options.import_variables_as_dense_resources)); + // A variable can have a partially known type, such as + // tensor, even if the initializer is a specific static + // shape. + auto op = GlobalTensorOp::create( + builder, builder.getUnknownLoc(), + builder.getStringAttr(object_names.GetSymbolTableName(node_id)), + value_attr, + /*type=*/mlir::TypeAttr::get(type), + /*is_mutable=*/builder.getUnitAttr()); + op->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr(object_names.GetExportedNames(node_id))); + } + + } else if (object.kind_case() == SavedObject::kConstant) { + const SavedConstant& constant = object.constant(); + const TensorProto* value = ExtractConstTensorFromGraph( + saved_model->meta_graph_def().graph_def(), constant.operation()); + if (!value) { + return absl::FailedPreconditionError(absl::StrCat( + "Unable to find const node referenced in object graph: ", + constant.operation())); + } + TF_ASSIGN_OR_RETURN(auto value_attr, + ConvertTensorProto(*value, &builder)); + auto op = GlobalTensorOp::create( + builder, builder.getUnknownLoc(), + builder.getStringAttr(object_names.GetSymbolTableName(node_id)), + value_attr, + /*type=*/mlir::TypeAttr::get(value_attr.getType()), + /*is_mutable=*/nullptr); + op->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr(object_names.GetExportedNames(node_id))); + } + } + AdjustBoundInputArgTypes(module); + module->setAttr("tf_saved_model.semantics", builder.getUnitAttr()); + SortSavedModelModule(module); + MarkSavedModelFunctionVisibility(module); + return absl::OkStatus(); +} + +absl::StatusOr> ConvertSavedModelObjectGraph( + SavedModelV2Bundle* saved_model, absl::Span exported_names, + mlir::MLIRContext* context, MLIRImportOptions import_options) { + LoadImporterDialects(*context); + GraphDebugInfo dummy_debug_info; + const GraphDebugInfo& debug_info = + saved_model->debug_info() ? *saved_model->debug_info() : dummy_debug_info; + + GraphImportConfig specs; + specs.prune_unused_nodes = true; + specs.unconditionally_use_set_output_shapes = + import_options.unconditionally_use_set_output_shapes; + mlir::OwningOpRef module = + mlir::ModuleOp::create(mlir::UnknownLoc::get(context)); + std::unordered_map tf_name_to_mlir_name; + + const auto& graphdef = saved_model->meta_graph_def().graph_def(); + PopulateTfVersions(module.get(), graphdef.versions()); + + GraphConstructorOptions options; + options.allow_internal_ops = true; + options.add_default_attributes = import_options.add_default_attributes; + options.upgrade_legacy = import_options.upgrade_legacy; + Graph graph(OpRegistry::Global()); + + TF_RETURN_IF_ERROR(ConvertGraphDefToGraph(options, graphdef, &graph)); + + NameUniquifier function_name_uniquifier(graph.flib_def()); + for (const auto& fn_name : graph.flib_def().ListFunctionNames()) { + std::string mlir_func_name(function_name_uniquifier.GetUniqueName(fn_name)); + (tf_name_to_mlir_name)[std::string(fn_name)] = mlir_func_name; + } + + specs.convert_all_functions_to_mlir = true; + TF_ASSIGN_OR_RETURN(module, tensorflow::tf2xla::v2::ConvertGraphToTfExecutor( + graph, debug_info, graph.flib_def(), specs, + module->getContext())); + + if (!saved_model->meta_graph_def().has_object_graph_def()) { + return absl::InvalidArgumentError( + "SavedModel does not have an object graph. Please use TF2."); + } + auto& object_graph = saved_model->meta_graph_def().object_graph_def(); + ObjectNames object_names(object_graph, exported_names); + + // Clean up a couple func's that always seem to be present when importing a + // SavedModel. This is not strictly needed, as there is a separate pass that + // will clean them up, but this makes staring at the raw IR of minimal + // examples quite a bit nicer. + for (auto func : + llvm::make_early_inc_range(module->getOps())) { + if (func.getName().starts_with("__inference__traced_save_") || + func.getName().starts_with("__inference__traced_restore_") || + func.getName().starts_with("__inference_signature_wrapper_") || + func.getName().starts_with("main")) { + func.erase(); + } + } + + // Diagnose SavedFunction's with multiple input signatures. + TF_RETURN_IF_ERROR( + DiagnoseMultipleConcreteFunctions(object_graph, object_names)); + + // Construct the SavedModel IR. + TF_RETURN_IF_ERROR(CreateSavedModelIR(object_names, module.get(), + object_graph, tf_name_to_mlir_name, + saved_model, import_options)); + assert(mlir::succeeded(mlir::verify(module.get()))); + + return module; +} + +class SimpleSavedModelMLIRImportInput : public SavedModelMLIRImportInput { + public: + static absl::StatusOr Create( + const MLIRImportOptions& import_options, + const MetaGraphDef* meta_graph_def, const GraphDebugInfo& debug_info) { + DCHECK(meta_graph_def); + GraphDef graph_def(meta_graph_def->graph_def()); + auto graph = std::make_unique(OpRegistry::Global()); + + GraphConstructorOptions graph_ctor_options; + graph_ctor_options.allow_internal_ops = true; + graph_ctor_options.add_default_attributes = true; + graph_ctor_options.upgrade_legacy = import_options.upgrade_legacy; + TF_RETURN_IF_ERROR(ConvertGraphDefToGraph( + graph_ctor_options, std::move(graph_def), graph.get())); + + if (import_options.upgrade_legacy) { + // TODO(jpienaar): Remove need to const_cast. + TF_RETURN_WITH_CONTEXT_IF_ERROR( + FunctionalizeControlFlow( + graph.get(), + const_cast(&graph->flib_def()), + NodeFilter{}, + /*include_functions=*/true), + tensorflow::kFunctionalizeControlFlowFailureMessage); + } + + return SimpleSavedModelMLIRImportInput(meta_graph_def, debug_info, + std::move(graph)); + } + + SimpleSavedModelMLIRImportInput(const MetaGraphDef* meta_graph_def, + const GraphDebugInfo& debug_info, + std::unique_ptr graph) + : SavedModelMLIRImportInput(meta_graph_def, debug_info), + graph_(std::move(graph)) {} + + absl::StatusOr GetSubGraph(absl::string_view name, + GraphImportConfig& specs) override { + DCHECK(CheckGraphNameValidity(name)); + DCHECK(CheckGraphContainsFeedsAndFetches(specs)); + return graph_.get(); + } + + private: + bool CheckGraphContainsFeedsAndFetches(const GraphImportConfig& specs) const { + absl::flat_hash_set feed_fetch_nodes; + for (const auto& iter : specs.inputs) { + TensorId tensor_id = ParseTensorName(iter.first); + feed_fetch_nodes.insert(std::string(tensor_id.node())); + } + for (const auto& output : llvm::concat( + specs.outputs, specs.control_outputs)) { + TensorId tensor_id = ParseTensorName(output); + feed_fetch_nodes.insert(std::string(tensor_id.node())); + } + + for (Node* node : graph_->op_nodes()) { + feed_fetch_nodes.erase(node->name()); + } + + return feed_fetch_nodes.empty(); + } + + bool CheckGraphNameValidity(absl::string_view name) const { + // If it is one of the signature name, it is valid. + const auto& signature_defs = meta_graph_def().signature_def(); + if (signature_defs.contains(std::string(name))) return true; + + // If it is the restore graph name, it is valid. + if (meta_graph_def().has_saver_def() && + meta_graph_def().saver_def().restore_op_name() == name) + return true; + + // If it is the init graph name, it is valid. + std::string init_op_name; + if (internal::GetInitOp("", meta_graph_def(), &init_op_name).ok()) { + if (init_op_name == name) return true; + } + + return false; + } + + // `graph_` contains the entire graph in the original MetaGraphDef. + std::unique_ptr graph_; +}; + +static absl::flat_hash_set GetOriginalTfFuncNamesFromGraphDef( + const GraphDef& graph_def) { + absl::flat_hash_set original_func_tf_names; + for (const auto& function : graph_def.library().function()) { + original_func_tf_names.insert(function.signature().name()); + } + return original_func_tf_names; +} + +// A helper class to import a TensorFlow model expressed in SavedModel V1 into +// an MLIR Module in SavedModel dialect. +// +// TODO(b/179683149): Rename this class to avoid confusion with TFLite. +class SavedModelSignatureDefImporterLite { + public: + // Main entry point: converts all functions (specified by SignatureDefs) in + // the given meta graph to an MLIR Module. + // + // `import_restore` is introduced to control whether restore graph + // is imported in eg. SavedModelSignatureDefImporter. Ideally, we don't need + // this option to control this as restore graph should be always imported. + // However, right now, SavedModelSignatureDefImporter cannot handle restore + // graph correctly. + // + // TODO(chky): Remove import_restore once the restore graph is correctly + // handled in SavedModelSignatureDefImporter. + static absl::StatusOr> Convert( + SavedModelMLIRImportInput& input, + std::optional> exported_names, + mlir::MLIRContext* context, bool import_restore = true, + bool unconditionally_use_set_output_shapes = false) { + SavedModelSignatureDefImporterLite importer( + input, exported_names, context, import_restore, + unconditionally_use_set_output_shapes); + return importer.ConvertSignatures(); + } + + private: + SavedModelSignatureDefImporterLite( + SavedModelMLIRImportInput& input, + std::optional> exported_names, + mlir::MLIRContext* context, bool import_restore, + bool unconditionally_use_set_output_shapes) + : input_(input), + original_func_tf_names_(GetOriginalTfFuncNamesFromGraphDef( + input.meta_graph_def().graph_def())), + exported_names_(exported_names), + module_(mlir::ModuleOp::create(mlir::UnknownLoc::get(context))), + symbol_table_(module_.get()), + import_restore_(import_restore), + unconditionally_use_set_output_shapes_( + unconditionally_use_set_output_shapes) {} + + // Converts the SavedModel to the SavedModel dialect. Creates an MLIR function + // for each signature. + absl::StatusOr> ConvertSignatures(); + absl::Status ConvertSignature(const std::string& sig_def_key, + const SignatureDef& signature_def); + + struct AssetInfo { + std::string tensor_name; + mlir::tf_saved_model::AssetOp op; + }; + absl::StatusOr> ConvertAssets(); + + // Converts the initialization graph in the SavedModel to an MLIR function. + // Attaches `tf_saved_model.initializer_type` attribute with value + // `initializer_type` to the created function. + absl::Status ConvertInitializer(const std::string& target_node_name, + const std::vector& assets, + llvm::StringRef initializer_type); + + // Converts a graph with feeds and fetches to an MLIR function. + absl::StatusOr> ConvertGraph( + const std::string& name, + const std::vector>& inputs, + const std::vector>& outputs, + std::vector control_outputs, + std::unordered_map* tf_name_to_mlir_name); + + // Moves the functions in `sub_module` to `module_` and skips the duplicate + // functions. + absl::Status MoveConvertedFunctionsToModule( + absl::string_view name, mlir::ModuleOp sub_module, + const std::unordered_map& tf_name_to_mlir_name); + + absl::StatusOr ParseInputArrays( + llvm::ArrayRef> inputs); + + private: + SavedModelMLIRImportInput& input_; + absl::flat_hash_set original_func_tf_names_; + std::optional> exported_names_; + mlir::OwningOpRef module_; + absl::Mutex symbol_table_mu_; + mlir::SymbolTable symbol_table_ ABSL_GUARDED_BY(symbol_table_mu_); + bool import_restore_ = true; + bool unconditionally_use_set_output_shapes_ = false; +}; + +absl::StatusOr> +SavedModelSignatureDefImporterLite::ConvertAssets() { + std::vector asset_file_defs; + TF_RETURN_IF_ERROR( + internal::GetAssetFileDefs(input_.meta_graph_def(), &asset_file_defs)); + + std::vector results; + results.reserve(asset_file_defs.size()); + + mlir::OpBuilder builder(module_->getBodyRegion()); + unsigned i = 0; // Use to generate unique sym_name(s) for duplicate assets. + for (const auto& asset : asset_file_defs) { + auto asset_op = mlir::tf_saved_model::AssetOp::create( + builder, module_->getLoc(), + /*sym_name=*/ + builder.getStringAttr( + absl::StrCat("__tf_saved_model_asset", i++, "_", asset.filename())), + /*filename=*/ + builder.getStringAttr( + io::JoinPath(kSavedModelAssetsDirectory, asset.filename()))); + + results.push_back({asset.tensor_info().name(), asset_op}); + } + + return results; +} + +absl::Status SavedModelSignatureDefImporterLite::MoveConvertedFunctionsToModule( + absl::string_view name, mlir::ModuleOp sub_module, + const std::unordered_map& tf_name_to_mlir_name) { + mlir::Builder builder(sub_module.getContext()); + mlir::SymbolTable sub_module_symbol_table(sub_module); + + // Functions originally from graphdef library might have a different name + // after conversion, we build the set of the converted names + absl::flat_hash_set original_func_mlir_names; + for (const auto& kv : tf_name_to_mlir_name) { + if (original_func_tf_names_.contains(kv.first)) + original_func_mlir_names.insert(kv.second); + } + + // Prefix private functions with the unique signature name, so that it cannot + // collide with private functions used in the other signatures. + for (auto func : sub_module.getOps()) { + if (mlir::tf_saved_model::IsExported(func)) continue; + + // Skip the original functions from graphdef library + if (original_func_mlir_names.count(func.getSymName().str())) continue; + + std::string new_sym_name = absl::StrCat(name, "/", func.getSymName().str()); + mlir::StringAttr new_sym_name_attr = builder.getStringAttr(new_sym_name); + if (mlir::failed(sub_module_symbol_table.replaceAllSymbolUses( + func, new_sym_name_attr, sub_module))) + return absl::InvalidArgumentError(absl::StrCat( + "SavedModelSignatureDefImporterLite: failed to assign a unique " + "name to the private function used in a signature: ", + func.getSymName().str())); + + mlir::SymbolTable::setSymbolName(func, new_sym_name); + } + + // Copy all functions used by this signature to the final MLIR module. + for (auto func : sub_module.getOps()) { + absl::MutexLock l(symbol_table_mu_); + // The insert here is a NO-OP if the function already exists. + symbol_table_.insert(func.clone()); + } + + return absl::OkStatus(); +} + +absl::Status SavedModelSignatureDefImporterLite::ConvertInitializer( + const std::string& target_node_name, const std::vector& assets, + llvm::StringRef initializer_type) { + std::vector> inputs; + inputs.reserve(assets.size()); + for (const auto& asset : assets) { + TensorInfo tensor_info; + tensor_info.set_name(asset.tensor_name); + tensor_info.set_dtype(DT_STRING); + tensor_info.mutable_tensor_shape(); + inputs.push_back({asset.tensor_name, tensor_info}); + } + + std::unordered_map tf_name_to_mlir_name; + TF_ASSIGN_OR_RETURN(auto sub_module, + ConvertGraph(target_node_name, inputs, {}, + {target_node_name}, &tf_name_to_mlir_name)); + + mlir::SymbolTable sub_symbol_table(*sub_module); + + auto init_func_op = + sub_symbol_table.lookup(target_node_name); + init_func_op->removeAttr("tf.entry_function"); + + mlir::OpBuilder builder(module_->getBodyRegion()); + + // Bind asset inputs to asset ops. + DCHECK_EQ(init_func_op.getNumArguments(), assets.size()); + for (const auto& iter : llvm::enumerate(assets)) { + auto asset_op = iter.value().op; + init_func_op.setArgAttr( + iter.index(), "tf_saved_model.bound_input", + mlir::SymbolRefAttr::get(builder.getContext(), asset_op.getName())); + } + + // Set the exported name of init function to an reserved name for + // tf_saved_model. + init_func_op->setAttr( + kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr({absl::StrCat( + "__tf_saved_model_session_initializer_", target_node_name)})); + init_func_op->setAttr(kTfSavedModelInitializerTypeAttr, + builder.getStringAttr(initializer_type)); + + // Move the converted functions to top level MLIR module. + return MoveConvertedFunctionsToModule(target_node_name, *sub_module, + tf_name_to_mlir_name); +} + +absl::StatusOr> +SavedModelSignatureDefImporterLite::ConvertGraph( + const std::string& name, + const std::vector>& inputs, + const std::vector>& outputs, + const std::vector control_outputs, + std::unordered_map* tf_name_to_mlir_name) { + VLOG(1) << "Importing Signature: " << name; + + GraphImportConfig specs; + specs.graph_func_name = name; + specs.prune_unused_nodes = true; + TF_ASSIGN_OR_RETURN(specs.inputs, ParseInputArrays(inputs)); + for (auto& output : outputs) { + TF_ASSIGN_OR_RETURN(std::string name, + GetDenseTensorNameFromTensorInfo(output.second)); + specs.outputs.push_back(std::move(name)); + } + specs.control_outputs = control_outputs; + specs.enable_shape_inference = false; + specs.unconditionally_use_set_output_shapes = + unconditionally_use_set_output_shapes_; + + TF_ASSIGN_OR_RETURN(const auto* subgraph, input_.GetSubGraph(name, specs)); + + // Convert sub-graph to MLIR module. + return tensorflow::tf2xla::v2::ConvertGraphToTfExecutor( + *subgraph, input_.debug_info(), subgraph->flib_def(), specs, + module_->getContext(), tf_name_to_mlir_name); +} + +absl::Status SavedModelSignatureDefImporterLite::ConvertSignature( + const std::string& sig_def_key, const SignatureDef& signature_def) { + // Create local vectors for the input and output and sort them to be + // deterministic. We don't want anyone to really depend on the order, client + // should lookup argument/result mapping by attribute name. + // To avoid accidentally depending on the order we use an unintuitive sorting. + std::vector> inputs( + signature_def.inputs().begin(), signature_def.inputs().end()); + llvm::sort(inputs, [](const auto& lhs, const auto& rhs) { + return tensorflow::Fingerprint64(lhs.first) < + tensorflow::Fingerprint64(rhs.first); + }); + std::vector> outputs( + signature_def.outputs().begin(), signature_def.outputs().end()); + llvm::sort(outputs, [](const auto& lhs, const auto& rhs) { + return tensorflow::Fingerprint64(lhs.first) < + tensorflow::Fingerprint64(rhs.first); + }); + + std::unordered_map tf_name_to_mlir_name; + + // Convert sub-graph to MLIR module. + TF_ASSIGN_OR_RETURN( + auto sub_module, + ConvertGraph(sig_def_key, inputs, outputs, {}, &tf_name_to_mlir_name)); + mlir::OpBuilder builder(sub_module->getBodyRegion()); + + // Find the FuncOp which corresponds to current SignatureDef. + mlir::SymbolTable sub_symbol_table(*sub_module); + auto func_op = sub_symbol_table.lookup(sig_def_key); + TF_RET_CHECK(func_op) + << "Graphdef importer should have created a function named " + << sig_def_key << "."; + + // Use unique SignatureDef key as exported name. + func_op->setAttr(kTfSavedModelExportedNamesAttr, + builder.getStrArrayAttr({sig_def_key})); + + // Transfer input and output parameter names to index_path attributes. + for (const auto& input_and_idx : llvm::enumerate(inputs)) { + func_op.setArgAttr(input_and_idx.index(), kTfSavedModelIndexPathAttr, + builder.getStrArrayAttr({input_and_idx.value().first})); + } + for (const auto& output_and_idx : llvm::enumerate(outputs)) { + func_op.setResultAttr( + output_and_idx.index(), kTfSavedModelIndexPathAttr, + builder.getStrArrayAttr({output_and_idx.value().first})); + } + + // Add the original TF function name as a function attribute. + // TODO(b/258817244) Remove this after TFRT exports functions. + for (const auto& [tf_name, mlir_name] : tf_name_to_mlir_name) { + auto func_op = sub_symbol_table.lookup(mlir_name); + TF_RET_CHECK(func_op) + << "Graphdef importer should have created a function named " + << mlir_name << "."; + func_op->setAttr("tf._original_func_name", builder.getStringAttr(tf_name)); + } + + // Move the converted functions to top level MLIR module. + return MoveConvertedFunctionsToModule(sig_def_key, *sub_module, + tf_name_to_mlir_name); +} + +absl::StatusOr +SavedModelSignatureDefImporterLite::ParseInputArrays( + llvm::ArrayRef> inputs) { + GraphImportConfig::InputArrays results; + for (const auto& iter : inputs) { + const auto& tensor_info = iter.second; + + TF_ASSIGN_OR_RETURN(std::string name, + GetDenseTensorNameFromTensorInfo(tensor_info)); + + VLOG(1) << "Importing Signature Input: input_name = " << iter.first + << ", tensor_info = " << tensor_info.DebugString(); + + ArrayInfo array_info; + array_info.imported_dtype = tensor_info.dtype(); + + if (tensor_info.has_tensor_shape()) { + array_info.shape = tensor_info.tensor_shape(); + } else { + // If there is no tensor shape in the tensor info, conservatively set + // unknown_rank to true. + array_info.shape.set_unknown_rank(true); + } + + results.insert(std::pair(std::move(name), + std::move(array_info))); + } + return results; +} + +absl::StatusOr> +SavedModelSignatureDefImporterLite::ConvertSignatures() { + LoadImporterDialects(*module_->getContext()); + + const auto& signatures = input_.meta_graph_def().signature_def(); + PopulateTfVersions(module_.get(), + input_.meta_graph_def().graph_def().versions()); + + llvm::DenseSet exported_name_set; + bool import_all_signatures = !exported_names_.has_value(); + if (exported_names_.has_value()) { + exported_name_set.insert(exported_names_->begin(), exported_names_->end()); + } + + absl::Mutex error_status_mu; // Needed since `error_status` is non-atomic. + absl::Status error_status; + { + // Start a threadpool to convert signatures, since signature conversion can + // be time consuming especially for large models. Threadpool destructor + // blocks until all work is done. + thread::ThreadPool thread_pool(Env::Default(), "ConvertSignatures", + kNumThreadToConvertSignatures); + for (const auto& key_and_signature_def : signatures) { + const std::string& sig_def_key = key_and_signature_def.first; + const SignatureDef& signature_def = key_and_signature_def.second; + + // It is safe to skip "__saved_model_init_op" since it is an internal + // signature that is not user-accessible. This signature will be handled + // in ConvertInitializer(). + if (sig_def_key == "__saved_model_init_op") { + continue; + } + if (!import_all_signatures && exported_name_set.count(sig_def_key) == 0) { + continue; + } + + thread_pool.Schedule([&]() { + auto status = ConvertSignature(sig_def_key, signature_def); + if (!status.ok()) { + absl::MutexLock l(error_status_mu); + error_status = std::move(status); + } + }); + } + } + TF_RETURN_IF_ERROR(error_status); + + TF_ASSIGN_OR_RETURN(auto assets, ConvertAssets()); + + mlir::OpBuilder builder(module_->getBodyRegion()); + llvm::SmallVector init_sym_refs; + + if (import_restore_ && input_.meta_graph_def().has_saver_def()) { + std::vector variable_and_assets; + + // Create an AssetOp for the variable checkpoint files. The relative + // filename is used here. + auto variable_filename_op = mlir::tf_saved_model::AssetOp::create( + builder, module_->getLoc(), + /*sym_name=*/ + builder.getStringAttr("__tf_saved_model_variables"), + /*filename=*/ + builder.getStringAttr(io::JoinPath(kSavedModelVariablesDirectory, + kSavedModelVariablesFilename))); + variable_and_assets.push_back( + {input_.meta_graph_def().saver_def().filename_tensor_name(), + variable_filename_op}); + variable_and_assets.insert(variable_and_assets.end(), assets.begin(), + assets.end()); + + const auto& restore_op_name = + input_.meta_graph_def().saver_def().restore_op_name(); + TF_RETURN_IF_ERROR(ConvertInitializer(restore_op_name, variable_and_assets, + kTfSavedModelInitializerRestoreType)); + init_sym_refs.push_back( + mlir::SymbolRefAttr::get(builder.getContext(), restore_op_name)); + } + + std::string init_op_name; + TF_RETURN_IF_ERROR( + internal::GetInitOp("", input_.meta_graph_def(), &init_op_name)); + if (!init_op_name.empty()) { + TF_RETURN_IF_ERROR(ConvertInitializer(init_op_name, assets, + kTfSavedModelInitializerInitType)); + init_sym_refs.push_back( + mlir::SymbolRefAttr::get(builder.getContext(), init_op_name)); + } + + mlir::tf_saved_model::SessionInitializerOp::create( + builder, module_->getLoc(), builder.getArrayAttr(init_sym_refs)); + + (*module_)->setAttr("tf_saved_model.semantics", builder.getUnitAttr()); + + SortSavedModelModule(*module_); + MarkSavedModelFunctionVisibility(*module_); + + return std::move(module_); +} + +// A helper class to import a TensorFlow model expressed in SavedModel V1 into +// an MLIR Module in SavedModel dialect. In addition to importing the model, it +// performs a few graph transformations, including: +// 1) Convert read-only ref variables to resource variables +// 2) Lift resource variables to global_tensors by using a TF session. +class SavedModelSignatureDefImporter { + public: + // Main entry point: converts all functions (specified by SignatureDefs) in + // the given meta graph to an MLIR Module. + static absl::StatusOr> Convert( + const SavedModelBundle& bundle, + std::optional> exported_names, + mlir::MLIRContext* context, tensorflow::MLIRImportOptions options) { + // debug_info might not be loaded with loader_lite. + GraphDebugInfo debug_info; + if (bundle.debug_info != nullptr) debug_info = *bundle.debug_info; + + TF_ASSIGN_OR_RETURN(auto input, + SimpleSavedModelMLIRImportInput::Create( + options, &bundle.meta_graph_def, debug_info)); + + TF_ASSIGN_OR_RETURN(auto module, + SavedModelSignatureDefImporterLite::Convert( + input, exported_names, context, + /*import_restore=*/false)); + + mlir::OpBuilder builder(module->getContext()); + (*module)->setAttr("tf_saved_model.under_construction", + builder.getUnitAttr()); + TF_RETURN_IF_ERROR( + LiftVariables(bundle, *module, options.lift_variables, + options.include_variables_in_initializers, + options.import_variables_as_dense_resources)); + (*module)->removeAttr("tf_saved_model.under_construction"); + + return module; + } + + private: + // Lifts the variables in `module`. + // If `include_variables_in_initializers` is set to false, then it removes all + // variables from the initializer functions (registered in the + // `tf_saved_model::SessionInitializerOp`) by running the + // `RemoveVariablesInSessionInitializerPass`, regardless of whether + // `lift_variable_ops_to_args` is true or not. + static absl::Status LiftVariables(const SavedModelBundle& bundle, + mlir::ModuleOp module, + bool lift_varhandle_ops_to_args, + bool include_variables_in_initializers, + bool import_variables_as_dense_resources); +}; + +absl::Status SavedModelSignatureDefImporter::LiftVariables( + const SavedModelBundle& bundle, mlir::ModuleOp module, + const bool lift_varhandle_ops_to_args, + const bool include_variables_in_initializers, + const bool import_variables_as_dense_resources) { + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + + mlir::PassManager pm(module.getContext()); + SetCrashReproducer(pm); + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass()); + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + if (!include_variables_in_initializers) { + pm.addPass( + mlir::tf_saved_model::CreateRemoveVariablesInSessionInitializerPass()); + } + pm.addNestedPass( + mlir::TF:: + CreateConvertReadonlyReferenceVariablesToResourceVariablesPass()); + if (mlir::failed(pm.run(module))) + return diag_handler.Combine( + absl::InternalError("Failed to prepare to lift variables.")); + + if (lift_varhandle_ops_to_args) { + if (failed(mlir::tf_saved_model::MarkInitializedVariablesInFunction( + module, bundle.GetSession()))) + return diag_handler.Combine(absl::InternalError( + "Failed to prepare to mark initialized variables.")); + pm.clear(); + pm.addPass(mlir::TF::CreatePromoteVarHandlesToArgsPass()); + if (mlir::failed(pm.run(module))) + return diag_handler.Combine( + absl::InternalError("Failed to promote var handles to args.")); + if (failed(mlir::tf_saved_model::LiftVariables( + module, bundle.GetSession(), import_variables_as_dense_resources))) + return diag_handler.Combine( + absl::InternalError("Failed to lift variables.")); + } else { + if (failed(mlir::tf_saved_model::InitializeVariablesInSessionInitializer( + module, bundle.GetSession()))) + return diag_handler.Combine(absl::InternalError( + "Failed to initialize variables in session init.")); + } + + pm.clear(); + pm.addNestedPass( + mlir::tf_saved_model::CreateDedupBoundInputBindingPass()); + if (mlir::failed(pm.run(module))) + return diag_handler.Combine( + absl::InternalError("Failed to dedup bound inputs.")); + + return absl::OkStatus(); +} + +} // namespace + +SavedModelMLIRImportInput::~SavedModelMLIRImportInput() = default; + +absl::StatusOr> ConvertSavedModelToMlir( + SavedModelV2Bundle* saved_model, mlir::MLIRContext* context, + absl::Span exported_names, MLIRImportOptions options) { + return ConvertSavedModelObjectGraph(saved_model, exported_names, context, + options); +} + +absl::StatusOr> ConvertSavedModelV1ToMlir( + const SavedModelBundle& saved_model, absl::Span exported_names, + mlir::MLIRContext* context, MLIRImportOptions options) { + std::optional> optional_exported_names; + // TODO(b/187062560): Change ConvertSavedModelV1ToMlir() to take an optional + // `exported_names` so that it can be configured to import only restore/init + // graphs. + if (!exported_names.empty()) optional_exported_names = exported_names; + return SavedModelSignatureDefImporter::Convert( + saved_model, optional_exported_names, context, options); +} + +absl::StatusOr> ConvertSavedModelV1ToMlirLite( + const MetaGraphDef& meta_graph_def, const GraphDebugInfo& debug_info, + std::optional> exported_names, + mlir::MLIRContext* context, MLIRImportOptions options) { + TF_ASSIGN_OR_RETURN(auto input, SimpleSavedModelMLIRImportInput::Create( + options, &meta_graph_def, debug_info)); + return ConvertSavedModelV1ToMlirLite( + input, exported_names, context, + options.unconditionally_use_set_output_shapes); +} + +absl::StatusOr> ConvertSavedModelV1ToMlirLite( + SavedModelMLIRImportInput& input, + std::optional> exported_names, + mlir::MLIRContext* context, bool unconditionally_use_set_output_shapes) { + return SavedModelSignatureDefImporterLite::Convert( + input, exported_names, context, + /*import_restore=*/true, unconditionally_use_set_output_shapes); +} + +std::string MlirModuleToString(mlir::ModuleOp module, + mlir::OpPrintingFlags flags) { + std::string txt_module; + { + llvm::raw_string_ostream os{txt_module}; + module.print(os, flags); + } + return txt_module; +} + +std::string MlirModuleToString(mlir::ModuleOp module, bool show_debug_info) { + mlir::OpPrintingFlags flags; + if (show_debug_info) flags.enableDebugInfo(); + return MlirModuleToString(module, flags); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.h new file mode 100644 index 0000000000000000000000000000000000000000..3d428f2c5f1b15cb1b1355cc727696e6c6686075 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/import_model.h @@ -0,0 +1,132 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_IMPORT_MODEL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_IMPORT_MODEL_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/log/check.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/cc/saved_model/bundle_v2.h" +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" + +namespace tensorflow { + +inline constexpr absl::string_view kImportModelDefaultGraphFuncName = "main"; + +// Given a SavedModel, returns a MLIR module containing the functions, expressed +// with tf_executor dialect. +absl::StatusOr> ConvertSavedModelToMlir( + SavedModelV2Bundle* saved_model, mlir::MLIRContext* context, + absl::Span exported_names, MLIRImportOptions options = {}); + +// Given a V1 SavedModel, returns a MLIR module containing the functions, +// expressed with tf_executor dialect. +absl::StatusOr> ConvertSavedModelV1ToMlir( + const SavedModelBundle& saved_model, absl::Span exported_names, + mlir::MLIRContext* context, MLIRImportOptions options = {}); + +// Given a V1 SavedModel, returns a MLIR module containing the functions, +// expressed with tf_executor dialect. It does not require a session to be +// created and it does not perform any graph transformation. If `exported_names` +// is std::nullopt, all signatures will be imported. Otherwise, only names +// in `exported_names` are imported. +// +// Note that the word `Lite` means it is a lighter version compared to +// ConvertSavedModelV1ToMlir(), and is not related to TFLite. +// +// TODO(b/179683149): Rename this class to avoid confusion with TFLite. +absl::StatusOr> ConvertSavedModelV1ToMlirLite( + const MetaGraphDef& meta_graph_def, const GraphDebugInfo& debug_info, + std::optional> exported_names, + mlir::MLIRContext* context, MLIRImportOptions options); + +// SavedModelMLIRImportInput is an adapter class for users to inject custom +// graph transformation logic on Tensorflow graphs before importing to MLIR. It +// serves as the source that provides the subgraphs requested by the savedmodel +// MLIR importer, and at the same time it allows the implementation of this +// class to transform the graph before feeding it to the importer. +class SavedModelMLIRImportInput { + public: + SavedModelMLIRImportInput(const MetaGraphDef* meta_graph_def, + const GraphDebugInfo& debug_info) + : meta_graph_def_(meta_graph_def), debug_info_(debug_info) { + DCHECK(meta_graph_def); + } + + virtual ~SavedModelMLIRImportInput(); + + // The original MetaGraphDef of the savedmodel. + const MetaGraphDef& meta_graph_def() const { return *meta_graph_def_; } + + const GraphDebugInfo& debug_info() const { return debug_info_; } + + // GetSubGraph() is expected to return a tensorflow::Graph that contains the + // node set specified in `specs`. The implementation is free to transform the + // graph in the original savedmodel as needed, as long as it produces the same + // results and effects. If the transformation requires some configs in `spec` + // (e.g., control_outputs) to be changed, they should be updated accordingly + // and remain valid for the graph. + // `name` is a unique identifier for this subgraph, so the implementation can + // use it for eg. debugging or caching compilation results. + virtual absl::StatusOr GetSubGraph( + absl::string_view name, GraphImportConfig& specs) = 0; + + private: + const MetaGraphDef* meta_graph_def_ = nullptr; + GraphDebugInfo debug_info_; +}; + +// Given the SavedModelMLIRImportInput for a saved model, returns a MLIR module +// containing the functions, expressed with tf_executor dialect. It does not +// require a session to be created. If `exported_names` is std::nullopt, all +// signatures will be imported. Otherwise, only names in `exported_names` are +// imported. + +// +// Note that the word `Lite` means it is a lighter version compared to +// ConvertSavedModelV1ToMlir(), and is not related to TFLite. +// +// TODO(b/179683149): Rename this class to avoid confusion with TFLite. +absl::StatusOr> ConvertSavedModelV1ToMlirLite( + SavedModelMLIRImportInput& input, + std::optional> exported_names, + mlir::MLIRContext* context, + bool unconditionally_use_set_output_shapes = false); + +// Serialize a MLIR module to a string. +std::string MlirModuleToString(mlir::ModuleOp m, bool show_debug_info = false); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_IMPORT_MODEL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h new file mode 100644 index 0000000000000000000000000000000000000000..59c43244df3f89cf54d99dbc6c3cac4a87eae06a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h @@ -0,0 +1,60 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_IMPORT_OPTIONS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_IMPORT_OPTIONS_H_ + +namespace tensorflow { + +// TODO(jpienaar): This file and class are confusingly named. This seems to be +// a SavedModel only import options file that exposes a subset of the +// GraphImportConfig options, but the naming would make one think it is more +// general. +struct MLIRImportOptions { + // If true, functionalize the input graph before importing it into MLIR. + bool upgrade_legacy = false; + + // Whether to unconditionally use the shape set via _output_shapes on import. + bool unconditionally_use_set_output_shapes = false; + + // Apply default attributes from the op definition to the loaded op. + bool add_default_attributes = true; + + // If set, promote tf.VarHandleOp to resource arguments for all functions. + bool lift_variables = true; + + // Keeps the variables in initializers before lifting variables (when + // `lift_variables == true`) or newly adding variable initialization patterns + // in the initializer functions. One might want to set this to `true` because + // the `RemoveVariablesInSessionInitializerPass` pass, which runs otherwise, + // may unexpectedly also remove the initialization patterns for non-variable + // resources (like hash tables) if they involve variables. Such a case is + // illustrated in the test file + // "../tests/tf_saved_model_remove_vars_in_session_initializer.mlir". + // This defaults to `false` to avoid breaking existing uses. + bool include_variables_in_initializers = false; + + // Load the model without restoring associated variables from disk. Enables + // loading raw programs without checkpoints. + bool allow_uninitialized_variables = false; + + // If true, variables are imported as DenseResourceElementsAttr; else, + // variables are imported as DenseElementsAttr. + bool import_variables_as_dense_resources = false; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_IMPORT_OPTIONS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h new file mode 100644 index 0000000000000000000000000000000000000000..d51667bea8a65c9ad1ebe29952586584ae08582e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h @@ -0,0 +1,22 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_ROUNDTRIP_FLAGS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_ROUNDTRIP_FLAGS_H_ + +// Transitional forwarding header. +#include "tensorflow/compiler/mlir/tf2xla/api/v2/mlir_roundtrip_flags.h" + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_ROUNDTRIP_FLAGS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc new file mode 100644 index 0000000000000000000000000000000000000000..1ed4cb4914f0734b7ee5d809ab34c355eaecef75 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc @@ -0,0 +1,143 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h" + +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/cc/saved_model/bundle_v2.h" +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/reader.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/import_model.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow/core/public/session_options.h" + +namespace tensorflow { + +absl::StatusOr> +SavedModelObjectGraphToMlirImport(absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, + mlir::MLIRContext* context, + bool unconditionally_use_set_output_shapes, + bool import_variables_as_dense_resources) { + tensorflow::SavedModelV2Bundle bundle; + auto load_status = tensorflow::SavedModelV2Bundle::Load( + std::string(saved_model_dir.data(), saved_model_dir.length()), &bundle); + if (!load_status.ok()) { + LOG(ERROR) << "Failed to load saved model '" << saved_model_dir + << "': " << load_status; + return load_status; + } + + MLIRImportOptions options; + options.add_default_attributes = true; + options.unconditionally_use_set_output_shapes = + unconditionally_use_set_output_shapes; + options.import_variables_as_dense_resources = + import_variables_as_dense_resources; + + auto module_or = + ConvertSavedModelToMlir(&bundle, context, exported_names, options); + if (!module_or.status().ok()) { + LOG(ERROR) << "SavedModel import failed: " << module_or.status(); + } + return module_or; +} + +absl::StatusOr> +SavedModelSignatureDefsToMlirImport( + absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, mlir::MLIRContext* context, + MLIRImportOptions options, + std::unique_ptr* saved_model_bundle) { + // Create local bundle if no one is provided to use. + std::unique_ptr bundle; + if (saved_model_bundle == nullptr) { + bundle = std::make_unique(); + } else if (*saved_model_bundle == nullptr) { + *saved_model_bundle = std::make_unique(); + } + SavedModelBundle* bundle_ptr = + saved_model_bundle ? saved_model_bundle->get() : bundle.get(); + tensorflow::SessionOptions session_options; + + // Force saved model states to be restored to CPU. + (*session_options.config.mutable_device_count())["GPU"] = 0; + auto load_status = tensorflow::LoadSavedModel( + session_options, /* run_options = */ {}, std::string(saved_model_dir), + tags, bundle_ptr); + if (!load_status.ok()) { + LOG(ERROR) << "Failed to load saved model v1 '" << saved_model_dir + << "': " << load_status; + return load_status; + } + + auto module_or = + ConvertSavedModelV1ToMlir(*bundle_ptr, exported_names, context, options); + if (!module_or.status().ok()) { + LOG(ERROR) << "SavedModel V1 import failed: " << module_or.status(); + } + return module_or; +} + +absl::StatusOr> +SavedModelSignatureDefsToMlirImportLite( + absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, mlir::MLIRContext* context, + MLIRImportOptions options) { + MetaGraphDef meta_graph_def; + auto status = + ReadMetaGraphDefFromSavedModel(saved_model_dir, tags, &meta_graph_def); + if (!status.ok()) { + LOG(ERROR) << "Failed to load saved model v1 '" << saved_model_dir + << "': " << status; + return status; + } + + std::optional> optional_exported_names; + if (!exported_names.empty()) optional_exported_names = exported_names; + + // TODO(b/186898924): debug info in the savedmodel should not be ignored and + // should be passed here. + auto module_or = + ConvertSavedModelV1ToMlirLite(meta_graph_def, /*debug_info=*/{}, + optional_exported_names, context, options); + if (!module_or.status().ok()) { + LOG(ERROR) << "SavedModel V1 import failed: " << module_or.status(); + } + return module_or; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h new file mode 100644 index 0000000000000000000000000000000000000000..9dec85d0398a67b528941792799049838a54f0ee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h @@ -0,0 +1,71 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TF_MLIR_TRANSLATE_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TF_MLIR_TRANSLATE_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_import_options.h" + +namespace tensorflow { + +// Converts a TensorFlow SavedModel stored in the directory with the given +// `saved_model_dir` into a MLIR module. Creates MLIR entities into the +// given MLIR `context`. +absl::StatusOr> +SavedModelObjectGraphToMlirImport( + absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, mlir::MLIRContext* context, + bool unconditionally_use_set_output_shapes = false, + bool import_variables_as_dense_resources = false); + +// Converts a TensorFlow V1 SavedModel stored in the directory with the given +// `saved_model_dir` into a MLIR module. Creates MLIR entities into the +// given MLIR `context`. +// 'saved_model_bundle' if not null, will be initialized with the model bundle. +absl::StatusOr> +SavedModelSignatureDefsToMlirImport( + absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, mlir::MLIRContext* context, + MLIRImportOptions options, + std::unique_ptr* saved_model_bundle = + nullptr); + +// Converts a TensorFlow V1 SavedModel stored in the directory with the given +// `saved_model_dir` into a MLIR module. Creates MLIR entities into the +// given MLIR `context`. This does not create session internally so it is faster +// and does not perform any graph transformation. +absl::StatusOr> +SavedModelSignatureDefsToMlirImportLite( + absl::string_view saved_model_dir, + const std::unordered_set& tags, + absl::Span exported_names, mlir::MLIRContext* context, + MLIRImportOptions options); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TF_MLIR_TRANSLATE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..15846f0b279be43d9c2017c37924ff509be497b7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/BUILD @@ -0,0 +1,63 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) + +cc_library( + name = "parsers", + srcs = ["parsers.cc"], + hdrs = ["parsers.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@xla//xla:status_macros", + ], +) + +cc_library( + name = "file_tf_mlir_translate", + srcs = ["file_tf_mlir_translate.cc"], + hdrs = ["file_tf_mlir_translate.h"], + deps = [ + ":parsers", + "//tensorflow/compiler/mlir/tensorflow:import_utils", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tf2xla/api/v2:graph_to_tf_executor", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:graph_proto_cc", + "//tensorflow/core/grappler/utils:transitive_fanin", + "//tensorflow/core/platform:byte_order", + "//tensorflow/core/platform:errors", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core/util/tensor_bundle:byteswaptensor", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.cc new file mode 100644 index 0000000000000000000000000000000000000000..2732e08a8398ae76844dedbf93665321ae38f857 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.cc @@ -0,0 +1,307 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.h" + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/import_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/tensor_id.h" +#include "tensorflow/core/grappler/utils/transitive_fanin.h" +#include "tensorflow/core/platform/byte_order.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow/core/util/tensor_bundle/byte_swap_tensor.h" +namespace tensorflow { + +// Returns true if the node with given name has a non primary output that is +// used by some other node as an input. Returns false if no outputs are in use +// or only the first output is in use. +bool HasNonPrimaryOutputInUse(const GraphDef& graph_def, + const std::string& node) { + for (const auto& node_def : graph_def.node()) { + for (const auto& input : node_def.input()) { + if (absl::StartsWith(input, node + ":") && input != node + ":0") { + return true; + } + } + } + return false; +} + +// Updates the given LegacyFedInput node with Placeholder node if it is one of +// the inputs. Returns an error if non primary output of the LegacyFedInput node +// is in use and therefore can not be replaced by the Placeholder node that only +// has a single output. +absl::Status UpdateLegacyFedInputNode( + const GraphDef& graph_def, const GraphImportConfig::InputArrays& inputs, + NodeDef* node) { + const std::string& node_name = node->name(); + auto it = inputs.find(node_name); + + // Node is not an input. + if (it == inputs.end()) return absl::OkStatus(); + + if (HasNonPrimaryOutputInUse(graph_def, node_name)) { + return absl::InvalidArgumentError(absl::StrCat( + "LegacyFedInput node ", node->name(), + " has non primary output in use and can not be replaced with " + "Placeholder node")); + } + + DataType dtype = it->second.imported_dtype; + // Uses the existing output type if it isn't specified by the user. + if (dtype == DT_INVALID && + node->attr().at("output_types").list().type_size() > 0) { + dtype = node->attr().at("output_types").list().type(0); + } + // Update op name, drop inputs and set attributes required by the Placeholder + // op. + *node->mutable_op() = "Placeholder"; + node->clear_attr(); + node->clear_input(); + AddNodeAttr("dtype", dtype, node); + AddNodeAttr("shape", it->second.shape, node); + return absl::OkStatus(); +} + +static absl::StatusOr> GraphdefToMlirImport( + llvm::StringRef input, const std::vector& input_arrays, + const std::vector& input_dtypes, + const std::vector>>& input_shapes, + const std::vector& output_arrays, + const std::vector& control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context) { + GraphDef graphdef; + TF_RETURN_IF_ERROR( + tensorflow::LoadProtoFromBuffer({input.data(), input.size()}, &graphdef)); + if (!port::kLittleEndian) + TF_RETURN_IF_ERROR(ByteSwapTensorContentInGraphDef(&graphdef)); + + GraphDebugInfo debug_info; + if (!import_options.debug_info_file.empty()) { + TF_RETURN_IF_ERROR( + LoadProtoFromFile(import_options.debug_info_file, &debug_info)); + } + + GraphImportConfig specs; + specs.prune_unused_nodes = import_options.prune_unused_nodes; + specs.convert_legacy_fed_inputs = import_options.convert_legacy_fed_inputs; + specs.graph_as_function = import_options.graph_as_function; + specs.upgrade_legacy = import_options.upgrade_legacy; + specs.enable_shape_inference = import_options.enable_shape_inference; + specs.unconditionally_use_set_output_shapes = + import_options.unconditionally_use_set_output_shapes; + specs.xla_compile_device_type = import_options.xla_compile_device_type; + specs.enable_soft_placement = import_options.enable_soft_placement; + specs.set_original_tf_func_name = import_options.set_original_tf_func_name; + TF_RETURN_IF_ERROR(ParseInputArrayInfo(input_arrays, input_dtypes, + input_shapes, &specs.inputs)); + TF_RETURN_IF_ERROR(ParseOutputArrayInfo(output_arrays, &specs.outputs)); + TF_RETURN_IF_ERROR( + ParseOutputArrayInfo(control_output_arrays, &specs.control_outputs)); + // TODO(hinsu): Completely deprecate support for LegacyFedInput ops. One + // solution could be have a tool to let users upgrade old serialized graphs. + for (auto& node_def : *graphdef.mutable_node()) { + if (specs.convert_legacy_fed_inputs && node_def.op() == "LegacyFedInput") { + TF_RETURN_IF_ERROR( + UpdateLegacyFedInputNode(graphdef, specs.inputs, &node_def)); + } + } + // TODO(b/142828368): Pruning should not be needed when TF import + // supports importing graphs w/ unregistered ops natively. + GraphDef pruned_graph_def; + if (specs.prune_unused_nodes) { + std::vector terminal_nodes; + terminal_nodes.reserve(specs.outputs.size() + specs.inputs.size()); + for (const auto& output : specs.outputs) { + terminal_nodes.push_back(std::string(ParseTensorName(output).node())); + } + for (const auto& control_output : specs.control_outputs) { + terminal_nodes.push_back(std::string(control_output)); + } + for (const auto& input : specs.inputs) { + terminal_nodes.push_back(input.first); + } + TF_RETURN_IF_ERROR(tensorflow::grappler::SetTransitiveFaninGraph( + graphdef, &pruned_graph_def, terminal_nodes)); + // TODO(ashwinm): Add a separate utility in grappler utils that abstracts + // both SetTransitiveFaninGraph and restoring the missing contents from the + // original graph like function def library and version. + pruned_graph_def.mutable_library()->Swap(graphdef.mutable_library()); + pruned_graph_def.mutable_versions()->Swap(graphdef.mutable_versions()); + } + + tensorflow::GraphConstructorOptions options; + options.allow_internal_ops = true; + options.upgrade_legacy = specs.upgrade_legacy; + options.add_default_attributes = true; + tensorflow::Graph graph(tensorflow::OpRegistry::Global()); + TF_RETURN_IF_ERROR(::tensorflow::ConvertGraphDefToGraph( + options, + specs.prune_unused_nodes ? std::move(pruned_graph_def) + : std::move(graphdef), + &graph)); + return tensorflow::tf2xla::v2::ConvertGraphToTfExecutor( + graph, debug_info, graph.flib_def(), specs, context); +} + +absl::StatusOr> +GraphdefToMlirTranslateFunction( + llvm::StringRef input, const std::vector& input_arrays, + const std::vector& input_dtypes, + const std::vector>>& input_shapes, + const std::vector& output_arrays, + const std::vector& control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context) { + auto module_or = GraphdefToMlirImport( + input, input_arrays, input_dtypes, input_shapes, output_arrays, + control_output_arrays, import_options, context); + if (!module_or.status().ok()) { + LOG(ERROR) << "Graph import failed: " << module_or.status(); + } + return module_or; +} + +absl::StatusOr> +GraphdefToMlirTranslateFunction( + llvm::StringRef input, absl::string_view input_arrays, + absl::string_view input_dtypes, absl::string_view input_shapes, + absl::string_view output_arrays, absl::string_view control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context) { + std::vector input_array_vector; + std::vector input_dtype_vector; + std::vector>> input_shapes_vector; + std::vector output_array_vector; + std::vector control_output_array_vector; + TF_RETURN_IF_ERROR(ParseNodeNames(input_arrays, input_array_vector)); + TF_RETURN_IF_ERROR(ParseNodeDataTypes(input_dtypes, input_dtype_vector)); + TF_RETURN_IF_ERROR(ParseNodeNames(output_arrays, output_array_vector)); + TF_RETURN_IF_ERROR(ParseNodeShapes(input_shapes, input_shapes_vector)); + TF_RETURN_IF_ERROR( + ParseNodeNames(control_output_arrays, control_output_array_vector)); + return GraphdefToMlirTranslateFunction( + input, input_array_vector, input_dtype_vector, input_shapes_vector, + output_array_vector, control_output_array_vector, import_options, + context); +} + +absl::StatusOr> +GraphdefToSplattedMlirTranslateFunction( + llvm::StringRef input, const std::vector& input_arrays, + const std::vector& input_dtypes, + const std::vector>>& input_shapes, + const std::vector& output_arrays, + const std::vector& control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context) { + auto module_or = GraphdefToMlirImport( + input, input_arrays, input_dtypes, input_shapes, output_arrays, + control_output_arrays, import_options, context); + if (!module_or.status().ok()) { + LOG(ERROR) << "Graph import failed: " << module_or.status(); + return module_or.status(); + } + auto& module = module_or.value(); + std::srand(0); + for (auto fn : module->getOps()) { + for (auto& bb : fn) { + for (auto& inst : bb) { + auto attr_id = mlir::StringAttr::get(context, "value"); + if (auto attr = inst.getAttrOfType(attr_id)) { + mlir::Attribute rand_val; + mlir::Type element_type = attr.getShapedType().getElementType(); + if (mlir::isa(element_type)) { + rand_val = mlir::IntegerAttr::get(element_type, std::rand()); + } else if (element_type.isF16() || element_type.isF32() || + element_type.isF64()) { + rand_val = mlir::FloatAttr::get(element_type, + std::rand() * 1.0 / RAND_MAX); + + } else { + inst.emitWarning() + << "Skipping splat conversion for " + << "an unsupported attribute type " << element_type; + continue; + } + auto new_attr = mlir::DenseElementsAttr::get( + llvm::cast(attr.getType()), rand_val); + inst.setAttr(attr_id, new_attr); + } + } + } + } + return module_or; +} + +absl::StatusOr> +GraphdefToSplattedMlirTranslateFunction( + llvm::StringRef input, absl::string_view input_arrays, + absl::string_view input_dtypes, absl::string_view input_shapes, + absl::string_view output_arrays, absl::string_view control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context) { + std::vector input_array_vector; + std::vector input_dtype_vector; + std::vector>> input_shapes_vector; + std::vector output_array_vector; + std::vector control_output_array_vector; + TF_RETURN_IF_ERROR(ParseNodeNames(input_arrays, input_array_vector)); + TF_RETURN_IF_ERROR(ParseNodeDataTypes(input_dtypes, input_dtype_vector)); + TF_RETURN_IF_ERROR(ParseNodeNames(output_arrays, output_array_vector)); + TF_RETURN_IF_ERROR(ParseNodeShapes(input_shapes, input_shapes_vector)); + TF_RETURN_IF_ERROR( + ParseNodeNames(control_output_arrays, control_output_array_vector)); + return GraphdefToSplattedMlirTranslateFunction( + input, input_array_vector, input_dtype_vector, input_shapes_vector, + output_array_vector, control_output_array_vector, import_options, + context); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.h new file mode 100644 index 0000000000000000000000000000000000000000..b19e8cc655edc4a2aa4c983a46803bc622b389d9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.h @@ -0,0 +1,101 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_FILE_TF_MLIR_TRANSLATE_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_FILE_TF_MLIR_TRANSLATE_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "xla/tsl/platform/statusor.h" + +namespace tensorflow { + +using tsl::Status; +using tsl::StatusOr; + +struct GraphdefToMlirOptions { + std::string debug_info_file; + std::string xla_compile_device_type; + bool prune_unused_nodes; + bool convert_legacy_fed_inputs; + bool graph_as_function; + bool upgrade_legacy; + bool enable_shape_inference; + bool unconditionally_use_set_output_shapes; + bool enable_soft_placement; + bool set_original_tf_func_name = false; +}; + +// TODO(antiagainst): Directly manipulating files in library functions is not +// a good idea. We should pass in a string/stream here. + +// Converts a TensorFlow GraphDef contained in `input` param into a MLIR module. +// Creates MLIR entities into the given MLIR `context`. +absl::StatusOr> +GraphdefToMlirTranslateFunction( + llvm::StringRef input, const std::vector& input_arrays, + const std::vector& input_dtypes, + const std::vector>>& input_shapes, + const std::vector& output_arrays, + const std::vector& control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context); + +ABSL_DEPRECATED( + "Please use the other overload of this function which accepts structured " + "inputs instead of strings") +// Converts a TensorFlow GraphDef contained in `input` param into a MLIR module. +// Creates MLIR entities into the given MLIR `context`. +absl::StatusOr> +GraphdefToMlirTranslateFunction( + llvm::StringRef input, absl::string_view input_arrays, + absl::string_view input_dtypes, absl::string_view input_shapes, + absl::string_view output_arrays, absl::string_view control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context); + +// Similar as the above function, but replaces all constant tensors +// with randomly generated splat values. +absl::StatusOr> +GraphdefToSplattedMlirTranslateFunction( + llvm::StringRef input, const std::vector& input_arrays, + const std::vector& input_dtypes, + const std::vector>& input_shapes, + const std::vector& output_arrays, + const std::vector& control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context); + +ABSL_DEPRECATED( + "Please use the other overload of this function which accepts structured " + "inputs instead of strings") +// Similar as the above function, but replaces all constant tensors +// with randomly generated splat values. +absl::StatusOr> +GraphdefToSplattedMlirTranslateFunction( + llvm::StringRef input, absl::string_view input_arrays, + absl::string_view input_dtypes, absl::string_view input_shapes, + absl::string_view output_arrays, absl::string_view control_output_arrays, + const GraphdefToMlirOptions& import_options, mlir::MLIRContext* context); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_FILE_TF_MLIR_TRANSLATE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.cc new file mode 100644 index 0000000000000000000000000000000000000000..371ecab18342787b9ff5f0e91d4b28e23bfbb80a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.cc @@ -0,0 +1,269 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/STLExtras.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "xla/status_macros.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/types.h" +namespace tensorflow { + +absl::Status ParseOutputArrayInfo(absl::string_view array_names, + std::vector* outputs) { + TF_RETURN_IF_ERROR(ParseNodeNames(array_names, *outputs)); + return absl::OkStatus(); +} + +absl::Status ParseOutputArrayInfo(const std::vector& output_names, + std::vector* outputs) { + for (auto& output_name : output_names) { + if (output_name.empty()) continue; + outputs->push_back(output_name); + } + return absl::OkStatus(); +} + +absl::Status ParseInputArrayInfo(absl::string_view array_names, + absl::string_view data_types, + absl::string_view shapes, + GraphImportConfig::InputArrays* inputs) { + std::vector node_names; + std::vector node_dtypes; + std::vector>> node_shapes; + TF_RETURN_IF_ERROR(ParseNodeNames(array_names, node_names)); + TF_RETURN_IF_ERROR(ParseNodeDataTypes(data_types, node_dtypes)); + TF_RETURN_IF_ERROR(ParseNodeShapes(shapes, node_shapes)); + return ParseInputArrayInfo(node_names, node_dtypes, node_shapes, inputs); +} + +static absl::StatusOr> ParseShapeStr( + absl::string_view node_shapes_str) { + std::vector dims; + for (absl::string_view dim_str : absl::StrSplit(node_shapes_str, ',')) { + // Treats empty input shape as scalar + if (dim_str.empty()) continue; + if (dim_str == "?") { + dims.push_back(-1); + continue; + } + int size; + TF_RET_CHECK(absl::SimpleAtoi(dim_str, &size)); + dims.push_back(size); + } + return dims; +} + +static absl::Status HandleSubtype(absl::string_view subtype, + ArrayInfo::SubTypeInfo* result) { + std::vector shape_and_type = absl::StrSplit(subtype, ':'); + + std::vector dims; + if (shape_and_type.size() > 2) { + return absl::FailedPreconditionError( + absl::StrCat("Invalid argument: '", subtype, + "', expected a single shape and type pair" + " separated with a ':'")); + } else if (shape_and_type.size() == 2) { + const auto& shape_str = shape_and_type[0]; + TF_ASSIGN_OR_RETURN(dims, ParseShapeStr(shape_str)); + } + + const auto& subtype_str = shape_and_type.back(); + DataType subtype_dtype; + if (!DataType_Parse(subtype_str, &subtype_dtype)) { + return absl::FailedPreconditionError( + absl::StrCat("Invalid type: '", subtype_str, "'")); + } + + TensorShapeProto subtype_tensor_shape; + for (auto& dim : dims) { + subtype_tensor_shape.add_dim()->set_size(dim); + } + *result = {subtype_dtype, subtype_tensor_shape}; + return absl::OkStatus(); +} + +absl::Status ParseInputArrayInfo( + const std::vector& node_names, + const std::vector& node_dtypes, + const std::vector>>& node_shapes, + GraphImportConfig::InputArrays* inputs) { + std::vector used_node_dtypes; + if (node_dtypes.empty()) { + // Mark all the node dtypes Invalid, so the importer can handle them by + // using the type from the graph. + used_node_dtypes.resize(node_names.size(), DataType_Name(DT_INVALID)); + } else if (node_names.size() == node_dtypes.size()) { + for (const auto& dtype : node_dtypes) { + if (dtype.empty()) { + used_node_dtypes.push_back(DataType_Name(DT_INVALID)); + } else if (dtype != DataType_Name(DT_INVALID)) { + used_node_dtypes.push_back(dtype); + } else { + return absl::FailedPreconditionError( + "Use '' if want to use the type from graph."); + } + } + } else { + return absl::InvalidArgumentError(absl::StrCat( + "Length of input node array and data type doesn't match (#arrays ", + node_names.size(), ", #data_types ", node_dtypes.size(), ")")); + } + + if (!node_shapes.empty() && node_names.size() != node_shapes.size()) { + return absl::InvalidArgumentError(absl::StrCat( + "Length of input node array and data shape doesn't match (#arrays ", + node_names.size(), ", #input_shapes ", node_shapes.size(), ")")); + } + + // StringMap doesn't support reserve else reserve input map size here. + for (int i = 0, end = node_names.size(); i < end; i++) { + auto& name = node_names[i]; + const std::string& type = used_node_dtypes[i]; + if (name.empty()) continue; + + auto it_inserted_pair = inputs->insert({name, {}}); + if (!it_inserted_pair.second) + return absl::FailedPreconditionError( + absl::StrCat("tensor ", name, " is repeated in the arrays flag")); + + ArrayInfo& info = it_inserted_pair.first->second; + // Splitting the type and subtype into parts + std::vector parts = + absl::StrSplit(type, absl::ByAnyChar("()")); + // If type has subtypes then parts[0] = type, parts[1] = subtypes, + // parts[2] = "" + if (parts.size() != 3 && parts.size() != 1) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid type '", type, "'")); + } else if (parts.size() == 3) { + // First part is the type, second is the subtype + ArrayInfo::SubTypeInfo subtype; + TF_RETURN_IF_ERROR(HandleSubtype(parts[1], &subtype)); + info.subtypes.push_back(std::move(subtype)); + } + if (!DataType_Parse(parts[0], &info.imported_dtype)) { + return absl::FailedPreconditionError( + absl::StrCat("Invalid node type '", node_dtypes[i], "'")); + } + + if (!node_shapes.empty()) { + if (!node_shapes[i].has_value()) { + info.shape.set_unknown_rank(true); + continue; + } + for (auto& dim : node_shapes[i].value()) { + info.shape.add_dim()->set_size(dim); + } + } + } + return absl::OkStatus(); +} + +absl::Status ParseNodeShapes( + absl::string_view shapes_str, + std::vector>>& shapes_vector) { + shapes_vector.clear(); + if (!shapes_str.empty()) { + std::vector node_shapes_str = absl::StrSplit(shapes_str, ':'); + for (int i = 0; i < node_shapes_str.size(); i++) { + if (node_shapes_str[i] == "*") { + shapes_vector.push_back(std::nullopt); + continue; + } + TF_ASSIGN_OR_RETURN(auto shape, ParseShapeStr(node_shapes_str[i])); + shapes_vector.push_back(std::move(shape)); + } + } + return absl::OkStatus(); +} + +absl::Status ParseNodeNames(absl::string_view names_str, + std::vector& names_vector) { + names_vector = absl::StrSplit(names_str, ',', absl::SkipEmpty()); + return absl::OkStatus(); +} + +static absl::StatusOr> ParseDTypesHelper( + absl::string_view data_types_str) { + bool inside_subtype = false; + int cur_pos = 0; + std::vector dtypes; + for (const auto& it : llvm::enumerate(data_types_str)) { + char c = it.value(); + int i = it.index(); + // Skip parsing the subtypes of a type + if (c == '(') { + if (inside_subtype) { + return absl::FailedPreconditionError( + absl::StrCat("Syntax error: unexpected '(' in input data types: '", + data_types_str, "'")); + } + inside_subtype = true; + } else if (c == ')') { + if (!inside_subtype) { + return absl::FailedPreconditionError( + absl::StrCat("Syntax error: unexpected ')' in input data types: '", + data_types_str, "'")); + } + inside_subtype = false; + } + if (inside_subtype) continue; + if (c == ',') { + dtypes.push_back( + std::string(data_types_str.substr(cur_pos, i - cur_pos))); + cur_pos = i + 1; + } + } + if (inside_subtype) { + return absl::FailedPreconditionError( + absl::StrCat("Syntax error: expected a ')' in input data types '", + data_types_str, "'")); + } + if (!data_types_str.empty()) { + dtypes.push_back(std::string( + data_types_str.substr(cur_pos, data_types_str.size() - cur_pos))); + } + return dtypes; +} + +absl::Status ParseNodeDataTypes(absl::string_view data_types_str, + std::vector& data_type_vector) { + data_type_vector.clear(); + if (!data_types_str.empty()) { + TF_ASSIGN_OR_RETURN(data_type_vector, ParseDTypesHelper(data_types_str)); + } + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h new file mode 100644 index 0000000000000000000000000000000000000000..8a25eedc1f29b72c421b535c6e96d4da356ec13b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h @@ -0,0 +1,75 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_PARSERS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_PARSERS_H_ + +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/StringMap.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { + +// Parses the command line flag strings to the specification of nodes in +// the Graph. +absl::Status ParseOutputArrayInfo(absl::string_view array_names, + std::vector* outputs); + +absl::Status ParseOutputArrayInfo(const std::vector& output_names, + std::vector* outputs); + +// Parses the command line flag strings to the specification of nodes in +// the Graph. `data_types` input string can be empty since the flag is optional. +absl::Status ParseInputArrayInfo(absl::string_view array_names, + absl::string_view data_types, + absl::string_view shapes, + GraphImportConfig::InputArrays* inputs); + +absl::Status ParseInputArrayInfo( + const std::vector& node_names, + const std::vector& node_dtypes, + const std::vector>>& node_shapes, + GraphImportConfig::InputArrays* inputs); + +// Parses shapes from the given string into shapes_vector which is a structured +// format. +// NOTE: If shapes_str is empty, shapes_vector will also be empty. +absl::Status ParseNodeShapes( + absl::string_view shapes_str, + std::vector>>& shapes_vector); + +// Parses names from the given string into the names_vector. +// NOTE: If names_str is empty, names_vector will also be empty. +absl::Status ParseNodeNames(absl::string_view names_str, + std::vector& names_vector); + +// Parses data types from the given string into the data_type_vector. +// NOTE: If data_types_str is empty, data_type_vector will also be empty. +absl::Status ParseNodeDataTypes(absl::string_view data_types_str, + std::vector& data_type_vector); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_TOOLS_PARSERS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..01494bb1f6127157d96007cf52286c0b118049ed --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.cc @@ -0,0 +1,95 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +#include +#include +#include + +#include "absl/strings/str_split.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" + +namespace mlir { +namespace TF { + +using ::tensorflow::kValidDeviceTypes; + +LogicalResult HasValidCompilationAndReplicationAttributes(Operation& op) { + auto replicate_attr = op.getAttrOfType(kReplicationInfoAttr); + auto compile_attr = op.getAttrOfType(kCompileDeviceTypeAttr); + if (!replicate_attr && !compile_attr) return success(); + if (!replicate_attr || !compile_attr) + return op.emitOpError() << "is expected to have either both or none of '" + << kReplicationInfoAttr << "' and '" + << kCompileDeviceTypeAttr << "' attributes."; + if (replicate_attr.getValue().empty()) + return op.emitOpError() + << "has an empty '" << kReplicationInfoAttr << "' attribute."; + if (failed(IsValidDeviceTypeOrEmpty(compile_attr))) { + return op.emitOpError() << "has invalid '" << kCompileDeviceTypeAttr + << "' value '" << compile_attr.getValue() << "'"; + } + return success(); +} + +LogicalResult IsValidDeviceTypeOrEmpty(StringAttr device_attr) { + auto value = device_attr.getValue(); + // TODO(b/229028654): Remove string conversion once we have C++17. + absl::string_view device_type(value.data(), value.size()); + // Device type may be empty for some ops, e.g. tf.PartitionedCall. + auto it = std::find(kValidDeviceTypes.begin(), kValidDeviceTypes.end(), + device_type); + if (it == kValidDeviceTypes.end()) return failure(); + return success(); +} + +LogicalResult ParseParallelExecutionIds(Operation* op, + ParallelExecutionIdPairs& id_pairs) { + auto attr = op->getAttrOfType(kParallelExecAnnotation); + if (!attr) return success(); + + // ID pairs are separated by `,`. + llvm::SmallVector str_list = + absl::StrSplit(attr.getValue().str(), ',', absl::SkipWhitespace()); + id_pairs.reserve(str_list.size()); + for (const std::string& str : str_list) { + // IDs of one pair are separated by `:`. + llvm::SmallVector id_pair = absl::StrSplit(str, ':'); + + // Check for malformed IDs. + if (id_pair.size() != 2) return failure(); + if (id_pair[0].empty() || id_pair[1].empty()) return failure(); + + auto is_digit = [](char c) { return absl::ascii_isdigit(c); }; + const std::string& group_id = id_pair[0]; + if (group_id[0] != 'p' && group_id[0] != 'r') return failure(); + if (!std::all_of(std::next(group_id.begin()), group_id.end(), is_digit)) { + return failure(); + } + const std::string& branch_id = id_pair[1]; + if (!std::all_of(branch_id.begin(), branch_id.end(), is_digit)) { + return failure(); + } + id_pairs.push_back(std::make_pair(id_pair[0], id_pair[1])); + } + return success(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..4dacde0c7754df842bb8206f1f7a7ca5ed31947f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h @@ -0,0 +1,206 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ATTRIBUTE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ATTRIBUTE_UTILS_H_ + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" + +namespace mlir { +namespace TF { + +// TODO(b/229028654) Use definitions from tf2xla_defs.h directly. We currently +// don't do this to avoid explicit casts (implicit conversion from +// `absl::string_view` to `llvm::StringRef` is not supported until C++17). + +// Whether soft placement is allowed. If true, the marked node is eligible for +// outside compilation. +inline constexpr llvm::StringRef kAllowSoftPlacementAttr = + "allow_soft_placement"; + +// Marks a node for XLA compilation. The attribute value indicates the +// compilation device type. +inline constexpr llvm::StringRef kCompileDeviceTypeAttr = + "_xla_compile_device_type"; +// The attribute value speicifes the preferred outlined function name in +// ClusterOutliningPass. +inline constexpr llvm::StringRef kClusterOutlinedFunctionNameAttr = + "_cluster_outlined_function_name"; +// Marks a node for replication. The attribute value indicates the replication +// metadata op. +inline constexpr llvm::StringRef kReplicationInfoAttr = "_replication_info"; +// Marks a node for XLA-TPU compilation. The attribute value indicates the +// associated compilation cluster and replication metadata op. +inline constexpr llvm::StringRef kTpuReplicateAttr = "_tpu_replicate"; +// Device types. +inline constexpr llvm::StringRef kTpuDevice = "TPU"; +// _xla_outside_compilation +inline constexpr llvm::StringRef kXlaOutsideCompilationAttr = + "_xla_outside_compilation"; +// device attr +inline constexpr llvm::StringRef kDeviceAttr = "device"; +// Function attribute to signal that a function should be skipped from TPU +// island outlining. The attribute is set in +// `TpuV1BridgeExecutorIslandCoarsening` and removed in the subsequent +// `TPUBridgeExecutorIslandOutlining` pass. +inline constexpr llvm::StringRef kSkipIslandOutlining = + "_skip_island_outlining"; +// Function attribute to signal which argument contains bounded dynamic +// dimension. +inline constexpr llvm::StringRef kDynamicArgIndexAttr = "_dynamic_arg_index"; + +// This string attribute encodes parallel execution groups and their associated +// branches. It has the following format: +// `_parallel_execution_ids= group1:branch1,group2:branch2,...` +// For example, if we have IR as follows: +// +// tf_executor.island wraps "tf.OpA" +// tf_executor.island { +// "tf_device.replicate" {n = 2} { +// "tf.OpB" +// "tf_device.parallel_execute"() ({ +// "tf.OpC" +// }, { +// "tf.OpD" +// }) +// } +// +// The above IR will be flattened after `ReplicateToIslandPass` and +// `ParallelExecuteToIslandsPass` as follows: +// +// tf_executor.island wraps "tf.OpA" +// tf_executor.island {_parallel_execution_ids=r0:0} wraps "tf.OpB" +// tf_executor.island {_parallel_execution_ids=r0:0,p0:0} wraps "tf.OpC" +// tf_executor.island {_parallel_execution_ids=r0:0,p0:1} wraps "tf.OpD" +// tf_executor.island {_parallel_execution_ids=r0:1} wraps "tf.OpB" +// tf_executor.island {_parallel_execution_ids=r0:1,p0:0} wraps "tf.OpC" +// tf_executor.island {_parallel_execution_ids=r0:1,p0:1} wraps "tf.OpD" +// +// "tf.OpA" will not have `_parallel_execution_ids` attr, +// means it does not belong to any parallel execution groups. +// First instance of "tf.OpB" after flattening will have +// `_parallel_execution_ids = "r0:0"`, +// which represents the first branch of replicate group 0. +// Second instance of "tf.OpB" after flattening will have +// `_parallel_execution_ids = "r0:1"` +// which represents the second branch of replicate group 0. +// First instance of "tf.OpC" after flattening will have +// `_parallel_execution_ids = "r0:0,p0:0"` +// which represents the first branch of replicate group 0 and +// the first branch of parallel group 0. +// Second instance of "tf.OpC" after flattening will have +// `_parallel_execution_ids = "r0:1,p0:0"` +// which represents the second branch of replicate group 0 and +// the first branch of parallel group 0. +// First instance of "tf.OpD" after flattening will have +// `_parallel_execution_ids = "r0:0,p0:1"` +// which represents the first branch of replicate group 0 and +// the second branch of parallel group 0. +// Second instance of "tf.OpD" after flattening will have +// `_parallel_execution_ids = "r0:1,p0:1"` +// which represents the second branch of replicate group 0 and +// the second branch of parallel group 0. +inline constexpr llvm::StringRef kParallelExecAnnotation = + "_parallel_execution_ids"; + +// Logging + +// Name of component for error logging. This name is fixed and required to +// enable logging. +inline const char kBridgeComponent[] = "TFXLABridge"; +inline const char kMlirPh1BridgeCounterReplicated[] = "replicated"; +inline const char kMlirPh1BridgeCounterNonReplicated[] = "nonreplicated"; +inline const char kMlirPh1BridgeCounterV1[] = "v1"; +inline const char kMlirPh1BridgeCounterV2[] = "v2"; +inline const char kMlirPh1BridgeCounterTpu[] = "tpu"; +inline const char kMlirPh1BridgeCounterNonTpu[] = "cpu/gpu"; +inline const char kXlaOutsideCompilation[] = "_xla_outside_compilation"; + +// Copies attributes that satisfy the given predicate from `from` to `to`. +template +void CopyAttributes(Operation *from, Operation *to, Predicate P) { + for (const NamedAttribute &attr : from->getAttrs()) + if (P(attr)) to->setAttr(attr.getName(), attr.getValue()); +} + +// Copies attributes whose name begins with an _ from `from` to `to`. +inline void CopyUnderscoredAttributes(Operation *from, Operation *to) { + CopyAttributes(from, to, [](const NamedAttribute &attr) { + return attr.getName().strref().front() == '_'; + }); +} + +// Copies outside compilation attribute from `from` to `to`. +inline void CopyXlaOutsideCompilationAttributes(Operation *from, + Operation *to) { + CopyAttributes(from, to, [](const NamedAttribute &attr) { + return attr.getName().strref() == kXlaOutsideCompilationAttr; + }); +} + +// Copies attributes that are either `device` or whose name begins with an _ +// from `from` to `to`. +// TODO(b/158769932): This should be a general feature instead post some policy +// discussion. +inline void CopyDeviceAndUnderscoredAttributes(Operation *from, Operation *to) { + auto device = mlir::StringAttr::get(from->getContext(), "device"); + CopyAttributes(from, to, [&device](const NamedAttribute &attr) { + return attr.getName().strref().front() == '_' || attr.getName() == device; + }); +} + +// Forward declare these passthrough ops. +// TODO(jpienaar): Remove these and use trait instead. +class IdentityOp; +class IdentityNOp; + +// Returns if a value corresponds to a constant, returns the matched constant +// as an attribute. +template +bool GetValueAsConstant(Value val, AttrT &attr) { + while (auto result = mlir::dyn_cast(val)) { + Operation *op = result.getOwner(); + if (!isa(op) && !isa(op)) break; + val = op->getOperand(result.getResultNumber()); + } + return matchPattern(val, m_Constant(&attr)); +} + +// Checks if both compilation and replication attributes are present in the +// operation, and if their values are valid. +LogicalResult HasValidCompilationAndReplicationAttributes(Operation &op); + +// Checks if the device attribute is valid. +LogicalResult IsValidDeviceTypeOrEmpty(StringAttr attr); + +using ParallelExecutionIdPairs = + llvm::SmallVector, 8>; +// Parses the parallel execution attribute for `op` and fills `id_pairs` with +// the corresponding (group ID,branch ID) pairs. +// Returns `failure` if the attribute is malformed. +LogicalResult ParseParallelExecutionIds(Operation *op, + ParallelExecutionIdPairs &id_pairs); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ATTRIBUTE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.cc new file mode 100644 index 0000000000000000000000000000000000000000..8abf5c410cf71acf3176e63d28fe3f1269499f6f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.cc @@ -0,0 +1,156 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h" + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_split.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/core/platform/env.h" + +namespace tensorflow { + +constexpr char kPassFilterEnvVar[] = "MLIR_BRIDGE_LOG_PASS_FILTER"; +constexpr char kStringFilterEnvVar[] = "MLIR_BRIDGE_LOG_STRING_FILTER"; +constexpr char kEnableOnlyTopLevelPassesEnvVar[] = + "MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES"; + +// Counter is used as a prefix for filenames. +static std::atomic log_counter(0); + +BridgeLoggerConfig::BridgeLoggerConfig(bool print_module_scope, + bool print_after_only_on_change, + mlir::OpPrintingFlags op_printing_flags) + : mlir::PassManager::IRPrinterConfig( + print_module_scope, print_after_only_on_change, + /*printAfterOnlyOnFailure=*/false, op_printing_flags), + pass_filter_(GetFilter(kPassFilterEnvVar)), + string_filter_(GetFilter(kStringFilterEnvVar)) { + setenv(/*name=*/kEnableOnlyTopLevelPassesEnvVar, + /*value=*/"false", /*overwrite=*/0); +} + +// Logs op to file with name of format +// `_mlir_bridge__.mlir`. +inline static void Log(BridgeLoggerConfig::PrintCallbackFn print_callback, + mlir::Pass* pass, mlir::Operation* op, + llvm::StringRef file_suffix) { + std::string pass_name = pass->getName().str(); + + // Add 4-digit counter as prefix so the order of the passes is obvious. + std::string name = llvm::formatv("{0,0+4}_mlir_bridge_{1}_{2}", log_counter++, + pass_name, file_suffix); + + std::unique_ptr os; + std::string filepath; + if (CreateFileForDumping(name, &os, &filepath).ok()) { + print_callback(*os); + LOG(INFO) << "Dumped MLIR module to " << filepath; + } +} + +void BridgeLoggerConfig::printBeforeIfEnabled(mlir::Pass* pass, + mlir::Operation* op, + PrintCallbackFn print_callback) { + if (ShouldPrint(pass, op)) Log(print_callback, pass, op, "before"); +} + +void BridgeLoggerConfig::printAfterIfEnabled(mlir::Pass* pass, + mlir::Operation* op, + PrintCallbackFn print_callback) { + if (ShouldPrint(pass, op)) Log(print_callback, pass, op, "after"); +} + +std::vector BridgeLoggerConfig::GetFilter( + const std::string& env_var) { + std::vector filter; + const char* filter_str = getenv(env_var.c_str()); + if (filter_str) { + filter = absl::StrSplit(filter_str, ';', absl::SkipWhitespace()); + } + return filter; +} + +bool BridgeLoggerConfig::ShouldOnlyDumpTopLevelPasses() { + const char* env_var = getenv(kEnableOnlyTopLevelPassesEnvVar); + std::string value = absl::AsciiStrToLower(env_var); + // Return true if value is "1" or "true"; otherwise, false. + return value == "1" || value == "true"; +} + +bool BridgeLoggerConfig::MatchesFilter(const std::string& str, + const std::vector& filter, + bool exact_match) { + if (filter.empty()) return true; + for (const std::string& filter_str : filter) { + if (str == filter_str) return true; + if (!exact_match && str.find(filter_str) != std::string::npos) return true; + } + return false; +} + +bool BridgeLoggerConfig::ShouldPrint(mlir::Pass* pass, mlir::Operation* op) { + // Check pass filter first since it's cheaper. + std::string pass_name = pass->getName().str(); + if (!MatchesFilter(pass_name, pass_filter_, /*exact_match=*/true)) { + // No string in filter matches pass name. + VLOG(1) << "Not logging invocation of pass `" << pass_name + << "` because the pass name does not match any string in " + "`MLIR_BRIDGE_LOG_PASS_FILTER`"; + return false; + } + if (ShouldOnlyDumpTopLevelPasses()) { + // Check if the operation is the top-level module. + // Top-level module has no parent. + if (op->getParentOp() != nullptr) { + // This is a nested operation; do not print. + VLOG(1) << "Not logging invocation of pass `" << pass_name + << "` because it is applied to a nested operation, due to " + "`MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES` being set " + "to true"; + return false; + } + } + if (!string_filter_.empty()) { + std::string serialized_op; + llvm::raw_string_ostream os(serialized_op); + op->print(os); + if (!MatchesFilter(serialized_op, string_filter_, /*exact_match=*/false)) { + // No string in filter was found in serialized `op`. + VLOG(1) << "Not logging invocation of pass `" << pass_name + << "` because the serialized operation on which the pass is " + "invoked does not contain any of the strings specified by " + "MLIR_BRIDGE_LOG_STRING_FILTER"; + return false; + } + } + return true; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h new file mode 100644 index 0000000000000000000000000000000000000000..59c2988ba4974c47a32a73f74dc7d25592371dc9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h @@ -0,0 +1,100 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_BRIDGE_LOGGER_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_BRIDGE_LOGGER_H_ + +#include +#include + +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project + +namespace tensorflow { + +// Logger for logging MLIR modules before and after passes in MLIR TPU bridge. +// +// The IR logging can be restricted to a particular set of pass invocations via +// filters that are specified with the `MLIR_BRIDGE_LOG_PASS_FILTER` and +// `MLIR_BRIDGE_LOG_STRING_FILTER` environment variables. +// `MLIR_BRIDGE_LOG_PASS_FILTER` takes a semicolon-separated list of pass class +// names, `MLIR_BRIDGE_LOG_STRING_FILTER` takes a semicolon-separated list of +// strings, and IR is only dumped for a pass invocation if the pass name exactly +// matches any of the provided pass names and if the serialized operation on +// which the pass is invoked contains any of the specified strings as a +// substring. An empty list is interpreted as no restriction. The string filter +// can be handy e.g. if one is only interested in a certain function or when +// checking where a certain attribute gets lost. Note that we use a semicolon +// instead of comma as the separator to allow strings that contain commas (which +// frequently appear in MLIR). The strings can contain any characters (including +// spaces) except semicolons. +// +// Example: Setting the environment variables +// `MLIR_BRIDGE_LOG_PASS_FILTER="LegalizeTF;Canonicalizer"` and +// `MLIR_BRIDGE_LOG_STRING_FILTER="my_string"` will dump IR only for invocations +// of `LegalizeTF` and `Canonicalizer` where the string `my_string` is contained +// in the serialized operation on which the pass is invoked. For verbose log +// level >= 1, `bridge_logger.cc` prints details about pass invocations for +// which the IR dumping was skipped because of a filter. +class BridgeLoggerConfig : public mlir::PassManager::IRPrinterConfig { + public: + explicit BridgeLoggerConfig( + bool print_module_scope = false, bool print_after_only_on_change = true, + mlir::OpPrintingFlags op_printing_flags = mlir::OpPrintingFlags()); + + // A hook that may be overridden by a derived config that checks if the IR + // of 'operation' should be dumped *before* the pass 'pass' has been + // executed. If the IR should be dumped, 'print_callback' should be invoked + // with the stream to dump into. + void printBeforeIfEnabled(mlir::Pass* pass, mlir::Operation* op, + PrintCallbackFn print_callback) override; + + // A hook that may be overridden by a derived config that checks if the IR + // of 'operation' should be dumped *after* the pass 'pass' has been + // executed. If the IR should be dumped, 'print_callback' should be invoked + // with the stream to dump into. + void printAfterIfEnabled(mlir::Pass* pass, mlir::Operation* op, + PrintCallbackFn print_callback) override; + + // Returns `true` iff we should log IR for given `pass` and `op`. + // Note: Visibility of this function is public for use in unit testing. + bool ShouldPrint(mlir::Pass* pass, mlir::Operation* op); + + private: + // Get `filter` encoded by environment variable `env_var`. + static std::vector GetFilter(const std::string& env_var); + // Returns `true` iff any of the strings in `filter` matches `str`, either + // exactly or as a substring, depending on `exact_match`. + static bool MatchesFilter(const std::string& str, + const std::vector& filter, + bool exact_match); + // Determines whether only top-level passes should be dumped. + // Returns true unless the environment variable is set to "0" or "false". + static bool ShouldOnlyDumpTopLevelPasses(); + + // Only log pass invocations whose pass name exactly matches any string in + // `pass_filter_` (or when `pass_filter_` is empty). + const std::vector pass_filter_; + // Only log pass invocations where the serialized operation on which the pass + // is invoked contains any of the specified strings as a substring (or when + // `string_filter_` is empty). + const std::vector string_filter_; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_BRIDGE_LOGGER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..d9718378a164fc237161f45756d901b23222d77f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/bridge_logger_test.cc @@ -0,0 +1,422 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h" + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +// Define test modules that are deserialized to module ops. +static const char *const module_with_add = + R"(module { +func.func @main(%arg0: tensor<3x4x5xf32>, %arg1: tensor<3x4x5xf32>) -> tensor<3x4x5xf32> { + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<3x4x5xf32>) -> tensor<3x4x5xf32> + func.return %0 : tensor<3x4x5xf32> +} +} +)"; + +static const char *const module_with_sub = + R"(module { +func.func @main(%arg0: tensor<7x8x9xi8>, %arg1: tensor<7x8x9xi8>) -> tensor<7x8x9xi8> { + %0 = "tf.Sub"(%arg0, %arg1) : (tensor<7x8x9xi8>, tensor<7x8x9xi8>) -> tensor<7x8x9xi8> + func.return %0 : tensor<7x8x9xi8> +} +} +)"; + +void UnsetEnvironmentVariables() { + unsetenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER"); + unsetenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER"); + unsetenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES"); +} + +class BridgeLoggerFilters : public ::testing::Test { + protected: + void SetUp() override { UnsetEnvironmentVariables(); } + + mlir::MLIRContext CreateMlirContext() { + mlir::DialectRegistry mlir_registry; + mlir::RegisterAllTensorFlowDialects(mlir_registry); + return mlir::MLIRContext(mlir_registry); + } + + mlir::func::FuncOp GetFuncOp(mlir::ModuleOp module_op) { + auto func_ops = module_op.getOps(); + EXPECT_FALSE(func_ops.empty()); + return *func_ops.begin(); + } +}; + +// Test pass filter. +TEST_F(BridgeLoggerFilters, TestPassFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr partitioning_pass = + mlir::TFTPU::CreateTPUResourceReadsWritesPartitioningPass(); + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + std::unique_ptr inliner_pass = mlir::createInlinerPass(); + + // partitioning_pass and shape_inference_pass should match the filter, + // inliner_pass should not. + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/ + "TPUResourceReadsWritesPartitioningPass;TensorFlowShapeInferencePass", + /*overwrite=*/1); + BridgeLoggerConfig logger_config; + EXPECT_TRUE(logger_config.ShouldPrint(partitioning_pass.get(), + mlir_module_with_add.get())); + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + EXPECT_FALSE(logger_config.ShouldPrint(inliner_pass.get(), + mlir_module_with_add.get())); +} + +// Test string filter. +TEST_F(BridgeLoggerFilters, TestStringFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add, mlir_module_with_sub; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + TF_ASSERT_OK(DeserializeMlirModule(module_with_sub, &mlir_context, + &mlir_module_with_sub)); + // The pass is not relevant for this test since we don't define a pass filter. + std::unique_ptr dummy_pass = + mlir::TF::CreateTFShapeInferencePass(); + + // One string appears in both modules and the other one not. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", + /*value=*/"func @main(%arg0: tensor;XXX", /*overwrite=*/1); + BridgeLoggerConfig logger_config1; + EXPECT_TRUE( + logger_config1.ShouldPrint(dummy_pass.get(), mlir_module_with_add.get())); + EXPECT_TRUE( + logger_config1.ShouldPrint(dummy_pass.get(), mlir_module_with_sub.get())); + + // Both strings do not appear in any module. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", + /*value=*/"func @main(%arg0:tensor;XXX", /*overwrite=*/1); + BridgeLoggerConfig logger_config2; + EXPECT_FALSE( + logger_config2.ShouldPrint(dummy_pass.get(), mlir_module_with_add.get())); + EXPECT_FALSE( + logger_config2.ShouldPrint(dummy_pass.get(), mlir_module_with_sub.get())); + + // String appears in one module but not in the other. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", + /*value=*/"\"tf.AddV2\"(%arg0, %arg1) : (tensor<3x4x5xf32>", + /*overwrite=*/1); + BridgeLoggerConfig logger_config3; + EXPECT_TRUE( + logger_config3.ShouldPrint(dummy_pass.get(), mlir_module_with_add.get())); + EXPECT_FALSE( + logger_config3.ShouldPrint(dummy_pass.get(), mlir_module_with_sub.get())); +} + +// Test enable only top level passes filter. +TEST_F(BridgeLoggerFilters, TestEnableOnlyTopLevelPassesFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + + BridgeLoggerConfig logger_config; + // ShouldPrint returns true for the top-level module operation. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + // Find the nested function operation within the module. + mlir::func::FuncOp func_op = GetFuncOp(mlir_module_with_add.get()); + // ShouldPrint returns true for the nested function operation. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), func_op)); + + // Set the environment variable to enable only top-level passes. + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_filter; + // ShouldPrint returns false for the nested function operation. + EXPECT_FALSE( + logger_config_filter.ShouldPrint(shape_inference_pass.get(), func_op)); +} + +// Additional tests for various possible values of +// MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES. +TEST_F(BridgeLoggerFilters, TestEnableOnlyTopLevelPassesEnvVarValues) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + + mlir::ModuleOp module_op = *mlir_module_with_add; + // Find the nested function operation within the module. + mlir::func::FuncOp func_op = GetFuncOp(module_op); + + // Test with environment variable set to "FALSE". + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", + /*value=*/"FALSE", /*overwrite=*/1); + BridgeLoggerConfig logger_config_false; + // ShouldPrint should return true for top-level operation. + EXPECT_TRUE(logger_config_false.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + // ShouldPrint should return true for nested function. + EXPECT_TRUE( + logger_config_false.ShouldPrint(shape_inference_pass.get(), func_op)); + + // Test with environment variable unset (default behavior). + unsetenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES"); + BridgeLoggerConfig logger_config_default; + // ShouldPrint should return true for top-level operation. + EXPECT_TRUE(logger_config_default.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + // ShouldPrint should return true for nested function since default + // is disabled. + EXPECT_TRUE( + logger_config_default.ShouldPrint(shape_inference_pass.get(), func_op)); + + // Test with environment variable set to "1". + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_one; + // ShouldPrint should return false for nested function since filter + // is enabled. + EXPECT_FALSE( + logger_config_one.ShouldPrint(shape_inference_pass.get(), func_op)); +} + +// Test combinations of pass filter and string filter. +TEST_F(BridgeLoggerFilters, TestPassFilterAndStringFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + + // String filter is matched but pass filter is not. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", + /*value=*/ + "(tensor<3x4x5xf32>, tensor<3x4x5xf32>) -> " + "tensor<3x4x5xf32>", + /*overwrite=*/1); + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/"ensorFlowShapeInferencePass", /*overwrite=*/1); + BridgeLoggerConfig logger_config1; + EXPECT_FALSE(logger_config1.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + + // Pass filter is matched but string filter is not. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", /*value=*/"XXX", 1); + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/"TensorFlowShapeInferencePass", /*overwrite=*/1); + BridgeLoggerConfig logger_config2; + EXPECT_FALSE(logger_config2.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + + // Both filters are matched. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", + /*value=*/ + "(tensor<3x4x5xf32>, tensor<3x4x5xf32>) -> " + "tensor<3x4x5xf32>", + /*overwrite=*/1); + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/"TensorFlowShapeInferencePass", /*overwrite=*/1); + BridgeLoggerConfig logger_config3; + EXPECT_TRUE(logger_config3.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); +} + +// Test combinations of pass filter and enable only top level passes filter. +TEST_F(BridgeLoggerFilters, TestPassFilterAndEnableOnlyTopLevelPassesFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_sub; + TF_ASSERT_OK(DeserializeMlirModule(module_with_sub, &mlir_context, + &mlir_module_with_sub)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + std::unique_ptr inliner_pass = mlir::createInlinerPass(); + + // Find the nested function operation within the module. + mlir::func::FuncOp func_op = GetFuncOp(mlir_module_with_sub.get()); + + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/"TensorFlowShapeInferencePass", /*overwrite=*/1); + BridgeLoggerConfig logger_config; + // ShouldPrint should return true for top-level operation with matching pass + // filter. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_sub.get())); + // ShouldPrint should return true for nested operation when + // enable_only_top_level_passes_ is false. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), func_op)); + // ShouldPrint should return false for pass not matching the pass filter. + EXPECT_FALSE(logger_config.ShouldPrint(inliner_pass.get(), + mlir_module_with_sub.get())); + + // Set the environment variable to enable only top-level passes. + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_filter; + // ShouldPrint should return false for nested operation + EXPECT_FALSE( + logger_config_filter.ShouldPrint(shape_inference_pass.get(), func_op)); +} + +// Test combinations of string filter and enable only top level passes filter. +TEST_F(BridgeLoggerFilters, TestStringFilterAndEnableOnlyTopLevelPassesFilter) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + + // Find the nested function operation within the module. + mlir::func::FuncOp func_op = GetFuncOp(mlir_module_with_add.get()); + + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", /*value=*/"tf.AddV2", + /*overwrite=*/1); + BridgeLoggerConfig logger_config; + // ShouldPrint should return true for top-level operation containing + // "tf.AddV2". + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + // ShouldPrint should return true for nested operation since + // enable_only_top_level_passes_ is false. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), func_op)); + + // Set the environment variable to enable only top-level passes. + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_filter; + // ShouldPrint should return false for nested operation since string + // filter matches but enable_only_top_level_passes_ is true. + EXPECT_FALSE( + logger_config_filter.ShouldPrint(shape_inference_pass.get(), func_op)); + + // Change string filter to not match any operation. + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", /*value=*/"NonExistentOp", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_no_match; + // ShouldPrint should return false since string filter does not match. + EXPECT_FALSE(logger_config_no_match.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); +} + +// Test combinations where all filters are set but none match. +TEST_F(BridgeLoggerFilters, TestAllFiltersNoMatch) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_sub; + TF_ASSERT_OK(DeserializeMlirModule(module_with_sub, &mlir_context, + &mlir_module_with_sub)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + + // Set pass filter to not match any pass + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", /*value=*/"NonExistentPass", + /*overwrite=*/1); + // Set string filter to not match any string + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", /*value=*/"NonExistentOp", + /*overwrite=*/1); + BridgeLoggerConfig logger_config; + // ShouldPrint should return false since none of the filters match. + EXPECT_FALSE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_sub.get())); + + // Set the environment variable to enable only top-level passes. + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_filter; + // ShouldPrint should still return false since pass and string filters do not + // match. + EXPECT_FALSE(logger_config_filter.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_sub.get())); +} + +// Test combinations of all three filters. +TEST_F(BridgeLoggerFilters, TestAllFiltersCombination) { + mlir::MLIRContext mlir_context = CreateMlirContext(); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + std::unique_ptr inliner_pass = mlir::createInlinerPass(); + + // Find the nested function operation within the module. + mlir::func::FuncOp func_op = GetFuncOp(mlir_module_with_add.get()); + + // Set all three filters. + setenv(/*name=*/"MLIR_BRIDGE_LOG_PASS_FILTER", + /*value=*/"TensorFlowShapeInferencePass", /*overwrite=*/1); + setenv(/*name=*/"MLIR_BRIDGE_LOG_STRING_FILTER", /*value=*/"tf.AddV2", + /*overwrite=*/1); + BridgeLoggerConfig logger_config; + // ShouldPrint should return true if all filters pass and operation is + // top-level. + EXPECT_TRUE(logger_config.ShouldPrint(shape_inference_pass.get(), + mlir_module_with_add.get())); + + // ShouldPrint should return false if enable_only_top_level_passes_ is + // true and operation is nested. + setenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES", /*value=*/"1", + /*overwrite=*/1); + BridgeLoggerConfig logger_config_filter; + EXPECT_FALSE( + logger_config_filter.ShouldPrint(shape_inference_pass.get(), func_op)); + // Change to a pass that does not match the pass filter. + EXPECT_FALSE(logger_config_filter.ShouldPrint(inliner_pass.get(), + mlir_module_with_add.get())); + // Set the environment variable to disable only top-level passes. + unsetenv(/*name=*/"MLIR_BRIDGE_LOG_ENABLE_ONLY_TOP_LEVEL_PASSES"); + BridgeLoggerConfig logger_config_no_filter; + // ShouldPrint should return true for nested operation since + // enable_only_top_level_passes_ is false. + EXPECT_TRUE( + logger_config_no_filter.ShouldPrint(shape_inference_pass.get(), func_op)); + // Change to a pass that does not match the pass filter. + EXPECT_FALSE(logger_config_no_filter.ShouldPrint(inliner_pass.get(), + mlir_module_with_add.get())); +} +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..af13a06e0f80f9fdcadecd67212bd47ce5749d67 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.cc @@ -0,0 +1,75 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { + +std::vector GetEntryFunctionAttributeNames() { + return {"tf.entry_function", + tf_saved_model::kTfSavedModelInitializerTypeAttr}; +} + +bool IsEntryFunction(func::FuncOp func) { + for (const auto &attr : GetEntryFunctionAttributeNames()) { + if (func->hasAttr(attr)) { + return true; + } + } + return false; +} + +llvm::SmallVector GetEntryFunctions(ModuleOp module) { + llvm::SmallVector entry_funcs; + module.walk([&](func::FuncOp func) { + // A model may have multiple graphs, with each graph having its own entry. + // When a graph is imported to MLIR, `tf.entry_function` will be added to + // each entry function. The one exception are initializer functions, which + // have `tf_saved_model.initializer_type` instead. + if (IsEntryFunction(func)) { + entry_funcs.push_back(func); + } + }); + return entry_funcs; +} + +LogicalResult GetCallees(SymbolUserOpInterface op, SymbolTable &symtab, + llvm::SmallVector &callees) { + for (auto attr : op->getAttrs()) { + auto sym = mlir::dyn_cast(attr.getValue()); + if (!sym) continue; + auto callee = symtab.lookup(sym.getRootReference()); + if (!callee) { + // This is not expected to happen in practice. + return op->emitError() + << "Cannot find function " << sym.getRootReference(); + } + callees.push_back(callee); + } + return success(); +} + +bool HasSingleBlock(func::FuncOp func) { + return func->getNumRegions() == 1 && func.getBody().hasOneBlock(); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h new file mode 100644 index 0000000000000000000000000000000000000000..e788df685f65775731a659b5738301a00f0764a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h @@ -0,0 +1,119 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CALL_GRAPH_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CALL_GRAPH_UTIL_H_ + +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { + +// Return a list of attribute names that indicates an entry function. +std::vector GetEntryFunctionAttributeNames(); + +// Check if a function is an entry in an MLIR module. +bool IsEntryFunction(func::FuncOp func); + +// Get all the entry functions in an MLIR module. +llvm::SmallVector GetEntryFunctions(ModuleOp module); + +// Get all the functions referenced in a symber user op and save them in +// `callees`. +LogicalResult GetCallees(SymbolUserOpInterface op, SymbolTable &symtab, + llvm::SmallVector &callees); + +// Find the first op with any of the specified types on each path rooted at the +// `root` node in a tree. Additional checks can be applied via `predicate`. The +// results are stored in `ops`. +template +LogicalResult GetFirstOpsOfType( + func::FuncOp root, SymbolTable &symtab, + const std::function &predicate, + llvm::SmallVector &ops) { + std::stack worklist; + worklist.push(root); + while (!worklist.empty()) { + func::FuncOp u = worklist.top(); + worklist.pop(); + auto result = u.walk([&](SymbolUserOpInterface op) { + if (llvm::isa(op) && (!predicate || predicate(op))) { + ops.push_back(op); + return WalkResult::advance(); + } + llvm::SmallVector callees; + if (GetCallees(op, symtab, callees).failed()) { + return WalkResult::interrupt(); + } + for (auto callee : callees) { + worklist.push(callee); + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return failure(); + } + return success(); +} + +// Find the nodes with any of the specified types on the tree rooted at `root` +// node. Additional checks can be applied via `predicate`. The search skips +// the current path if a node with the specified types fails the check, and +// continues on the next path. The passing ops are stored in `hits`, while the +// first failing on on each path is stored in `first_misses`. +template +LogicalResult GetOpsOfTypeUntilMiss( + func::FuncOp root, SymbolTable &symtab, + const std::function &predicate, + llvm::SmallVector &hits, + llvm::SmallVector &first_misses) { + std::stack worklist; + worklist.push(root); + while (!worklist.empty()) { + func::FuncOp u = worklist.top(); + worklist.pop(); + auto result = u.walk([&](SymbolUserOpInterface op) { + if (llvm::isa(op)) { + if (!predicate || predicate(op)) { + hits.push_back(op); + } else { + first_misses.push_back(op); + return WalkResult::advance(); + } + } + llvm::SmallVector callees; + if (GetCallees(op, symtab, callees).failed()) { + return WalkResult::interrupt(); + } + for (auto callee : callees) { + worklist.push(callee); + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return failure(); + } + return success(); +} + +// Check if a function has one region and one block only. +bool HasSingleBlock(func::FuncOp func); + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CALL_GRAPH_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..96b7263c40f80d6aefa6cdba03e5b7a21bb596b4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/call_graph_util_test.cc @@ -0,0 +1,281 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h" + +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +TEST(CallGraphUtilTest, GetEntryFunctionAttributeNames) { + auto attr_names = mlir::GetEntryFunctionAttributeNames(); + EXPECT_EQ(attr_names.size(), 2); + EXPECT_EQ(attr_names[0], "tf.entry_function"); + EXPECT_EQ(attr_names[1], + mlir::tf_saved_model::kTfSavedModelInitializerTypeAttr); +} + +TEST(CallGraphUtilTest, GetEntryFunctions) { + const char *const code = R"mlir( +func.func @entry_func_1(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @entry_func_2(%arg0: tensor) -> tensor attributes {tf_saved_model.initializer_type = ""} { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} +)mlir"; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + auto entry_funcs = GetEntryFunctions(*module); + EXPECT_EQ(entry_funcs.size(), 2); + EXPECT_EQ(entry_funcs[0].getSymName(), "entry_func_1"); + EXPECT_EQ(entry_funcs[1].getSymName(), "entry_func_2"); +} + +TEST(CallGraphUtilTest, GetCallees) { + const char *const code = R"mlir( +func.func @entry_func(%arg0: tensor) -> tensor attributes {tf_saved_model.initializer_type = ""} { + %0 = "tf.While"(%arg0) {cond = @while_cond_func, body = @while_body_func, is_stateless = true} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @while_cond_func(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + func.return %0 : tensor +} + +func.func @while_body_func(%arg0: tensor) -> (tensor) { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + func.return %0 : tensor +} + + +)mlir"; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + mlir::SymbolTable symtab(*module); + llvm::SmallVector callees; + module->walk([&](mlir::SymbolUserOpInterface op) { + auto result = GetCallees(op, symtab, callees).succeeded(); + ASSERT_TRUE(result); + EXPECT_EQ(callees.size(), 2); + EXPECT_EQ(callees[0].getSymName(), "while_body_func"); + EXPECT_EQ(callees[1].getSymName(), "while_cond_func"); + }); +} + +TEST(CallGraphUtilTest, GetFirstOpsOfType) { + const char *const code = R"mlir( +func.func @entry_func(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.While"(%arg0) {cond = @while_cond_func, body = @while_body_func, is_stateless = true} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @while_cond_func(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @while_body_func +func.func @while_body_func(%arg0: tensor) -> (tensor) { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @outer_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @outer_stateful_pcall_func(%arg0: tensor) -> (tensor) { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @inner_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @inner_stateful_pcall_func(%arg0: tensor) -> tensor { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} +)mlir"; + auto has_compile_device_type = [](mlir::SymbolUserOpInterface op) { + return op->hasAttr(tensorflow::kCompileDeviceTypeAttr); + }; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + mlir::SymbolTable symtab(*module); + llvm::SmallVector entry_funcs = + GetEntryFunctions(*module); + EXPECT_EQ(entry_funcs.size(), 1); + EXPECT_EQ(entry_funcs[0].getSymName(), "entry_func"); + llvm::SmallVector outermost_pcall_ops; + auto result = + mlir::GetFirstOpsOfType( + entry_funcs[0], symtab, has_compile_device_type, outermost_pcall_ops) + .succeeded(); + ASSERT_TRUE(result); + EXPECT_EQ(outermost_pcall_ops.size(), 1); + auto func = + llvm::dyn_cast(outermost_pcall_ops[0]->getParentOp()); + ASSERT_TRUE(func); + EXPECT_EQ(func.getSymName(), "outer_stateful_pcall_func"); +} + +TEST(CallGraphUtilTest, GetOpsOfTypeUntilMiss) { + const char *const code = R"mlir( +func.func @entry_func(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.While"(%arg0) {cond = @while_cond_func, body = @while_body_func, is_stateless = true} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @while_cond_func(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func.func @while_body_func +func.func @while_body_func(%arg0: tensor) -> (tensor) { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @outer_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @outer_stateful_pcall_func(%arg0: tensor) -> (tensor) { + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @inner_stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @inner_stateful_pcall_func(%arg0: tensor) -> tensor { + %0 = "tf.StatefulPartitionedCall"(%arg0) {_xla_compile_device_type = "CPU", config = "", config_proto = "", device = "/device:CPU:0", executor_type = "", f = @func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} +)mlir"; + auto has_no_compile_device_type = [](mlir::SymbolUserOpInterface op) { + return !op->hasAttr(tensorflow::kCompileDeviceTypeAttr); + }; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + mlir::SymbolTable symtab(*module); + llvm::SmallVector entry_funcs = + GetEntryFunctions(*module); + EXPECT_EQ(entry_funcs.size(), 1); + EXPECT_EQ(entry_funcs[0].getSymName(), "entry_func"); + llvm::SmallVector noinline_pcall_ops, + outermost_pcall_ops; + auto result = + mlir::GetOpsOfTypeUntilMiss( + entry_funcs[0], symtab, has_no_compile_device_type, + /*hits*/ noinline_pcall_ops, /*first_misses*/ outermost_pcall_ops) + .succeeded(); + ASSERT_TRUE(result); + EXPECT_EQ(noinline_pcall_ops.size(), 2); + auto func = + llvm::dyn_cast(noinline_pcall_ops[0]->getParentOp()); + ASSERT_TRUE(func); + EXPECT_EQ(func.getSymName(), "while_body_func"); + func = + llvm::dyn_cast(noinline_pcall_ops[1]->getParentOp()); + ASSERT_TRUE(func); + EXPECT_EQ(func.getSymName(), "outer_stateful_pcall_func"); + + EXPECT_EQ(outermost_pcall_ops.size(), 1); + func = + llvm::dyn_cast(outermost_pcall_ops[0]->getParentOp()); + ASSERT_TRUE(func); + EXPECT_EQ(func.getSymName(), "inner_stateful_pcall_func"); +} + +TEST(CallGraphUtilTest, SingleBlockEntryFunction) { + const char *const code = R"mlir( +func.func @entry_func(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + func.return %arg0 : tensor +} +)mlir"; + + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + llvm::errs() << "module:\n"; + ASSERT_TRUE(module); + mlir::SymbolTable symtab(*module); + llvm::SmallVector entry_funcs = + GetEntryFunctions(*module); + EXPECT_EQ(entry_funcs.size(), 1); + EXPECT_EQ(entry_funcs[0].getSymName(), "entry_func"); + EXPECT_TRUE(HasSingleBlock(entry_funcs[0])); +} + +TEST(CallGraphUtilTest, MultipleBlocksEntryFunction) { + const char *const code = R"mlir( +func.func @entry_func(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + cf.br ^bb1 +^bb1: + func.return %arg0 : tensor +} +)mlir"; + + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + llvm::errs() << "module:\n"; + ASSERT_TRUE(module); + mlir::SymbolTable symtab(*module); + llvm::SmallVector entry_funcs = + GetEntryFunctions(*module); + EXPECT_EQ(entry_funcs.size(), 1); + EXPECT_EQ(entry_funcs[0].getSymName(), "entry_func"); + EXPECT_FALSE(HasSingleBlock(entry_funcs[0])); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..c5d8a94b93d33fed812436e070153303fe0d0691 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.cc @@ -0,0 +1,216 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h" + +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" + +namespace mlir::TF { + +namespace { + +// Exhaust search in the block to get all the ops that have data dependency of +// the cluster. +llvm::SetVector GetAllOpsDependOnCluster( + const Cluster& c, + const llvm::DenseMap& op_to_cluster_map) { + llvm::SetVector ops_depend_on_cluster; + for (Operation& op : *c.ops.front()->getBlock()) { + if (op.isBeforeInBlock(c.ops.front()) || c.ops.contains(&op)) { + continue; + } + // Gets the live in values of the `op` + llvm::SetVector live_ins(op.operand_begin(), op.operand_end()); + getUsedValuesDefinedAbove(op.getRegions(), live_ins); + // Inserts if any of the `live_ins` depends on the ops in the cluster. + if (llvm::any_of(live_ins, [&](Value value) { + Operation* defining_op = value.getDefiningOp(); + if (!defining_op) { + return false; + } + return c.ops.contains(defining_op) || + ops_depend_on_cluster.contains(defining_op); + })) { + ops_depend_on_cluster.insert(&op); + } + } + // The data dependency of the cluster includes the union of ops' data + // dependency. So includes all the ops in the same cluster of the op in + // `ops_depend_on_cluster`. + llvm::SetVector same_cluster_ops_with_dependency( + ops_depend_on_cluster.begin(), ops_depend_on_cluster.end()); + for (Operation* op : ops_depend_on_cluster) { + Cluster* cluster = op_to_cluster_map.lookup(op); + if (cluster == nullptr) { + continue; + } + for (Operation* ops_in_same_cluster : cluster->ops) { + same_cluster_ops_with_dependency.insert(ops_in_same_cluster); + } + } + return same_cluster_ops_with_dependency; +} + +// An op can be merged into cluster if it satisfies both of the following +// conditions: +// +// * Merging the op into the cluster doesn't break the acyclic nature of the +// * graph. This means all of its operands don't have data dependency of the +// * cluster. +// * Merging the op into the cluster does not reorder control dependencies. +// +bool CanMergeIntoCluster( + const Cluster& c, Operation* to_merge, + const TF::SideEffectAnalysis::Info& side_effect_analysis, + std::function get_target, + const llvm::DenseMap& op_to_cluster_map) { + // If any of the op's control predecessors appears after the last op in the + // cluster, merging the op may cause control dependencies to be reordered. + // Hence, the op cannot be merged to the cluster in such a case. + const bool has_control_predecessors_after_cluster = + !side_effect_analysis + .DirectControlPredecessors( + to_merge, + [&c](Operation* pred) { + Operation* const last_c_op = c.ops.back(); + return last_c_op->getBlock() == pred->getBlock() && + last_c_op->isBeforeInBlock(pred); + }) + .empty(); + if (has_control_predecessors_after_cluster) { + return false; + } + + // We can merge the op into the cluster if doing so doesn't break the acyclic + // nature of the graph. In this way, we need to check if there is any + // dependency from the oprands of the op to the current cluster. + llvm::SetVector ops_depend_on_cluster = + GetAllOpsDependOnCluster(c, op_to_cluster_map); + return llvm::none_of(to_merge->getOperands(), [&](Value value) { + Operation* defining_op = value.getDefiningOp(); + return defining_op && ops_depend_on_cluster.contains(defining_op); + }); +} +} // namespace + +llvm::StringMap> BuildAllClusters( + Block& block, const TF::SideEffectAnalysis::Info& side_effect_analysis, + std::function get_target, + std::function is_ignored_op) { + // Iteratively find clusters of different targets within the `block`. + // Whenever we see an operation that is assigned to an accelerator target + // (ie. get_target(op) != ""), we try to merge it into the last cluster + // of same target. If that is infeasible (say because of violating + // def-before-use), create a new cluster with that operation and move on. + llvm::StringMap> all_clusters; + // Map from operation to the cluster that contains the operation. + llvm::DenseMap op_to_cluster_map; + + llvm::StringMap nearest_clusters; + for (Operation& op : llvm::make_early_inc_range(block)) { + if (is_ignored_op(&op)) { + continue; + } + std::string target_name = get_target(&op); + + // If no cluster of same target has been formed yet, create a new cluster + // with op alone. + auto it = nearest_clusters.find(target_name); + if (it == nearest_clusters.end()) { + SetVector new_cluster_op_set; + new_cluster_op_set.insert(&op); + nearest_clusters[target_name] = Cluster{new_cluster_op_set, target_name}; + op_to_cluster_map[&op] = &nearest_clusters[target_name]; + continue; + } + + // Check if it is legal to merge op into nearest cluster of same target. + // If positive, update cluster and move on to next operation. + Cluster& nearest_cluster = it->second; + if (CanMergeIntoCluster(nearest_cluster, &op, side_effect_analysis, + get_target, op_to_cluster_map)) { + nearest_cluster.ops.insert(&op); + op_to_cluster_map[&op] = &nearest_cluster; + continue; + } + + // If nearest cluster of same target can not absorb `op`, then that + // cluster needs to be finalized by inserting into the final cluster map + // that contains all operations in clusters. + all_clusters[target_name].push_back(nearest_cluster); + + // Create a new cluster to hold op alone and update nearest_clusters. + SetVector new_cluster_op_set; + new_cluster_op_set.insert(&op); + nearest_clusters[target_name] = Cluster{new_cluster_op_set, target_name}; + op_to_cluster_map[&op] = &nearest_clusters[target_name]; + } + + // At the end, there might be left-over found clusters that need to be + // built. + for (auto& target_cluster : nearest_clusters) { + all_clusters[target_cluster.first()].push_back(target_cluster.second); + } + + return all_clusters; +} + +void ReorderOpResultUses(mlir::Operation* cluster) { + mlir::Block* const cluster_block = cluster->getBlock(); + llvm::SetVector ops_to_reorder; + + llvm::SmallVector worklist; + llvm::append_range(worklist, cluster->getResults()); + + while (!worklist.empty()) { + mlir::Value value = worklist.back(); + worklist.pop_back(); + + for (mlir::Operation* const user : value.getUsers()) { + mlir::Operation* const op = cluster_block->findAncestorOpInBlock(*user); + if (op == nullptr || !op->isBeforeInBlock(cluster)) { + continue; + } + + if (ops_to_reorder.insert(op)) { + llvm::append_range(worklist, op->getResults()); + } + } + } + + llvm::SmallVector sorted = ops_to_reorder.takeVector(); + llvm::sort(sorted, [](mlir::Operation* lhs, mlir::Operation* rhs) { + return lhs->isBeforeInBlock(rhs); + }); + + for (mlir::Operation* const op : llvm::reverse(sorted)) { + op->moveAfter(cluster); + } +} + +} // namespace mlir::TF diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h new file mode 100644 index 0000000000000000000000000000000000000000..d41a9848f32c56d7c5c08cd3bac0d85667c62f5f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h @@ -0,0 +1,67 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CLUSTER_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CLUSTER_UTIL_H_ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" + +namespace mlir::TF { + +// Cluster structure captures all the operations that are assigned to same +// device and can form a legal strict cluster. +// Ops must follow same ordering in their parent block. We rely on this +// assumption to perform analysis. +struct Cluster { + llvm::SetVector ops; + std::string target; +}; + +// Builds the op clusters in the `block`. Ops are filtered by the function +// `get_target` that takes an op and returns the target name. `is_ignored_op` is +// a hook to ignore certain ops that are not included in any clusters. +llvm::StringMap> BuildAllClusters( + Block& block, const TF::SideEffectAnalysis::Info& side_effect_analysis, + std::function get_target, + std::function is_ignored_op); + +// Reorder all users of the given op's results to after the op. +// +// Since launch ops are inserted after the last op in the region, the region is +// guaranteed to dominate all live-in values. On the other hand, it is still +// possible that live-out values don't dominate the region. For example: +// +// ``` +// %0 = "tf.OpA"() +// %1 = "tf.OpB"(%0) +// %2 = "tf.OpC"(%0) +// ``` +// +// Assuming `tf.OpA` and `tf.OpC` are clustered together, the region will be +// inserted right after `tf.OpC`. The live-out `%0`, however, is used by +// `tf.OpB`, which won't dominate the region. This function reorders all users +// of the cluster op to be placed after the cluster op itself so that SSA +// dominance is preserved after cluster op creation. +void ReorderOpResultUses(mlir::Operation* cluster); + +} // namespace mlir::TF + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CLUSTER_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..0f6ff17eb8845d853f60c3b50eecccb0ec9f8744 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/cluster_util_test.cc @@ -0,0 +1,202 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/cluster_util.h" + +#include + +#include +#include +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/platform/errors.h" +#include "tsl/platform/statusor.h" + +namespace mlir::TF { + +namespace { + +constexpr StringRef kTestClusterName = "tpu0"; + +absl::StatusOr> GetMlirModuleFromString( + StringRef string, MLIRContext* context) { + DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +std::string GetDevice(Operation* op) { + auto device_attr = op->getAttrOfType("device"); + return device_attr ? device_attr.getValue().str() : ""; +} + +bool CanBeIgnoredInCluster(Operation* op) { + auto device_attr = op->getAttrOfType("device"); + return !device_attr || device_attr.getValue().empty(); +} + +llvm::StringMap> GetClusters(ModuleOp module) { + TF::SideEffectAnalysis side_effect_analysis(module); + auto main_func = module.lookupSymbol("main"); + const TF::SideEffectAnalysis::Info& info = + side_effect_analysis.GetAnalysisForFunc(main_func); + llvm::StringMap> clusters = BuildAllClusters( + main_func.front(), info, GetDevice, CanBeIgnoredInCluster); + return clusters; +} + +TEST(BuildClusters, TestSingleCluster) { + static const char* const module_with_single_cluster = + R"(module { +func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + %1 = "tf.B"(%0) {device = "tpu0"} : (tensor) -> tensor + %2 = "tf.C"(%0, %1) {device = "tpu0"} : (tensor, tensor) -> tensor + %3 = "tf.D"(%2) : (tensor) -> tensor + func.return %3 : tensor + } +} +)"; + + MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN( + OwningOpRef module, + GetMlirModuleFromString(module_with_single_cluster, &context)); + auto clusters = GetClusters(module.get()); + EXPECT_EQ(clusters.count(kTestClusterName), 1); + EXPECT_EQ(clusters.lookup(kTestClusterName).size(), 1); + EXPECT_EQ(clusters.lookup(kTestClusterName)[0].ops.size(), 2); +} + +TEST(BuildClusters, TestMultipleClusters) { + static const char* const module_with_two_clusters = + R"(module { +func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + %1 = "tf.B"(%0) {device = "tpu0"} : (tensor) -> tensor + %2 = "tf.C"(%0, %1) {device = "tpu0"} : (tensor, tensor) -> tensor + %3 = "tf.D"(%2) : (tensor) -> tensor + %4 = "tf.E"(%3) {device = "tpu0"} : (tensor) -> tensor + %5 = "tf.F"(%3, %4) {device = "tpu0"} : (tensor, tensor) -> tensor + func.return %5 : tensor + } +} +)"; + + MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN( + OwningOpRef module, + GetMlirModuleFromString(module_with_two_clusters, &context)); + auto clusters = GetClusters(module.get()); + EXPECT_EQ(clusters.count(kTestClusterName), 1); + EXPECT_EQ(clusters[kTestClusterName].size(), 2); + EXPECT_EQ(clusters[kTestClusterName][0].ops.size(), 2); + EXPECT_EQ(clusters[kTestClusterName][1].ops.size(), 2); +} + +TEST(BuildClusters, TestMultipleTargets) { + static const char* const module_with_two_clusters = + R"(module { +func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + %1 = "tf.B"(%0) {device = "tpu0"} : (tensor) -> tensor + %2 = "tf.C"(%0, %1) {device = "tpu0"} : (tensor, tensor) -> tensor + %3 = "tf.D"(%2) : (tensor) -> tensor + %4 = "tf.E"(%3) {device = "tpu1"} : (tensor) -> tensor + %5 = "tf.F"(%3, %4) {device = "tpu1"} : (tensor, tensor) -> tensor + func.return %5 : tensor + } +} +)"; + + MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN( + OwningOpRef module, + GetMlirModuleFromString(module_with_two_clusters, &context)); + auto clusters = GetClusters(module.get()); + constexpr StringRef kTarget0 = "tpu0"; + EXPECT_EQ(clusters.count(kTarget0), 1); + EXPECT_EQ(clusters[kTarget0].size(), 1); + EXPECT_EQ(clusters[kTarget0][0].ops.size(), 2); + + constexpr StringRef kTarget1 = "tpu1"; + EXPECT_EQ(clusters.count(kTarget1), 1); + EXPECT_EQ(clusters[kTarget1].size(), 1); + EXPECT_EQ(clusters[kTarget1][0].ops.size(), 2); +} + +TEST(BuildClusters, TestMergedClusters) { + static const char* const module_with_single_cluster = + R"(module { +func.func @main(%arg0: tensor) -> (tensor, tensor) { + %0 = "tf.Relu"(%arg0) : (tensor) -> tensor + %1 = "tf.Relu"(%0) {device = "tpu0"} : (tensor) -> tensor + %2 = "tf.Add"(%0, %1) {device = "tpu0"} : (tensor, tensor) -> tensor + %3 = "tf.Relu"(%2) : (tensor) -> tensor + %4 = "tf.Relu"(%1) {device = "tpu0"} : (tensor) -> tensor + %5 = "tf.Add"(%1, %2) {device = "tpu0"} : (tensor, tensor) -> tensor + func.return %3, %5 : tensor, tensor + } +} +)"; + + MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN( + OwningOpRef module, + GetMlirModuleFromString(module_with_single_cluster, &context)); + auto clusters = GetClusters(module.get()); + EXPECT_EQ(clusters.count(kTestClusterName), 1); + EXPECT_EQ(clusters[kTestClusterName].size(), 1); + EXPECT_EQ(clusters[kTestClusterName][0].ops.size(), 4); +} + +TEST(BuildClusters, TestMergedClustersWithDataDependen) { + static const char* const module_with_single_cluster = + R"(module { +func.func @main(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %0 = "tf.Relu"(%arg0) : (tensor) -> tensor + %1 = "tf.Relu"(%0) {device = "tpu0"} : (tensor) -> tensor + %2 = "tf.Add"(%0, %1) {device = "tpu0"} : (tensor, tensor) -> tensor + %3 = "tf.Relu"(%arg1) {device = "tpu1"} : (tensor) -> tensor + %4 = "tf.Add"(%3, %arg1) {device = "tpu1"} : (tensor, tensor) -> tensor + %5 = "tf.Relu"(%4) {device = "tpu0"} : (tensor) -> tensor + %6 = "tf.Add"(%4, %5) {device = "tpu0"} : (tensor, tensor) -> tensor + func.return %3, %5 : tensor, tensor + } +} +)"; + + MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN( + OwningOpRef module, + GetMlirModuleFromString(module_with_single_cluster, &context)); + auto clusters = GetClusters(module.get()); + EXPECT_EQ(clusters.count(kTestClusterName), 1); + EXPECT_EQ(clusters[kTestClusterName].size(), 1); + EXPECT_EQ(clusters[kTestClusterName][0].ops.size(), 4); +} + +} // namespace + +} // namespace mlir::TF diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.cc new file mode 100644 index 0000000000000000000000000000000000000000..40012b56182322737a8139bb9ebfde1dafd4c977 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.cc @@ -0,0 +1,115 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h" + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/platform/errors.h" + +namespace tensorflow { + +// Converts non func AttrValue proto into an MLIR attribute. Func attribute is +// exclused in this function because the function might be renamed when the +// function definition is imported. +absl::StatusOr ConvertNonFuncAttributeValue( + const AttrValue& value, mlir::Builder* builder) { + switch (value.value_case()) { + case AttrValue::kI: + return builder->getI64IntegerAttr(value.i()); + case AttrValue::kS: + return builder->getStringAttr(value.s()); + case AttrValue::kF: + return builder->getFloatAttr(builder->getF32Type(), value.f()); + case AttrValue::kB: + return builder->getBoolAttr(value.b()); + case AttrValue::kType: { + mlir::Type type; + TF_RETURN_IF_ERROR(ConvertDataType(value.type(), *builder, &type)); + return mlir::TypeAttr::get(type); + } + case AttrValue::kShape: + return ConvertTensorShapeProto(value.shape(), builder->getContext()); + case AttrValue::kTensor: + return ConvertTensorProto(value.tensor(), builder); + case AttrValue::kList: { + absl::InlinedVector attrs; + for (const auto& item : value.list().i()) + attrs.push_back(builder->getI64IntegerAttr(item)); + for (const auto& item : value.list().s()) + attrs.push_back(builder->getStringAttr(item)); + for (const auto& item : value.list().f()) + attrs.push_back(builder->getFloatAttr(builder->getF32Type(), item)); + for (const auto& item : value.list().b()) + attrs.push_back(builder->getBoolAttr(item)); + for (const auto& item : value.list().type()) { + mlir::Type type; + TF_RETURN_IF_ERROR(ConvertDataType(DataType(item), *builder, &type)); + attrs.push_back(mlir::TypeAttr::get(type)); + } + for (const auto& item : value.list().shape()) { + TF_ASSIGN_OR_RETURN( + auto attr, ConvertTensorShapeProto(item, builder->getContext())); + attrs.push_back(attr); + } + for (const auto& item : value.list().tensor()) { + TF_ASSIGN_OR_RETURN(auto attr, ConvertTensorProto(item, builder)); + attrs.push_back(attr); + } + if (!value.list().func().empty()) { + return tensorflow::errors::Unimplemented( + absl::StrCat("Attribute ", value.DebugString())); + } + return builder->getArrayAttr(llvm::ArrayRef(attrs.begin(), attrs.end())); + } + case AttrValue::VALUE_NOT_SET: + return builder->getUnitAttr(); + // kPlaceholder is not implemented. + case AttrValue::kPlaceholder: + return mlir::TF::PlaceholderAttr::get(builder->getContext(), + value.placeholder()); + default: + return tensorflow::errors::Unimplemented( + absl::StrCat("Attribute ", value.DebugString())); + } +} + +absl::StatusOr ConvertAttributeValue(const AttrValue& value, + mlir::Builder* builder) { + switch (value.value_case()) { + case AttrValue::kFunc: { + // TODO(b/156546237): Unify kFunc/NameAttrList attribute representation. + // Currently kFunc/NameAttrList attributes in a kList/repeated AttrValue + // will not use this representation. + mlir::NamedAttrList attrs; + for (const auto& func_attr : value.func().attr()) { + TF_ASSIGN_OR_RETURN(auto attr, + ConvertAttributeValue(func_attr.second, builder)); + attrs.push_back(builder->getNamedAttr(func_attr.first, attr)); + } + auto func_attrs = builder->getDictionaryAttr(attrs); + return mlir::TF::FuncAttr::get(builder->getContext(), value.func().name(), + func_attrs); + } + default: + return ConvertNonFuncAttributeValue(value, builder); + } +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h new file mode 100644 index 0000000000000000000000000000000000000000..11419ca226238271b0222b45c21b9e48ea24e33f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h @@ -0,0 +1,39 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_ATTR_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_ATTR_H_ + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { + +using tsl::StatusOr; + +// Converts non func AttrValue proto into an MLIR attribute. Func attribute is +// exclused in this function because the function might be renamed when the +// function definition is imported. +absl::StatusOr ConvertNonFuncAttributeValue( + const AttrValue& value, mlir::Builder* builder); + +// Converts all kinds of AttrValue proto into an MLIR attribute. +absl::StatusOr ConvertAttributeValue(const AttrValue& value, + mlir::Builder* builder); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_ATTR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.cc new file mode 100644 index 0000000000000000000000000000000000000000..06715e69ee70cd01fe454a2f83577b6d8644d70d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.cc @@ -0,0 +1,800 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/AsmState.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectResourceBlobManager.h" // from @llvm-project // IWYU pragma: keep +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/bfloat16.h" +#include "tensorflow/core/platform/tstring.h" +#include "tsl/platform/ml_dtypes.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { + +using llvm::ArrayRef; +using llvm::SmallVector; +using mlir::Builder; +using mlir::DenseStringElementsAttr; +using mlir::ElementsAttr; +using mlir::RankedTensorType; +using mlir::ShapedType; +using mlir::Type; +using tensorflow::errors::InvalidArgument; + +static TensorProto ConvertToProto(const Tensor& input_tensor, + bool use_tensor_content = true) { + TensorProto tensor_proto; + // Using tensor content (mostly*) reduces serialization overhead during RPC + // calls, but is less human reader friendly. People reading protobufs are less + // frequent than serialization, so default to using tensor content + // representation. + // * For scalars and short strings it may be marginally worse and a more + // intelligent decision could be made by caller. + if (use_tensor_content) + input_tensor.AsProtoTensorContent(&tensor_proto); + else + input_tensor.AsProtoField(&tensor_proto); + return tensor_proto; +} + +static std::string MangleTensor(const Tensor& tensor) { + return mangling_util::MangleTensor(ConvertToProto(tensor)); +} + +template +static absl::Status CopyDataIntoBlob(mlir::AsmResourceBlob& blob, + absl::string_view raw_src_data) { + ArrayRef data = blob.getDataAs(); + llvm::MutableArrayRef raw_dest_data = + mlir::MutableArrayRef(const_cast(data.data()), + data.size()); + if (raw_src_data.size() != blob.getData().size()) { + return absl::InvalidArgumentError( + "Size mismatch between raw_src_data and blob data"); + } + // Memcpy. + std::memcpy(raw_dest_data.data(), raw_src_data.data(), raw_src_data.size()); + + return absl::OkStatus(); +} + +// Converts a TensorFlow tensor into an MLIR elements attribute. +template +absl::StatusOr ConvertFlatTensor(const Tensor& input_tensor, + ShapedType shaped_type, + bool convert_to_dense_resource) { + // Only convert to dense resource if the data type is integer or floating. + if (convert_to_dense_resource && DataTypeCanUseMemcpy(input_tensor.dtype()) && + (DataTypeIsInteger(input_tensor.dtype()) || + DataTypeIsFloating(input_tensor.dtype()))) { + auto element_type = shaped_type.getElementType(); + auto num_elements = shaped_type.getNumElements(); + auto bit_width = element_type.getIntOrFloatBitWidth(); + auto tensor_data = input_tensor.tensor_data(); + mlir::AsmResourceBlob blob; + + if (llvm::isa(element_type)) { + switch (bit_width) { + case 1: + blob = mlir::HeapAsmResourceBlob::allocate(num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_i1", std::move(blob)); + case 8: + blob = mlir::HeapAsmResourceBlob::allocate(num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_i8", std::move(blob)); + case 16: + blob = mlir::HeapAsmResourceBlob::allocate(2 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_i16", std::move(blob)); + case 32: + blob = mlir::HeapAsmResourceBlob::allocate(4 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_i32", std::move(blob)); + case 64: + blob = mlir::HeapAsmResourceBlob::allocate(8 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_i64", std::move(blob)); + default: + return absl::InvalidArgumentError("Unsupported bit width"); + } + } else if (llvm::isa(element_type)) { + mlir::AsmResourceBlob blob; + switch (bit_width) { + case 8: + blob = mlir::HeapAsmResourceBlob::allocate(num_elements, /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_f8", std::move(blob)); + case 16: + blob = mlir::HeapAsmResourceBlob::allocate(2 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_f16", std::move(blob)); + case 32: { + blob = mlir::HeapAsmResourceBlob::allocate(4 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_f32", std::move(blob)); + } + case 64: + blob = mlir::HeapAsmResourceBlob::allocate(8 * num_elements, + /*align=*/64, + /*dataIsMutable=*/true); + TF_RETURN_IF_ERROR(CopyDataIntoBlob(blob, tensor_data)); + return mlir::DenseResourceElementsAttr::get( + shaped_type, "dense_elements_f64", std::move(blob)); + default: + return absl::InvalidArgumentError("Unsupported bit width"); + } + } else { + return absl::InvalidArgumentError("Unsupported element type"); + } + } else { + auto tensor_data = llvm::ArrayRef(input_tensor.flat().data(), + input_tensor.flat().size()); + return ElementsAttr(mlir::DenseElementsAttr::get(shaped_type, tensor_data)); + } +} + +ElementsAttr ConvertTensorOfCustomFloatType(const Tensor& tensor, + RankedTensorType type) { + auto buffer = + llvm::ArrayRef(static_cast(tensor.data()), tensor.TotalBytes()); + return mlir::DenseElementsAttr::getFromRawBuffer(type, buffer); +} + +template +static ElementsAttr ConvertSubByteTensor(const Tensor& tensor, + RankedTensorType type) { + auto flat = tensor.flat(); + std::vector unpacked_data; + unpacked_data.reserve(flat.size()); + for (int i = 0; i < flat.size(); ++i) { + unpacked_data.push_back(static_cast(static_cast(flat(i)))); + } + return mlir::DenseElementsAttr::getFromRawBuffer( + type, llvm::ArrayRef(unpacked_data)); +} + +absl::StatusOr ConvertStringTensor(const Tensor& input_tensor, + ShapedType type) { + // Extract to a vector of StringRefs for converting. + auto arr = input_tensor.flat(); + std::vector string_refs; + string_refs.reserve(arr.size()); + for (int i = 0; i < arr.size(); i++) { + const auto& val = arr(i); + string_refs.push_back({val.data(), val.size()}); + } + + return ElementsAttr(DenseStringElementsAttr::get(type, string_refs)); +} + +absl::StatusOr ConvertTensor(const Tensor& input_tensor, + Builder* builder, + bool convert_to_dense_resource) { + const auto& input_dtype = input_tensor.dtype(); + const auto& input_shape = input_tensor.shape(); + Type elt_type; + TF_RETURN_IF_ERROR(ConvertDataType(input_dtype, *builder, &elt_type)); + SmallVector shape; + ConvertToMlirShape(input_shape, &shape); + auto type = RankedTensorType::get(shape, elt_type); + +#define CONVERT_FLAT(DTYPE, CTYPE) \ + case DTYPE: \ + return ConvertFlatTensor(input_tensor, type, \ + convert_to_dense_resource); + + // TODO(fengliuai): customize the conversions for quantized types. + switch (input_dtype) { + CONVERT_FLAT(DT_BOOL, bool) + CONVERT_FLAT(DT_FLOAT, float) + CONVERT_FLAT(DT_DOUBLE, double) + CONVERT_FLAT(DT_INT8, int8_t) + CONVERT_FLAT(DT_INT16, int16_t) + CONVERT_FLAT(DT_INT32, int32_t) + CONVERT_FLAT(DT_INT64, int64_t) + CONVERT_FLAT(DT_UINT8, uint8_t) + CONVERT_FLAT(DT_UINT16, uint16_t) + CONVERT_FLAT(DT_UINT32, uint32_t) + CONVERT_FLAT(DT_UINT64, uint64_t) + CONVERT_FLAT(DT_COMPLEX64, std::complex) + CONVERT_FLAT(DT_COMPLEX128, std::complex) + + case DT_BFLOAT16: + case DT_HALF: + case DT_FLOAT8_E5M2: + case DT_FLOAT8_E4M3FN: + case DT_FLOAT8_E4M3FNUZ: + case DT_FLOAT8_E4M3B11FNUZ: + case DT_FLOAT8_E5M2FNUZ: + return ConvertTensorOfCustomFloatType(input_tensor, type); + case DT_INT4: + return ConvertSubByteTensor(input_tensor, type); + case DT_UINT4: + return ConvertSubByteTensor(input_tensor, type); + case DT_INT2: + return ConvertSubByteTensor(input_tensor, type); + case DT_UINT2: + return ConvertSubByteTensor(input_tensor, type); + case DT_STRING: + return ConvertStringTensor(input_tensor, type); + default: + // TODO(hinsu): Remove mangling now that there is a special attribute. + return ElementsAttr( + mlir::TF::TensorProtoAttr::get(type, MangleTensor(input_tensor))); + } + +#undef CONVERT_FLAT +} + +// Returns the number of elements present in this TensorProto, or -1 if that +// could not be determined. This might be less than the shape of the proto might +// indicate, if we're storing a splat tensor. +int NumberOfMaterializedElements(const TensorProto& tensor) { + if (!tensor.tensor_content().empty()) return -1; + // We don't know which element type this protocol buffer is storing, and the + // metaprogramming facilities for TensorProto are too limited to check their + // number without knowing this, so we need to manually dispatch to each + // possible member of TensorProto, depening on its dtype. +#define MATCH(DTYPE, FIELD) \ + case DTYPE: \ + return tensor.FIELD##_val().size() + + switch (tensor.dtype()) { + MATCH(DT_FLOAT, float); + MATCH(DT_DOUBLE, double); + MATCH(DT_INT8, int); + MATCH(DT_UINT8, int); + MATCH(DT_INT16, int); + MATCH(DT_UINT16, int); + MATCH(DT_INT32, int); + MATCH(DT_UINT32, uint32); + MATCH(DT_INT64, int64); + MATCH(DT_UINT64, uint64); + MATCH(DT_BOOL, bool); + MATCH(DT_HALF, half); + MATCH(DT_BFLOAT16, half); + MATCH(DT_STRING, string); + + // TODO(b/188995810): DenseElementsAttr::get doesn't support complex + // Attributes being passed, so we bail out for now. This should just be + // MATCH(DT_COMPLEX64, scomplex) / 2; + // MATCH(DT_COMPLEX128, dcomplex) / 2; + // when DenseElementsAttr is updated. + case DT_COMPLEX64: + case DT_COMPLEX128: + default: + return -1; + } +} + +absl::StatusOr ConvertTensorProto( + const TensorProto& input_tensor, Builder* builder, + bool convert_to_dense_resource) { + // If there is only one actual element in the proto, but its shape would + // indicate there are more values, then this is representing a splat tensor. + // We can create an MLIR Attribute more efficiently in this case. + TensorShape input_tensor_shape(input_tensor.tensor_shape()); + if (NumberOfMaterializedElements(input_tensor) == 1 && + input_tensor_shape.num_elements() > 1) { + // We first convert this TensorProto to one of shape [1]. We then create an + // Attribute for that proto, and finally splat the Attribute. + + TensorProto tensor_copy = input_tensor; + auto* shape = tensor_copy.mutable_tensor_shape(); + shape->clear_dim(); + shape->add_dim()->set_size(1); + + TF_ASSIGN_OR_RETURN(ElementsAttr single_attr, + ConvertTensorProto(tensor_copy, builder)); + + llvm::SmallVector original_dimensions; + for (auto dim : input_tensor_shape) original_dimensions.push_back(dim.size); + return ElementsAttr(mlir::SplatElementsAttr::get( + single_attr.getShapedType().clone(original_dimensions), + single_attr.getValues()[0])); + } + + Tensor t; + if (!t.FromProto(input_tensor)) + return InvalidArgument("Failed to parse input_tensor."); + return ConvertTensor(t, builder, convert_to_dense_resource); +} + +void ConvertToTensorShapeProto(ArrayRef shape, + TensorShapeProto* output_shape) { + for (auto d : shape) { + output_shape->add_dim()->set_size(ShapedType::isDynamic(d) ? kTFDynamicSize + : d); + } +} + +PartialTensorShape ConvertTypeToTensorShape(const mlir::Type& type) { + if (mlir::isa(type)) { + // An empty PartialTensorShape indicates an unranked tensor. + return PartialTensorShape(); + } + + if (auto tensor_type = mlir::dyn_cast(type)) { + TensorShapeProto tensor_shape_proto; + ConvertToTensorShapeProto(tensor_type.getShape(), &tensor_shape_proto); + return PartialTensorShape(tensor_shape_proto); + } + + // If type is not a RankedTensor or UnrankedTensor, it must be a scalar. + // Empty TensorShape indicates a scalar. + return TensorShape(); +} + +mlir::TF::ShapeAttr ConvertTypeToTensorShapeAttr(const mlir::Type& type) { + if (mlir::isa(type)) { + return mlir::TF::ShapeAttr::get(type.getContext(), std::nullopt); + } + + if (auto tensor_type = mlir::dyn_cast(type)) { + return mlir::TF::ShapeAttr::get(type.getContext(), tensor_type.getShape()); + } + + // If type is not a RankedTensor or UnrankedTensor, it must be a scalar. + // Empty TensorShape indicates a scalar. + return mlir::TF::ShapeAttr::get(type.getContext(), ArrayRef()); +} + +absl::StatusOr ConvertTypeToTensorSpecProto( + const mlir::Type& type) { + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(type, &dtype)); + TensorSpecProto tensor_spec; + tensor_spec.set_dtype(dtype); + *tensor_spec.mutable_shape() = ConvertTypeToTensorShape(type).AsProto(); + return tensor_spec; +} + +// Converts the tensor shape proto into an MLIR shape attribute. +absl::StatusOr ConvertTensorShapeProto( + const TensorShapeProto& shape, mlir::MLIRContext* context) { + if (shape.unknown_rank()) + return mlir::TF::ShapeAttr::get(context, std::nullopt); + + llvm::SmallVector dims; + dims.reserve(shape.dim().size()); + for (const auto& dim : shape.dim()) { + dims.push_back(dim.size() == kTFDynamicSize ? ShapedType::kDynamic + : dim.size()); + } + return mlir::TF::ShapeAttr::get(context, llvm::ArrayRef(dims)); +} + +// Converts an MLIR dense string elements attribute to a TensorFlow tensor +// proto. +absl::Status ConvertStringElementsAttr( + const DenseStringElementsAttr attr, + protobuf::RepeatedPtrField* output) { + for (const auto& val : attr.getRawStringData()) + output->Add({val.data(), val.size()}); + return absl::OkStatus(); +} + +template +absl::Status ConvertComplexElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output) { + auto attr = llvm::dyn_cast(elem_attr); + if (!attr) + return absl::InvalidArgumentError("Unsupported elements attr found"); + + auto elementType = attr.getType().getElementType(); + if (!llvm::isa(elementType)) { + return absl::InvalidArgumentError("Complex elements attr not found"); + } + + auto complex_elem_ty = + llvm::cast(elementType).getElementType(); + if (complex_elem_ty.isF32()) { + for (const auto& val : attr.getValues>()) { + output->Add(val.real().convertToFloat()); + output->Add(val.imag().convertToFloat()); + } + } else if (complex_elem_ty.isF64()) { + for (const auto& val : attr.getValues>()) { + output->Add(val.real().convertToDouble()); + output->Add(val.imag().convertToDouble()); + } + } else { + return absl::InvalidArgumentError("Unsupported complex element type"); + } + return absl::OkStatus(); +} + +// Converts an Tensor proto attribute to a TensorFlow tensor proto. +absl::Status ConvertTensorProtoAttr(const mlir::TF::TensorProtoAttr attr, + TensorProto* output_tensor) { + auto mangled_tensor = attr.getValue(); + absl::string_view tensor_view(mangled_tensor.data(), mangled_tensor.size()); + return mangling_util::DemangleTensor(tensor_view, output_tensor); +} + +template +absl::Status ConvertElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output) { + auto attr = llvm::dyn_cast(elem_attr); + if (!attr) + return absl::InvalidArgumentError("Unsupported elements attr found"); + if (attr.isSplat()) { + if (attr.getSplatValue() != T(0)) output->Add(attr.getSplatValue()); + } else { + output->Reserve(attr.getNumElements()); + for (auto value : attr.getValues()) output->AddAlreadyReserved(value); + } + return absl::OkStatus(); +} + +// Converts an MLIR elements attribute and adds it to specified repeated field. +template +absl::Status ConvertFloatElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output, + Cord* tensor_content) { + if (auto attr = llvm::dyn_cast(elem_attr)) { + if (attr.isSplat()) { + if (attr.getSplatValue() != T(0)) output->Add(attr.getSplatValue()); + } else { + port::CopyFromArray(tensor_content, attr.getRawData().data(), + attr.getRawData().size()); + } + } else if (auto dense_resource_ttr = + llvm::dyn_cast(elem_attr)) { + mlir::AsmResourceBlob* blob = dense_resource_ttr.getRawHandle().getBlob(); + if (blob) { + size_t dst_block_length = blob->getData().size(); + const char* raw_dst_block = blob->getData().data(); + if constexpr (std::is_same_v) { + *tensor_content = absl::string_view(raw_dst_block, dst_block_length); + } else { + *tensor_content = absl::MakeCordFromExternal( + absl::string_view(raw_dst_block, dst_block_length), + [](absl::string_view data) {}); + } + } else { + return absl::InvalidArgumentError("No blob found in dense resource"); + } + } else { + return absl::InvalidArgumentError("Unsupported elements attr found"); + } + return absl::OkStatus(); +} + +// Converts an MLIR elements attribute containing half values and adds it to +// specified repeated field. +absl::Status ConvertHalfElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output) { + auto attr = llvm::dyn_cast(elem_attr); + if (!attr) + return absl::InvalidArgumentError( + "DenseResourceElementsAttr of type half found"); + if (attr.isSplat()) { + if (attr.getSplatValue() != Eigen::half(0)) + output->Add( + Eigen::numext::bit_cast(attr.getSplatValue())); + } else { + output->Reserve(attr.getNumElements()); + for (const Eigen::half value : attr.getValues()) + output->AddAlreadyReserved(Eigen::numext::bit_cast(value)); + } + return absl::OkStatus(); +} + +// Converts an MLIR elements attribute containing signed int values and adds it +// to specified repeated field. +template +absl::Status ConvertIntElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output, + Cord* tensor_content) { + if (auto attr = llvm::dyn_cast(elem_attr)) { + if (attr.isSplat()) { + if (attr.getSplatValue() != U(0)) + output->Add(static_cast(attr.getSplatValue())); + } else { + port::CopyFromArray(tensor_content, attr.getRawData().data(), + attr.getRawData().size()); + } + } else if (auto dense_resource_ttr = + llvm::dyn_cast(elem_attr)) { + mlir::AsmResourceBlob* blob = dense_resource_ttr.getRawHandle().getBlob(); + if (blob) { + size_t dst_block_length = blob->getData().size(); + const char* raw_dst_block = blob->getData().data(); + if constexpr (std::is_same_v) { + *tensor_content = absl::string_view(raw_dst_block, dst_block_length); + } else { + *tensor_content = absl::MakeCordFromExternal( + absl::string_view(raw_dst_block, dst_block_length), + [](absl::string_view data) {}); + } + } else { + return absl::InvalidArgumentError("No blob found in dense resource"); + } + } else { + return absl::InvalidArgumentError("Unsupported elements attr found"); + } + return absl::OkStatus(); +} + +// Converts an MLIR elements attribute containing unsigned int values and adds +// it to specified repeated field. +template +absl::Status ConvertUIntElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output, + Cord* tensor_content) { + if (auto attr = llvm::dyn_cast(elem_attr)) { + if (attr.isSplat()) { + if (attr.getSplatValue() != U(0)) + output->Add(static_cast(attr.getSplatValue())); + } else { + port::CopyFromArray(tensor_content, attr.getRawData().data(), + attr.getRawData().size()); + } + } else if (auto dense_resource_ttr = + llvm::dyn_cast(elem_attr)) { + mlir::AsmResourceBlob* blob = dense_resource_ttr.getRawHandle().getBlob(); + if (blob) { + size_t dst_block_length = blob->getData().size(); + const char* raw_dst_block = blob->getData().data(); + if constexpr (std::is_same_v) { + *tensor_content = absl::string_view(raw_dst_block, dst_block_length); + } else { + *tensor_content = absl::MakeCordFromExternal( + absl::string_view(raw_dst_block, dst_block_length), + [](absl::string_view data) {}); + } + } else { + return absl::InvalidArgumentError("No blob found in dense resource"); + } + } else { + return absl::InvalidArgumentError("Unsupported elements attr found"); + } + return absl::OkStatus(); +} + +absl::Status ConvertBfloat16ElementsAttr(const mlir::ElementsAttr elem_attr, + protobuf::RepeatedField* output) { + auto attr = llvm::dyn_cast(elem_attr); + if (!attr) + return absl::InvalidArgumentError("Unsupported elements attr found"); + if (attr.isSplat()) { + if (attr.getSplatValue() != bfloat16(0)) + output->Add( + Eigen::numext::bit_cast(attr.getSplatValue())); + } else { + output->Reserve(attr.getNumElements()); + for (const bfloat16 value : attr.getValues()) + output->AddAlreadyReserved(Eigen::numext::bit_cast(value)); + } + return absl::OkStatus(); +} + +template +absl::Status ConvertFloat8ElementsAttr(const mlir::ElementsAttr elem_attr, + std::string* output) { + auto attr = llvm::dyn_cast(elem_attr); + if (!attr) + return absl::InvalidArgumentError("Unsupported elements attr found"); + if (attr.isSplat()) { + if (attr.getSplatValue() != T(0)) + output->push_back( + Eigen::numext::bit_cast(attr.getSplatValue())); + } else { + output->reserve(attr.getNumElements()); + for (const T value : attr.getValues()) + output->push_back(Eigen::numext::bit_cast(value)); + } + return absl::OkStatus(); +} + +absl::Status ConvertToTensorProto(const ElementsAttr attr, + TensorProto* output) { + auto type = attr.getShapedType(); + auto shape = type.getShape(); + DataType output_dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(type, &output_dtype)); + output->set_dtype(output_dtype); + ConvertToTensorShapeProto(shape, output->mutable_tensor_shape()); + + if (auto tensor_attr = mlir::dyn_cast(attr)) + return ConvertTensorProtoAttr(tensor_attr, output); + + switch (output_dtype) { + case DT_BOOL: + TF_RETURN_IF_ERROR(ConvertElementsAttr(attr, output->mutable_bool_val())); + break; + case DT_BFLOAT16: + TF_RETURN_IF_ERROR( + ConvertBfloat16ElementsAttr(attr, output->mutable_half_val())); + break; + case DT_COMPLEX64: + TF_RETURN_IF_ERROR( + ConvertComplexElementsAttr(attr, output->mutable_scomplex_val())); + break; + case DT_COMPLEX128: + TF_RETURN_IF_ERROR( + ConvertComplexElementsAttr(attr, output->mutable_dcomplex_val())); + break; + case DT_DOUBLE: + TF_RETURN_IF_ERROR( + ConvertFloatElementsAttr(attr, output->mutable_double_val(), + output->mutable_tensor_content())); + break; + case DT_HALF: + TF_RETURN_IF_ERROR( + ConvertHalfElementsAttr(attr, output->mutable_half_val())); + break; + case DT_FLOAT: + TF_RETURN_IF_ERROR(ConvertFloatElementsAttr( + attr, output->mutable_float_val(), output->mutable_tensor_content())); + break; + case DT_FLOAT8_E5M2: + TF_RETURN_IF_ERROR(ConvertFloat8ElementsAttr( + attr, output->mutable_float8_val())); + break; + case DT_FLOAT8_E4M3FN: + TF_RETURN_IF_ERROR(ConvertFloat8ElementsAttr( + attr, output->mutable_float8_val())); + break; + case DT_FLOAT8_E4M3FNUZ: + TF_RETURN_IF_ERROR(ConvertFloat8ElementsAttr( + attr, output->mutable_float8_val())); + break; + case DT_FLOAT8_E4M3B11FNUZ: + TF_RETURN_IF_ERROR(ConvertFloat8ElementsAttr( + attr, output->mutable_float8_val())); + break; + case DT_FLOAT8_E5M2FNUZ: + TF_RETURN_IF_ERROR(ConvertFloat8ElementsAttr( + attr, output->mutable_float8_val())); + break; + case tensorflow::DT_INT4: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case tensorflow::DT_UINT4: + TF_RETURN_IF_ERROR(ConvertUIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case tensorflow::DT_INT2: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case tensorflow::DT_UINT2: + TF_RETURN_IF_ERROR(ConvertUIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_QUINT8: + case DT_INT8: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_QUINT16: + case DT_INT16: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_INT32: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_INT64: + TF_RETURN_IF_ERROR(ConvertIntElementsAttr( + attr, output->mutable_int64_val(), output->mutable_tensor_content())); + break; + case DT_STRING: + TF_RETURN_IF_ERROR( + ConvertStringElementsAttr(mlir::cast(attr), + output->mutable_string_val())); + break; + case DT_UINT8: + TF_RETURN_IF_ERROR(ConvertUIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_UINT16: + TF_RETURN_IF_ERROR(ConvertUIntElementsAttr( + attr, output->mutable_int_val(), output->mutable_tensor_content())); + break; + case DT_UINT32: + TF_RETURN_IF_ERROR( + ConvertUIntElementsAttr(attr, output->mutable_uint32_val(), + output->mutable_tensor_content())); + break; + case DT_UINT64: + TF_RETURN_IF_ERROR( + ConvertUIntElementsAttr(attr, output->mutable_uint64_val(), + output->mutable_tensor_content())); + break; + default: + return absl::UnimplementedError(absl::StrCat( + "Unimplemented data type ", DataTypeString(output_dtype))); + } + return absl::OkStatus(); +} + +absl::Status ConvertToTensor(const mlir::ElementsAttr attr, + Tensor* output_tensor) { + TensorProto tensor_proto; + TF_RETURN_IF_ERROR(ConvertToTensorProto(attr, &tensor_proto)); + if (!output_tensor->FromProto(tensor_proto)) { + return InvalidArgument("Couldn't convert tensor proto to tensor."); + } + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h new file mode 100644 index 0000000000000000000000000000000000000000..591e70ad19883bb8ec5ab81f09367eec793d5793 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h @@ -0,0 +1,70 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TENSOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TENSOR_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/protobuf/struct.pb.h" + +namespace tensorflow { + +using tsl::StatusOr; + +// Converts an TensorFlow tensor proto into an MLIR elements attribute. +absl::StatusOr ConvertTensorProto( + const TensorProto& input_tensor, mlir::Builder* builder, + bool convert_to_dense_resource = false); + +// Converts an TensorFlow tensor into an MLIR elements attribute. +absl::StatusOr ConvertTensor( + const Tensor& input_tensor, mlir::Builder* builder, + bool convert_to_dense_resource = false); + +// Converts a shape from MLIR to a TensorFlow tensor shape proto. +void ConvertToTensorShapeProto(llvm::ArrayRef shape, + TensorShapeProto* output_shape); + +// Converts an MLIR type to a TensorFlow tensor shape. +PartialTensorShape ConvertTypeToTensorShape(const mlir::Type& type); + +// Converts an MLIR shaped type to a TensorFlow shape attribute. +mlir::TF::ShapeAttr ConvertTypeToTensorShapeAttr(const mlir::Type& type); + +// Converts an MLIR shaped type to a Tensorflow tensor spec proto. +absl::StatusOr ConvertTypeToTensorSpecProto( + const mlir::Type& type); + +// Converts a TensorFlow shape attribute to an MLIR shape attribute. +absl::StatusOr ConvertTensorShapeProto( + const TensorShapeProto& shape, mlir::MLIRContext* context); + +// Converts an MLIR elements attribute to a TensorFlow tensor proto. +absl::Status ConvertToTensorProto(mlir::ElementsAttr attr, + TensorProto* output_tensor); + +// Converts an MLIR elements attribute to a TensorFlow tensor. +absl::Status ConvertToTensor(mlir::ElementsAttr attr, Tensor* output_tensor); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TENSOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..2468c81feb3b6b1fb7ddaae285bf1e592cf4ed4a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_tensor_test.cc @@ -0,0 +1,353 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" + +#include +#include + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "xla/hlo/testlib/test.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/tensor_util.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/types.h" +#include "tsl/platform/ml_dtypes.h" + +namespace tensorflow { +namespace { + +using ::testing::Eq; +using ::testing::IsFalse; +using ::testing::IsTrue; + +static void RegisterDialects(mlir::MLIRContext& context) { + context.loadDialect(); +} + +TEST(ConvertTypeToTensorTypeTest, UnrankedTensorType) { + mlir::MLIRContext context; + RegisterDialects(context); + mlir::Builder b(&context); + + PartialTensorShape output_shape = + ConvertTypeToTensorShape(mlir::UnrankedTensorType::get(b.getF32Type())); + EXPECT_TRUE(output_shape.IsIdenticalTo(PartialTensorShape())); +} + +TEST(ConvertTypeToTensorTypeTest, NonFullyDefinedRankedTensorType) { + mlir::MLIRContext context; + RegisterDialects(context); + mlir::Builder b(&context); + + PartialTensorShape output_shape = ConvertTypeToTensorShape( + GetTypeFromTFTensorShape({-1, 2, 3}, b.getF32Type())); + EXPECT_TRUE(output_shape.IsIdenticalTo(PartialTensorShape({-1, 2, 3}))); +} + +TEST(ConvertTypeToTensorTypeTest, FullyDefinedRankedTensorType) { + mlir::MLIRContext context; + RegisterDialects(context); + mlir::Builder b(&context); + + PartialTensorShape output_shape = ConvertTypeToTensorShape( + mlir::RankedTensorType::get({1, 2, 3}, b.getF32Type())); + EXPECT_TRUE(output_shape.IsIdenticalTo(PartialTensorShape({1, 2, 3}))); +} + +TEST(ConvertTypeToTensorTypeTest, ScalarTensorType) { + mlir::MLIRContext context; + mlir::Builder b(&context); + + PartialTensorShape output_shape = ConvertTypeToTensorShape(b.getF32Type()); + EXPECT_TRUE(output_shape.IsIdenticalTo(TensorShape())); +} + +TEST(ConvertTypeToTensorTypeTest, ConvertStringTensor) { + mlir::MLIRContext context; + RegisterDialects(context); + mlir::Builder b(&context); + + // Create the sample tensor to convert. + Tensor tensor(DT_STRING, TensorShape({1, 2, 2, 1})); + EXPECT_EQ(4, tensor.NumElements()); + auto Tt = tensor.flat(); + Tt.setValues({"one", "two", "three", "four"}); + auto value_or_status = ConvertTensor(tensor, &b); + ASSERT_TRUE(value_or_status.ok()); + auto attr = value_or_status.value(); + + EXPECT_TRUE(mlir::isa(attr)); + auto string_attr = mlir::cast(attr); + auto string_values = string_attr.getRawStringData(); + ASSERT_EQ(string_values.size(), 4); + EXPECT_EQ(string_values[0], mlir::StringRef("one")); + EXPECT_EQ(string_values[1], mlir::StringRef("two")); + EXPECT_EQ(string_values[2], mlir::StringRef("three")); + EXPECT_EQ(string_values[3], mlir::StringRef("four")); +} + +class ConvertTensorTest : public ::testing::Test { + protected: + template + void VerifyConversion(std::initializer_list values, DataType dtype, + mlir::Type expected_ty, + bool convert_to_dense_resource = false) { + mlir::Builder b(expected_ty.getContext()); + Tensor tensor(dtype, TensorShape({static_cast(values.size())})); + tensor.flat().setValues(values); + + auto value_or = ConvertTensor(tensor, &b, convert_to_dense_resource); + TF_ASSERT_OK(value_or.status()); + auto attr = value_or.value(); + + EXPECT_EQ(attr.getShapedType().getElementType(), expected_ty); + + Tensor out; + TF_ASSERT_OK(ConvertToTensor(attr, &out)); + + test::ExpectTensorEqual(tensor, out); + } +}; + +TEST_F(ConvertTensorTest, Simple) { + mlir::MLIRContext context; + RegisterDialects(context); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {Eigen::half(1.0)}, DT_HALF, mlir::Float16Type::get(&context))); + ASSERT_NO_FATAL_FAILURE( + VerifyConversion({bfloat16(1.0), bfloat16(-1.0)}, DT_BFLOAT16, + mlir::BFloat16Type::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1.0, -1.0}, DT_FLOAT, mlir::Float32Type::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1.0, -1.0}, DT_DOUBLE, mlir::Float64Type::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e5m2{1.0}, tsl::float8_e5m2{-1.0}}, DT_FLOAT8_E5M2, + mlir::Float8E5M2Type::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e4m3fn{1.0}, tsl::float8_e4m3fn{-1.0}}, DT_FLOAT8_E4M3FN, + mlir::Float8E4M3FNType::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e4m3fnuz{1.0}, tsl::float8_e4m3fnuz{-1.0}}, + DT_FLOAT8_E4M3FNUZ, mlir::Float8E4M3FNUZType::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e4m3b11fnuz{1.0}, tsl::float8_e4m3b11fnuz{-1.0}}, + DT_FLOAT8_E4M3B11FNUZ, mlir::Float8E4M3B11FNUZType::get(&context))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e5m2fnuz{1.0}, tsl::float8_e5m2fnuz{-1.0}}, + DT_FLOAT8_E5M2FNUZ, mlir::Float8E5M2FNUZType::get(&context))); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(-1)}, DT_INT4, + mlir::IntegerType::get(&context, 4))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT8, mlir::IntegerType::get(&context, 8))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT16, mlir::IntegerType::get(&context, 16))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT32, mlir::IntegerType::get(&context, 32))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT64, mlir::IntegerType::get(&context, 64))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(0)}, DT_INT2, + mlir::IntegerType::get(&context, 2))); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(2)}, DT_UINT4, + mlir::IntegerType::get( + &context, 4, mlir::IntegerType::SignednessSemantics::Unsigned))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT8, + mlir::IntegerType::get( + &context, 8, mlir::IntegerType::SignednessSemantics::Unsigned))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT16, + mlir::IntegerType::get( + &context, 16, mlir::IntegerType::SignednessSemantics::Unsigned))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT32, + mlir::IntegerType::get( + &context, 32, mlir::IntegerType::SignednessSemantics::Unsigned))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT64, + mlir::IntegerType::get( + &context, 64, mlir::IntegerType::SignednessSemantics::Unsigned))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(2)}, DT_UINT2, + mlir::IntegerType::get( + &context, 2, mlir::IntegerType::SignednessSemantics::Unsigned))); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion>( + {{0.0, 1.0}, {1.0, 0.0}}, DT_COMPLEX64, + mlir::ComplexType::get(mlir::Float32Type::get(&context)))); + ASSERT_NO_FATAL_FAILURE(VerifyConversion>( + {{0.0, 1.0}, {1.0, 0.0}}, DT_COMPLEX128, + mlir::ComplexType::get(mlir::Float64Type::get(&context)))); +} + +TEST_F(ConvertTensorTest, SimpleDenseResourceElements) { + mlir::MLIRContext context; + RegisterDialects(context); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {Eigen::half(1.0)}, DT_HALF, mlir::Float16Type::get(&context), true)); + ASSERT_NO_FATAL_FAILURE( + VerifyConversion({bfloat16(1.0), bfloat16(-1.0)}, DT_BFLOAT16, + mlir::BFloat16Type::get(&context), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1.0, -1.0}, DT_FLOAT, mlir::Float32Type::get(&context), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1.0, -1.0}, DT_DOUBLE, mlir::Float64Type::get(&context), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e5m2{1.0}, tsl::float8_e5m2{-1.0}}, DT_FLOAT8_E5M2, + mlir::Float8E5M2Type::get(&context), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {tsl::float8_e4m3fn{1.0}, tsl::float8_e4m3fn{-1.0}}, DT_FLOAT8_E4M3FN, + mlir::Float8E4M3FNType::get(&context), true)); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(-1)}, DT_INT4, + mlir::IntegerType::get(&context, 4), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT8, mlir::IntegerType::get(&context, 8), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT16, mlir::IntegerType::get(&context, 16), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT32, mlir::IntegerType::get(&context, 32), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, -1}, DT_INT64, mlir::IntegerType::get(&context, 64), true)); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {static_cast(1), static_cast(2)}, DT_UINT4, + mlir::IntegerType::get(&context, 4, + mlir::IntegerType::SignednessSemantics::Unsigned), + true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT8, + mlir::IntegerType::get(&context, 8, + mlir::IntegerType::SignednessSemantics::Unsigned), + true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT16, + mlir::IntegerType::get(&context, 16, + mlir::IntegerType::SignednessSemantics::Unsigned), + true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT32, + mlir::IntegerType::get(&context, 32, + mlir::IntegerType::SignednessSemantics::Unsigned), + true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion( + {1, 2}, DT_UINT64, + mlir::IntegerType::get(&context, 64, + mlir::IntegerType::SignednessSemantics::Unsigned), + true)); + + ASSERT_NO_FATAL_FAILURE(VerifyConversion>( + {{0.0, 1.0}, {1.0, 0.0}}, DT_COMPLEX64, + mlir::ComplexType::get(mlir::Float32Type::get(&context)), true)); + ASSERT_NO_FATAL_FAILURE(VerifyConversion>( + {{0.0, 1.0}, {1.0, 0.0}}, DT_COMPLEX128, + mlir::ComplexType::get(mlir::Float64Type::get(&context)))); +} + +bool IsSplat(mlir::ElementsAttr attr) { + return mlir::cast(attr).isSplat(); +} + +TEST(ConvertTensorProtoTest, SplatTensor) { + // We construct a sparse TensorProto representing 2^35 float elements, all of + // them 42. Our conversion routine should not materialize these elements when + // creating the Attribute. If it tries to, we'll crash OOM here. + TensorProto tensor; + tensor.set_dtype(DT_FLOAT); + tensor.mutable_tensor_shape()->add_dim()->set_size(1ULL << 35); + tensor.add_float_val(42.0); + + mlir::MLIRContext context; + mlir::Builder builder(&context); + TF_ASSERT_OK_AND_ASSIGN(mlir::ElementsAttr attribute, + ConvertTensorProto(tensor, &builder)); + EXPECT_THAT( + attribute, + AllOf(Eq(mlir::DenseElementsAttr::get( + mlir::RankedTensorType::get({1ULL << 35}, builder.getF32Type()), + 42.0f)), + ResultOf(IsSplat, IsTrue()))); +} + +TEST(ConvertTensorProtoTest, NonSplatTensor) { + TensorProto proto = tensor::CreateTensorProto( + /*values=*/{1.0f, 2.0f, 3.0f, 4.0f}, /*shape=*/{2, 2}); + mlir::MLIRContext context; + mlir::Builder builder(&context); + + TF_ASSERT_OK_AND_ASSIGN(mlir::ElementsAttr attribute, + ConvertTensorProto(proto, &builder)); + EXPECT_THAT( + attribute, + AllOf(Eq(mlir::DenseElementsAttr::get( + mlir::RankedTensorType::get({2, 2}, builder.getF32Type()), + {1.0f, 2.0f, 3.0f, 4.0f})), + ResultOf(IsSplat, IsFalse()))); +} + +TEST(ConvertTypeToTensorSpecProtoTest, UnrankedTensorType) { + mlir::MLIRContext context; + mlir::Builder b(&context); + + auto output_proto = ConvertTypeToTensorSpecProto( + mlir::UnrankedTensorType::get(b.getF32Type())); + TF_ASSERT_OK(output_proto.status()); + EXPECT_EQ(output_proto->dtype(), DT_FLOAT); + EXPECT_TRUE(output_proto->shape().unknown_rank()); +} + +TEST(ConvertTypeToTensorSpecProtoTest, RankedTensorType) { + mlir::MLIRContext context; + mlir::Builder b(&context); + + auto output_proto = ConvertTypeToTensorSpecProto( + mlir::RankedTensorType::get({1, 2, 3}, b.getF32Type())); + TF_ASSERT_OK(output_proto.status()); + EXPECT_EQ(output_proto->dtype(), DT_FLOAT); + EXPECT_EQ(output_proto->shape().dim_size(), 3); + EXPECT_EQ(output_proto->shape().dim().at(0).size(), 1); + EXPECT_EQ(output_proto->shape().dim().at(1).size(), 2); + EXPECT_EQ(output_proto->shape().dim().at(2).size(), 3); +} + +TEST(ConvertTypeToTensorSpecProtoTest, ScalarTensorType) { + mlir::MLIRContext context; + mlir::Builder b(&context); + + auto output_proto = ConvertTypeToTensorSpecProto(b.getF32Type()); + TF_ASSERT_OK(output_proto.status()); + EXPECT_EQ(output_proto->dtype(), DT_FLOAT); + EXPECT_FALSE(output_proto->shape().unknown_rank()); + EXPECT_EQ(output_proto->shape().dim_size(), 0); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc new file mode 100644 index 0000000000000000000000000000000000000000..687b6c622460626396cf0df694e7acbd971baabe --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc @@ -0,0 +1,250 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" + +#include + +#include "absl/strings/str_cat.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { + +using mlir::Builder; +using mlir::ShapedType; +using mlir::Type; + +absl::Status ConvertDataType(DataType dtype, Builder builder, Type* type) { + switch (dtype) { + case DT_HALF: + *type = builder.getF16Type(); + return absl::OkStatus(); + case DT_FLOAT: + *type = builder.getF32Type(); + return absl::OkStatus(); + case DT_DOUBLE: + *type = builder.getF64Type(); + return absl::OkStatus(); + case DT_BOOL: + *type = builder.getIntegerType(1); + return absl::OkStatus(); + case DT_INT8: + *type = builder.getIntegerType(8); + return absl::OkStatus(); + case DT_INT16: + *type = builder.getIntegerType(16); + return absl::OkStatus(); + case DT_INT32: + *type = builder.getIntegerType(32); + return absl::OkStatus(); + case DT_INT64: + *type = builder.getIntegerType(64); + return absl::OkStatus(); + case DT_UINT8: + *type = builder.getIntegerType(8, /*isSigned=*/false); + return absl::OkStatus(); + case DT_UINT16: + *type = builder.getIntegerType(16, /*isSigned=*/false); + return absl::OkStatus(); + case DT_UINT32: + *type = builder.getIntegerType(32, /*isSigned=*/false); + return absl::OkStatus(); + case DT_UINT64: + *type = builder.getIntegerType(64, /*isSigned=*/false); + return absl::OkStatus(); + case DT_BFLOAT16: + *type = builder.getBF16Type(); + return absl::OkStatus(); + case DT_COMPLEX64: + *type = mlir::ComplexType::get(builder.getF32Type()); + return absl::OkStatus(); + case DT_COMPLEX128: + *type = mlir::ComplexType::get(builder.getF64Type()); + return absl::OkStatus(); + case tensorflow::DT_FLOAT8_E4M3FN: + *type = builder.getType(); + return absl::OkStatus(); + case tensorflow::DT_FLOAT8_E5M2: + *type = builder.getType(); + return absl::OkStatus(); + case tensorflow::DT_FLOAT8_E4M3FNUZ: + *type = builder.getType(); + return absl::OkStatus(); + case tensorflow::DT_FLOAT8_E4M3B11FNUZ: + *type = builder.getType(); + return absl::OkStatus(); + case tensorflow::DT_FLOAT8_E5M2FNUZ: + *type = builder.getType(); + return absl::OkStatus(); + case DT_INT4: + // build a **signless** integer type. + *type = builder.getIntegerType(4); + return absl::OkStatus(); + case DT_UINT4: + *type = builder.getIntegerType(4, /*isSigned=*/false); + return absl::OkStatus(); + case DT_INT2: + // build a **signless** integer type. + *type = builder.getIntegerType(2); + return absl::OkStatus(); + case DT_UINT2: + *type = builder.getIntegerType(2, /*isSigned=*/false); + return absl::OkStatus(); +#define HANDLE_TF_TYPE(tftype, enumerant, name) \ + case DT_##enumerant: \ + *type = builder.getType(); \ + return OkStatus(); +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.def" + + default: + return errors::Unimplemented(absl::StrCat( + "Converting DataType '", DataTypeString(dtype), "' to MLIR Type")); + } +} + +absl::Status ConvertScalarTypeToDataType(Type type, DataType* dtype) { + if (type.isF16()) { + *dtype = DT_HALF; + return absl::OkStatus(); + } else if (type.isF32()) { + *dtype = DT_FLOAT; + return absl::OkStatus(); + } else if (type.isF64()) { + *dtype = DT_DOUBLE; + return absl::OkStatus(); + } else if (type.isBF16()) { + *dtype = DT_BFLOAT16; + return absl::OkStatus(); + } else if (llvm::isa(type)) { + *dtype = DT_FLOAT8_E4M3FN; + return absl::OkStatus(); + } else if (llvm::isa(type)) { + *dtype = DT_FLOAT8_E5M2; + return absl::OkStatus(); + } else if (llvm::isa(type)) { + *dtype = DT_FLOAT8_E4M3FNUZ; + return absl::OkStatus(); + } else if (llvm::isa(type)) { + *dtype = DT_FLOAT8_E4M3B11FNUZ; + return absl::OkStatus(); + } else if (llvm::isa(type)) { + *dtype = DT_FLOAT8_E5M2FNUZ; + return absl::OkStatus(); + } else if (auto itype = mlir::dyn_cast(type)) { + switch (itype.getWidth()) { + case 1: + *dtype = DT_BOOL; + return absl::OkStatus(); + case 2: + *dtype = itype.isUnsigned() ? DT_UINT2 : DT_INT2; + return absl::OkStatus(); + case 4: + *dtype = itype.isUnsigned() ? DT_UINT4 : DT_INT4; + return absl::OkStatus(); + case 8: + *dtype = itype.isUnsigned() ? DT_UINT8 : DT_INT8; + return absl::OkStatus(); + case 16: + *dtype = itype.isUnsigned() ? DT_UINT16 : DT_INT16; + return absl::OkStatus(); + case 32: + *dtype = itype.isUnsigned() ? DT_UINT32 : DT_INT32; + return absl::OkStatus(); + case 64: + *dtype = itype.isUnsigned() ? DT_UINT64 : DT_INT64; + return absl::OkStatus(); + default: + return errors::Unimplemented( + absl::StrCat("Converting ", debugString(type), " to DataType")); + } + } else if (auto complex_type = mlir::dyn_cast(type)) { + auto etype = complex_type.getElementType(); + if (etype.isF32()) { + *dtype = DT_COMPLEX64; + return absl::OkStatus(); + } else if (etype.isF64()) { + *dtype = DT_COMPLEX128; + return absl::OkStatus(); + } + return errors::Unimplemented( + absl::StrCat("Converting ", debugString(type), " to DataType")); + } + +#define HANDLE_TF_TYPE(tftype, enumerant, name) \ + if (llvm::isa(type)) { \ + *dtype = DT_##enumerant; \ + return OkStatus(); \ + } +// NOLINTNEXTLINE +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.def" + + return errors::Unimplemented( + absl::StrCat("Converting ", debugString(type), " to DataType")); +} + +absl::Status ConvertToDataType(Type type, DataType* dtype) { + if (auto stype = mlir::dyn_cast(type)) { + TF_RETURN_IF_ERROR( + ConvertScalarTypeToDataType(stype.getElementType(), dtype)); + } else { + TF_RETURN_IF_ERROR(ConvertScalarTypeToDataType(type, dtype)); + } + return absl::OkStatus(); +} + +void ConvertToMlirShape(const TensorShape& input_shape, + llvm::SmallVectorImpl* shape) { + shape->reserve(input_shape.dims()); + for (const auto& d : input_shape) { + shape->push_back(d.size == kTFDynamicSize ? ShapedType::kDynamic : d.size); + } +} + +absl::Status ConvertToMlirShape(const TensorShapeProto& input_shape, + llvm::SmallVectorImpl* shape) { + shape->reserve(input_shape.dim_size()); + auto& dims = input_shape.dim(); + for (auto& d : dims) { + if (d.size() > std::numeric_limits::max()) { + return errors::InvalidArgument("Shape element overflows"); + } + shape->push_back(d.size() == kTFDynamicSize ? ShapedType::kDynamic + : d.size()); + } + return absl::OkStatus(); +} + +absl::StatusOr ConvertToMlirTensorType( + const TensorShapeProto& shape, DataType dtype, mlir::Builder* builder) { + mlir::Type element_type; + TF_RETURN_IF_ERROR(ConvertDataType(dtype, *builder, &element_type)); + if (shape.unknown_rank()) { + return mlir::UnrankedTensorType::get(element_type); + } + llvm::SmallVector shape_dims; + TF_RETURN_IF_ERROR(ConvertToMlirShape(shape, &shape_dims)); + return GetTypeFromTFTensorShape(shape_dims, element_type); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.h new file mode 100644 index 0000000000000000000000000000000000000000..b25fd0ccce00d3bb4365f8ee9093a44dae32d9da --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type.h @@ -0,0 +1,54 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TYPE_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TYPE_H_ + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace tensorflow { + +using tsl::StatusOr; + +// Converts the TensorFlow DataType 'dtype' into an MLIR (scalar) type. +absl::Status ConvertDataType(DataType dtype, mlir::Builder builder, + mlir::Type* type); + +// Converts a scalar MLIR type to a TensorFlow Datatype. +absl::Status ConvertScalarTypeToDataType(mlir::Type type, DataType* dtype); + +// Converts an MLIR type to TensorFlow DataType. If 'type' is a scalar type, it +// is converted directly. If it is a shaped type, the element type is converted. +absl::Status ConvertToDataType(mlir::Type type, DataType* dtype); + +// Converts an TensorFlow shape to the one used in MLIR. +void ConvertToMlirShape(const TensorShape& input_shape, + llvm::SmallVectorImpl* shape); + +// Converts an TensorFlow shape proto to the one used in MLIR. +absl::Status ConvertToMlirShape(const TensorShapeProto& input_shape, + llvm::SmallVectorImpl* shape); + +// Given a tensor shape and dtype, get the corresponding MLIR tensor type. +absl::StatusOr ConvertToMlirTensorType( + const TensorShapeProto& shape, DataType dtype, mlir::Builder* builder); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_CONVERT_TYPE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..05c83f5b493892c6f5daf45a2496721ea231cc87 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/convert_type_test.cc @@ -0,0 +1,73 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" + +#include +#include + +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace tensorflow { +namespace { + +std::string ConvertToMlirString(const std::vector& dims, + bool unknown_rank, DataType dtype) { + TensorShapeProto shape; + shape.set_unknown_rank(unknown_rank); + for (int64_t dim : dims) { + shape.add_dim()->set_size(dim); + } + mlir::MLIRContext context; + mlir::Builder b(&context); + auto status_or = ConvertToMlirTensorType(shape, dtype, &b); + std::string buf; + llvm::raw_string_ostream os(buf); + status_or.value().print(os); + return os.str(); +} + +TEST(MlirConvertType, ConvertToMlirTensorType) { + // Simple case of static shapes. + EXPECT_EQ("tensor<4x8x16xi32>", + ConvertToMlirString({4, 8, 16}, /*unknown_rank=*/false, + DataType::DT_INT32)); + + // Partially known shapes. + EXPECT_EQ("tensor", + ConvertToMlirString({-1, 27, -1}, /*unknown_rank=*/false, + DataType::DT_BFLOAT16)); + + // Unranked shapes. + EXPECT_EQ("tensor<*xf32>", + ConvertToMlirString({}, /*unknown_rank=*/true, DataType::DT_FLOAT)); + + // int2 and uint2. + EXPECT_EQ( + "tensor<1x2xi2>", + ConvertToMlirString({1, 2}, /*unknown_rank=*/false, DataType::DT_INT2)); + EXPECT_EQ( + "tensor<1x2xui2>", + ConvertToMlirString({1, 2}, /*unknown_rank=*/false, DataType::DT_UINT2)); +} + +} // namespace + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.cc new file mode 100644 index 0000000000000000000000000000000000000000..e4c0fb7acc1891953e08c1c1f3e29fd7786b26db --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.cc @@ -0,0 +1,62 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" + +#include +#include +#include + +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" + +namespace tensorflow { +DataDumperLoggerConfig::DataDumperLoggerConfig( + std::function + get_filename, + const std::string &pass_prefix, bool print_module_scope, + bool print_after_only_on_change, mlir::OpPrintingFlags op_printing_flags) + : ::tensorflow::BridgeLoggerConfig( + print_module_scope, print_after_only_on_change, op_printing_flags), + get_filename_(get_filename), + pass_prefix_(pass_prefix) {} + +void DataDumperLoggerConfig::printBeforeIfEnabled( + mlir::Pass *pass, mlir::Operation *op, PrintCallbackFn print_callback) { + std::string pass_name = pass->getName().str(); + std::string filename = + get_filename_(pass_prefix_ + "before_" + pass_name, op); + + if (ShouldPrint(pass, op)) DumpMlir(filename, print_callback); +} + +void DataDumperLoggerConfig::printAfterIfEnabled( + mlir::Pass *pass, mlir::Operation *op, PrintCallbackFn print_callback) { + std::string pass_name = pass->getName().str(); + std::string filename = get_filename_(pass_prefix_ + "after_" + pass_name, op); + + if (ShouldPrint(pass, op)) DumpMlir(filename, print_callback); +} + +void DataDumperLoggerConfig::DumpMlir( + const std::string &filename, + BridgeLoggerConfig::PrintCallbackFn print_callback) { + std::unique_ptr os; + std::string filepath; + if (tensorflow::CreateFileForDumping(filename, &os, &filepath).ok()) { + print_callback(*os); + LOG(INFO) << "Dumped MLIR module to " << filepath; + } +} +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h new file mode 100644 index 0000000000000000000000000000000000000000..1c7c2f58b93bceffead2c451a29010e4ea93ac91 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h @@ -0,0 +1,57 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DATA_DUMPER_LOGGER_CONFIG_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DATA_DUMPER_LOGGER_CONFIG_H_ + +#include +#include + +#include "tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h" + +namespace tensorflow { + +class DataDumperLoggerConfig : public ::tensorflow::BridgeLoggerConfig { + public: + explicit DataDumperLoggerConfig( + std::function + get_filename, + const std::string &pass_prefix = "", bool print_module_scope = false, + bool print_after_only_on_change = true, + mlir::OpPrintingFlags op_printing_flags = mlir::OpPrintingFlags()); + + void printBeforeIfEnabled(mlir::Pass *pass, mlir::Operation *op, + PrintCallbackFn print_callback) override; + + void printAfterIfEnabled(mlir::Pass *pass, mlir::Operation *op, + PrintCallbackFn print_callback) override; + + private: + static void DumpMlir(const std::string &filename, + BridgeLoggerConfig::PrintCallbackFn print_callback); + + // The function to dump the target MLIR string to file. + // The parameter that will be sent to the dump_func_ is: + // The pass name (std::string) + std::function + get_filename_; + + // The pass prefix. + std::string pass_prefix_; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DATA_DUMPER_LOGGER_CONFIG_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..d243485328e52d7c15f4b5cc2949b2cc9a4d1f91 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config_test.cc @@ -0,0 +1,100 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" + +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +// Define test modules that are deserialized to module ops. +static const char *const module_with_add = + R"(module { +func.func @main(%arg0: tensor<3x4x5xf32>, %arg1: tensor<3x4x5xf32>) -> tensor<3x4x5xf32> { + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<3x4x5xf32>, tensor<3x4x5xf32>) -> tensor<3x4x5xf32> + func.return %0 : tensor<3x4x5xf32> +} +} +)"; + +// Test pass filter. +TEST(DataDumperLoggerConfig, TestPassFilter) { + mlir::DialectRegistry mlir_registry; + mlir::RegisterAllTensorFlowDialects(mlir_registry); + mlir::MLIRContext mlir_context(mlir_registry); + mlir::OwningOpRef mlir_module_with_add; + TF_ASSERT_OK(DeserializeMlirModule(module_with_add, &mlir_context, + &mlir_module_with_add)); + + std::unique_ptr partitioning_pass = + mlir::TFTPU::CreateTPUResourceReadsWritesPartitioningPass(); + std::unique_ptr shape_inference_pass = + mlir::TF::CreateTFShapeInferencePass(); + std::unique_ptr inliner_pass = mlir::createInlinerPass(); + + // partitioning_pass and shape_inference_pass should match the filter, + // inliner_pass should not. + setenv("MLIR_BRIDGE_LOG_PASS_FILTER", + "TPUResourceReadsWritesPartitioningPass;TensorFlowShapeInferencePass", + 1); + setenv("TF_DUMP_GRAPH_PREFIX", "sponge", 1); + + const std::string kTestFilename = "test.txt"; + int print_callback_count = 0; + auto get_filename_fn = [](const std::string& filename, mlir::Operation* op) { + return filename; + }; + auto print_callback = [&](llvm::raw_ostream &out) { + print_callback_count++; + return; + }; + + DataDumperLoggerConfig data_dumper_logger_config(get_filename_fn); + + data_dumper_logger_config.printBeforeIfEnabled( + partitioning_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 1); + + data_dumper_logger_config.printBeforeIfEnabled( + shape_inference_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 2); + + data_dumper_logger_config.printBeforeIfEnabled( + inliner_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 2); + + data_dumper_logger_config.printAfterIfEnabled( + partitioning_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 3); + + data_dumper_logger_config.printAfterIfEnabled( + shape_inference_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 4); + + data_dumper_logger_config.printAfterIfEnabled( + inliner_pass.get(), mlir_module_with_add.get(), print_callback); + EXPECT_EQ(print_callback_count, 4); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..69666cd52bee5c61d35845542cea68856cf75dbe --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.cc @@ -0,0 +1,136 @@ +/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "xla/status_macros.h" +#include "tensorflow/core/lib/io/inputstream_interface.h" +#include "tensorflow/core/lib/io/zlib_compression_options.h" +#include "tensorflow/core/lib/io/zlib_inputstream.h" +#include "tensorflow/core/platform/tstring.h" + +namespace tensorflow { +namespace { +// Wrap memory buffer into InputStreamInterface +class MemoryInputStream : public tensorflow::io::InputStreamInterface { + public: + explicit MemoryInputStream(const char* buffer, size_t length) + : buf_(buffer), len_(length), pos_(0) {} + + ~MemoryInputStream() override = default; + + absl::Status ReadNBytes(int64_t bytes_to_read, tstring* result) override { + result->clear(); + if (bytes_to_read < 0) { + return absl::InvalidArgumentError(absl::StrCat( + "Can't read a negative number of bytes: ", bytes_to_read)); + } + absl::Status status = absl::OkStatus(); + int64_t bytes = bytes_to_read; + if (pos_ + bytes_to_read > len_) { + bytes = len_ - pos_; + status = absl::OutOfRangeError("Reached end of file"); + } + if (bytes > 0) { + result->resize(bytes); + memcpy(&(*result)[0], &buf_[pos_], bytes); + pos_ += bytes; + } + return status; + } + + int64_t Tell() const override { return pos_; } + + absl::Status Reset() override { + pos_ = 0; + return absl::OkStatus(); + } + + private: + const char* buf_; // Not owned. + int64_t len_; + int64_t pos_ = 0; // Tracks where we are in the file. +}; +} // namespace + +absl::Status DeserializeMlirModule( + llvm::StringRef serialized_mlir_module, mlir::MLIRContext* mlir_context, + mlir::OwningOpRef* mlir_module) { + TF_RET_CHECK(!serialized_mlir_module.empty()) + << "unexpected empty serialized MLIR module string"; + TF_RET_CHECK(mlir_module) << "unexpected null MLIR module pointer"; + + // Make sure we catch any error reported by MLIR and forward it to the TF + // error reporting system. + mlir::StatusScopedDiagnosticHandler error_handler(mlir_context); + + // Look for the GZIP magic number to check if this is a compressed bytecode. + if (serialized_mlir_module.starts_with("\x1f\x8b")) { + // Try to uncompress the and parse the bytecode. + auto input_stream = std::make_unique( + serialized_mlir_module.data(), serialized_mlir_module.size()); + io::ZlibCompressionOptions options = io::ZlibCompressionOptions::GZIP(); + auto zlib_stream = std::make_unique( + input_stream.get(), options.input_buffer_size, + options.output_buffer_size, options); + tstring uncompressed_bytecode; + absl::Status s = zlib_stream->ReadNBytes(/*bytes_to_read=*/INT_MAX, + &uncompressed_bytecode); + // OK status means the decompression is successful. + // OutOfRange error means the decompression is successful but end of input + // was reached before *bytes_to_read* bytes were read. + if (!s.ok() && !absl::IsOutOfRange(s)) { + // Failed to uncompress the bytecode and it is not the end of the input. + return error_handler.Combine(absl::InvalidArgumentError( + absl::StrCat("Failed to uncompress MLIR module", s.message()))); + } + // Parse the uncompressed bytecode. + auto uncompressed_bytecode_str = + std::string(uncompressed_bytecode.data(), uncompressed_bytecode.size()); + *mlir_module = mlir::parseSourceString( + uncompressed_bytecode_str, mlir_context); + if (!*mlir_module) { + // Uncompressing was successful but the parsed MLIR module is invalid. + return error_handler.Combine(absl::InvalidArgumentError( + "Failed to parse MLIR module after uncompressing")); + } + } else { + *mlir_module = mlir::parseSourceString( + serialized_mlir_module, mlir_context); + if (!*mlir_module) { + return error_handler.Combine( + absl::InvalidArgumentError("could not parse MLIR module")); + } + } + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..8b9fdf2eb4126d65c672d347c3c41517be303236 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h @@ -0,0 +1,34 @@ +/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DESERIALIZE_MLIR_MODULE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DESERIALIZE_MLIR_MODULE_UTILS_H_ + +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project + +namespace tensorflow { + +// Deserializes a MLIR module from a string. +absl::Status DeserializeMlirModule( + llvm::StringRef serialized_mlir_module, mlir::MLIRContext* mlir_context, + mlir::OwningOpRef* mlir_module); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DESERIALIZE_MLIR_MODULE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..cf866216a1212fc0f74af82d7f5f3badca74ff8d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.cc @@ -0,0 +1,176 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" + +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Regex.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_set.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +constexpr char kDevicesAttr[] = "tf.devices"; + +namespace { + +// Parse GPU compute capability from physical device description. If compute +// capability is not found in device description, return a unit attribute. +mlir::Attribute ParseGpuDeviceMetadata(const Device& device, + mlir::Builder* builder) { + // Parse GPU device compute capability from physical device description. + static auto* r = new llvm::Regex("compute capability: ([0-9]+)\\.([0-9]+)"); + + llvm::SmallVector cc; + if (r->match(device.attributes().physical_device_desc(), &cc)) { + return mlir::TF::GpuDeviceMetadata::get( + builder->getContext(), std::stoi(cc[1].str()), std::stoi(cc[2].str())); + } + + return builder->getUnitAttr(); +} + +// Get devices from an array of string attributes. +// TODO(ezhulenev): Update all tests to use dictionary attribute for +// `tf.devices` and remove this function. +mlir::LogicalResult GetDevicesFromOp(mlir::Operation* op, + mlir::ArrayAttr array_attr, + mlir::TF::RuntimeDevices* devices) { + DeviceNameUtils::ParsedName device; + + for (const auto& kv : llvm::enumerate(array_attr)) { + const int idx = kv.index(); + + auto string_attr = mlir::dyn_cast(kv.value()); + if (!string_attr) + return op->emitOpError(llvm::formatv( + "bad '{0}' attribute at index {1}, not a string", kDevicesAttr, idx)); + + if (DeviceNameUtils::ParseFullName(string_attr.getValue().str(), &device)) { + devices->AddDevice(device); + } else { + return op->emitOpError( + llvm::formatv("bad '{0}' attribute, '{1}', not a valid device", + kDevicesAttr, string_attr.getValue())); + } + } + + return mlir::success(); +} + +// Get devices from a dictionary attribute. +mlir::LogicalResult GetDevicesFromOp(mlir::Operation* op, + mlir::DictionaryAttr dict_attr, + mlir::TF::RuntimeDevices* devices) { + DeviceNameUtils::ParsedName device; + + // Parse device names and metadata from dictionary attribute. + for (auto& kv : dict_attr) { + const mlir::StringAttr name = kv.getName(); + const mlir::Attribute attr = kv.getValue(); + + if (!DeviceNameUtils::ParseFullName(name.str(), &device)) + return op->emitOpError( + llvm::formatv("bad '{0}' attribute, '{1}', not a valid device", + kDevicesAttr, name.strref())); + + if (auto gpu_metadata = mlir::dyn_cast(attr)) { + devices->AddGpuDevice(device, gpu_metadata); + } else { + devices->AddDevice(device); + } + } + + return mlir::success(); +} + +} // namespace + +void AddDevicesToOp(mlir::Operation* op, const DeviceSet* device_set) { + if (!device_set) return; + + mlir::MLIRContext* ctx = op->getContext(); + mlir::Builder builder(ctx); + + // Collect devices with attached metadata. + llvm::SmallVector devices; + devices.reserve(device_set->devices().size()); + + // For device that do not have any metadata, or if we failed to parse metadata + // from the DeviceSet, we add a unit attribute to the `tf.devices` attribute. + for (Device* device : device_set->devices()) { + std::string name = + DeviceNameUtils::ParsedNameToString(device->parsed_name()); + + if (device->device_type() == DEVICE_GPU) { + auto metadata = ParseGpuDeviceMetadata(*device, &builder); + devices.push_back(builder.getNamedAttr(name, metadata)); + } else { + auto metadata = builder.getUnitAttr(); + devices.push_back(builder.getNamedAttr(name, metadata)); + } + } + + op->setAttr(kDevicesAttr, builder.getDictionaryAttr(devices)); +} + +mlir::LogicalResult GetDevicesFromOp(mlir::Operation* op, + mlir::TF::RuntimeDevices* devices) { + auto devices_attr = op->getAttr(kDevicesAttr); + if (!devices_attr) return mlir::success(); + + if (auto array_attr = mlir::dyn_cast(devices_attr)) { + return GetDevicesFromOp(op, array_attr, devices); + + } else if (auto dict_attr = + mlir::dyn_cast(devices_attr)) { + return GetDevicesFromOp(op, dict_attr, devices); + } + + return op->emitOpError( + llvm::formatv("unsupported '{0}' attribute", kDevicesAttr)); +} + +mlir::LogicalResult GetDeviceOrdinalFromDeviceString(mlir::Location loc, + llvm::StringRef device, + int64_t* device_ordinal) { + DeviceNameUtils::ParsedName parsed_name; + if (!DeviceNameUtils::ParseFullName( + absl::string_view(device.data(), device.size()), &parsed_name)) + return mlir::emitError(loc) << "invalid device '" << device << "'"; + + if (!parsed_name.has_id) + return mlir::emitError(loc) << "device '" << device << "' has no id"; + + *device_ordinal = parsed_name.id; + return mlir::success(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.h new file mode 100644 index 0000000000000000000000000000000000000000..32acfe07de433a04d2f1a7fc1a73541dd87a289d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util.h @@ -0,0 +1,53 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DEVICE_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DEVICE_UTIL_H_ + +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/core/common_runtime/device_set.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +// Collects all devices known to the system by name and adds them as a +// `tf.devices` dictionary attribute with a full device name as a key, and +// device metadata as a value. +// +// Device names added in full parsed device form: +// /job:/replica:/task:/device:: +// +// Supported device metadata types: +// (1) GpuDeviceMetadata: GPU device compute capability. +void AddDevicesToOp(mlir::Operation* op, const DeviceSet* device_set); + +// Collects devices information from an op `tf.devices` attributes. Returns +// failure if can't parse device metadata from the attribute. +mlir::LogicalResult GetDevicesFromOp(mlir::Operation* op, + mlir::TF::RuntimeDevices* devices); + +// Parses a device string and returns its ordinal (id). This will return an +// error if the device string is invalid or has no id. +mlir::LogicalResult GetDeviceOrdinalFromDeviceString(mlir::Location loc, + llvm::StringRef device, + int64_t* device_ordinal); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DEVICE_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..0ed7b5fdb347a5cb8dc03e0e7a1322fd0ea645c5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/device_util_test.cc @@ -0,0 +1,257 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" + +#include +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_set.h" +#include "tensorflow/core/framework/device_attributes.pb.h" +#include "tensorflow/core/ir/types/dialect.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { +namespace { + +// A fake device used to populate a DeviceSet. +class FakeDevice : public Device { + public: + explicit FakeDevice(const DeviceAttributes& device_attributes) + : Device(nullptr, device_attributes) {} + + absl::Status Sync() override { + return errors::Unimplemented("FakeDevice::Sync()"); + } + + static std::unique_ptr Make(const std::string& name, + const std::string& desc = "") { + DeviceNameUtils::ParsedName parsed_name; + DeviceNameUtils::ParseFullName(name, &parsed_name); + + DeviceAttributes device_attributes; + device_attributes.set_name(name); + device_attributes.set_device_type(parsed_name.type); + device_attributes.set_physical_device_desc(desc); + return std::make_unique(device_attributes); + } +}; + +TEST(DeviceUtilTest, AddDeviceToOp) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + const std::string cpu0 = "/job:worker/replica:0/task:0/device:CPU:0"; + const std::string gpu0 = "/job:worker/replica:1/task:2/device:GPU:0"; + const std::string gpu1 = "/job:worker/replica:1/task:2/device:GPU:1"; + + llvm::SmallVector, 2> devices; + devices.push_back(FakeDevice::Make(cpu0)); + devices.push_back(FakeDevice::Make(gpu0, "compute capability: 7.0")); + devices.push_back(FakeDevice::Make(gpu1)); + + DeviceSet device_set; + for (auto& device : devices) device_set.AddDevice(device.get()); + AddDevicesToOp(*module_ref, &device_set); + + auto devices_attr = + (*module_ref)->getAttrOfType("tf.devices"); + ASSERT_NE(devices_attr, nullptr); + ASSERT_EQ(devices_attr.size(), 3); + + // CPU device added with an empty metadata. + auto device_meta_0 = mlir::dyn_cast(devices_attr.get(cpu0)); + ASSERT_NE(device_meta_0, nullptr); + + // GPU device successfully parsed compute capability from description. + auto device_meta_1 = + mlir::dyn_cast(devices_attr.get(gpu0)); + ASSERT_NE(device_meta_1, nullptr); + ASSERT_EQ(device_meta_1.getCcMajor(), 7); + ASSERT_EQ(device_meta_1.getCcMinor(), 0); + + // If description is empty GPU devices added with an empty metadata. + auto device_meta_2 = mlir::dyn_cast(devices_attr.get(gpu1)); + ASSERT_NE(device_meta_2, nullptr); +} + +TEST(DeviceUtilTest, AddDeviceToOpNullDeviceSet) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + AddDevicesToOp(*module_ref, /*device_set=*/nullptr); + EXPECT_EQ((*module_ref)->getAttr("tf.devices"), nullptr); +} + +TEST(DeviceUtilTest, GetDevicesFromOpNoDevicesAttribute) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::succeeded(GetDevicesFromOp(*module_ref, &devices))); +} + +TEST(DeviceUtilTest, GetDevicesFromOpBadDevicesAttributeType) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::Builder builder(*module_ref); + (*module_ref)->setAttr("tf.devices", builder.getBoolAttr(false)); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::failed(GetDevicesFromOp(*module_ref, &devices))); +} + +TEST(DeviceUtilTest, GetDevicesFromOpBadDevicesAttributeArraySubtype) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::Builder builder(*module_ref); + (*module_ref)->setAttr("tf.devices", builder.getI32ArrayAttr({8})); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::failed(GetDevicesFromOp(*module_ref, &devices))); +} + +TEST(DeviceUtilTest, GetDevicesFromOpBadDevicesInDevicesAttribute) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::Builder builder(*module_ref); + (*module_ref) + ->setAttr("tf.devices", + builder.getDictionaryAttr(builder.getNamedAttr( + "bad_device", builder.getDictionaryAttr({})))); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::failed(GetDevicesFromOp(*module_ref, &devices))); +} + +TEST(DeviceUtilTest, GetDevicesFromOpValidDeviceInDevicesAttribute) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::Builder builder(*module_ref); + + auto device_dict = builder.getDictionaryAttr( + {builder.getNamedAttr("/job:worker/replica:0/task:0/device:CPU:0", + builder.getDictionaryAttr({}))}); + (*module_ref)->setAttr("tf.devices", device_dict); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::succeeded(GetDevicesFromOp(*module_ref, &devices))); + + ASSERT_EQ(devices.NumDevices(), 1); + ASSERT_EQ(devices.device_names().size(), 1); + ASSERT_EQ(DeviceNameUtils::ParsedNameToString(devices.device_names()[0]), + "/job:worker/replica:0/task:0/device:CPU:0"); +} + +TEST(DeviceUtilTest, GetGpuDeviceMetadata) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + mlir::Builder builder(*module_ref); + + const std::string gpu0 = "/job:worker/replica:0/task:0/device:GPU:0"; + const std::string gpu1 = "/job:worker/replica:0/task:0/device:GPU:1"; + + llvm::SmallVector metadata; + metadata.push_back(builder.getNamedAttr( + gpu0, mlir::TF::GpuDeviceMetadata::get(module_ref->getContext(), 1, 2))); + + (*module_ref)->setAttr("tf.devices", builder.getDictionaryAttr(metadata)); + + mlir::TF::RuntimeDevices devices; + EXPECT_TRUE(mlir::succeeded(GetDevicesFromOp(*module_ref, &devices))); + + DeviceNameUtils::ParsedName parsed_name; + DeviceNameUtils::ParseFullName(gpu0, &parsed_name); + auto meta_0 = devices.GetGpuDeviceMetadata(parsed_name); + ASSERT_TRUE(meta_0.has_value()); + ASSERT_EQ(meta_0->getCcMajor(), 1); + ASSERT_EQ(meta_0->getCcMinor(), 2); + + DeviceNameUtils::ParseFullName(gpu1, &parsed_name); + auto meta_1 = devices.GetGpuDeviceMetadata(parsed_name); + ASSERT_FALSE(meta_1.has_value()); +} + +TEST(DeviceUtilTest, GetDeviceOrdinalFromDeviceString) { + const std::string tpu0 = "/job:worker/replica:0/task:0/device:TPU:0"; + const std::string tpu1 = "/job:worker/replica:0/task:0/device:TPU:1"; + + mlir::MLIRContext context; + auto unknown_loc = mlir::UnknownLoc::get(&context); + + int64_t device_ordinal0 = -1; + mlir::LogicalResult result0 = + GetDeviceOrdinalFromDeviceString(unknown_loc, tpu0, &device_ordinal0); + EXPECT_TRUE(mlir::succeeded(result0)); + EXPECT_EQ(device_ordinal0, 0); + + int64_t device_ordinal1 = -1; + mlir::LogicalResult result1 = + GetDeviceOrdinalFromDeviceString(unknown_loc, tpu1, &device_ordinal1); + EXPECT_TRUE(mlir::succeeded(result1)); + EXPECT_EQ(device_ordinal1, 1); +} + +TEST(DeviceUtilTest, GetDeviceOrdinalFromDeviceStringInvalid) { + mlir::MLIRContext context; + auto unknown_loc = mlir::UnknownLoc::get(&context); + + int64_t device_ordinal = -1; + mlir::LogicalResult result = GetDeviceOrdinalFromDeviceString( + unknown_loc, "bad_device", &device_ordinal); + EXPECT_TRUE(mlir::failed(result)); +} + +TEST(DeviceUtilTest, GetDeviceOrdinalFromDeviceStringNoId) { + const std::string tpu_no_id = "/job:worker/replica:0/task:0/device:TPU"; + + mlir::MLIRContext context; + auto unknown_loc = mlir::UnknownLoc::get(&context); + + int64_t device_ordinal = -1; + mlir::LogicalResult result = + GetDeviceOrdinalFromDeviceString(unknown_loc, tpu_no_id, &device_ordinal); + EXPECT_TRUE(mlir::failed(result)); +} + +} // anonymous namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.cc new file mode 100644 index 0000000000000000000000000000000000000000..ad7a266cce6014911efced200537030da3593ab9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.cc @@ -0,0 +1,109 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_graph.h" + +#include +#include +#include +#include + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/core/ir/importexport/graphdef_import.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/path.h" +#include "tensorflow/core/util/dump_graph.h" + +namespace tensorflow { + +namespace { + +// Simple raw_ostream that prints to a file (doesn't take ownership). +struct WritableFileRawStream : public llvm::raw_ostream { + explicit WritableFileRawStream(WritableFile* file) : file(file) { + SetUnbuffered(); + } + ~WritableFileRawStream() override = default; + uint64_t current_pos() const override { return 0; } + + void write_impl(const char* ptr, size_t size) override { + // If an error is encountered, null out the file. + if (file) { + absl::Status s = file->Append(absl::string_view(ptr, size)); + if (!s.ok()) { + LOG(WARNING) << "Write failed: " << s; + file = nullptr; + } + } + } + + // The file being written to. + WritableFile* file; +}; +} // namespace + +absl::Status DumpTextualIRToFile(const MlirDumpConfig& config, + const Graph& graph, + const FunctionLibraryDefinition* flib_def, + WritableFile* file) { + WritableFileRawStream os(std::move(file)); + mlir::MLIRContext context; + mlir::OwningOpRef module; + if (flib_def) { + flib_def = &graph.flib_def(); + } + auto convert = [&]() -> absl::Status { + mlir::StatusScopedDiagnosticHandler status_handler(&context); + // TODO(jpienaar): Both the graph debug info and import config should be + // specifiable. + GraphDebugInfo debug_info; + switch (config.dialect) { + case MlirDumpConfig::Dialect::kTFG: { + TF_ASSIGN_OR_RETURN(module, + mlir::tfg::ImportGraphAndFunctionsToMlir( + &context, debug_info, graph, + flib_def ? *flib_def : graph.flib_def())); + break; + } + } + if (failed(mlir::verify(*module))) { + return status_handler.ConsumeStatus(); + } + return status_handler.ConsumeStatus(); + }; + + TF_RETURN_IF_ERROR(convert()); + module->print(os, config.op_printing_flags); + return absl::OkStatus(); +} + +void UseMlirForGraphDump(const MlirDumpConfig& config) { + SetGraphDumper( + [config](const Graph& graph, const FunctionLibraryDefinition* flib_def, + WritableFile* file) -> absl::Status { + return DumpTextualIRToFile(config, graph, flib_def, file); + }, + /*suffix=*/".mlir"); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.h new file mode 100644 index 0000000000000000000000000000000000000000..4cdc6ea8ae4a1f91ee0d949f7491626ebcc95743 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph.h @@ -0,0 +1,76 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_GRAPH_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_GRAPH_H_ + +#include +#include + +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/platform/status.h" + +namespace tensorflow { + +struct MlirDumpConfig; + +// Dumps 'graph_def' to a file, as textual IR. Returns the file name chosen. +// +// Note: This is for debugging use and is not optimized for performance. +absl::Status DumpTextualIRToFile(const MlirDumpConfig& config, + const Graph& graph, + const FunctionLibraryDefinition* flib_def, + WritableFile* file); + +// Config of the textual dump. +struct MlirDumpConfig { + enum class Dialect { + // Tensorflow Graph Dialect + kTFG, + }; + + // The limit of element size that gets printed. + MlirDumpConfig& elide_large_attributes(int large_element_limit = 16) { + this->op_printing_flags.elideLargeElementsAttrs(large_element_limit); + return *this; + } + + // Enable printing of debug information. If 'pretty_form' is set to true, + // debug information is printed in a more readable 'pretty' form but this + // pretty form is not parsable (so only for human readability). + MlirDumpConfig& emit_location_information(bool pretty_form = false) { + this->op_printing_flags.enableDebugInfo(/*enable=*/true, pretty_form); + return *this; + } + + MlirDumpConfig& emit_dialect(Dialect dialect) { + this->dialect = dialect; + return *this; + } + + // Op printing flags. + mlir::OpPrintingFlags op_printing_flags = {}; + + // The target MLIR dialect. + Dialect dialect = Dialect::kTFG; +}; + +// Change DumpGraphToFile to dump MLIR textual IR instead of protobuf. +void UseMlirForGraphDump(const MlirDumpConfig& = {}); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_GRAPH_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..56a63f58f2aac5c9a0db49c79361d9ef2482d0e4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_graph_test.cc @@ -0,0 +1,129 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_graph.h" + +#include +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/file_system.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/util/dump_graph.h" + +namespace tensorflow { +namespace { + +void ExpectHasSubstr(const std::string& s, const std::string& expected) { + EXPECT_TRUE(absl::StrContains(s, expected)) + << "'" << s << "' does not contain '" << expected << "'"; +} + +void ExpectHasNoSubstr(const std::string& s, const std::string& expected) { + EXPECT_FALSE(absl::StrContains(s, expected)) + << "'" << s << "' should not contain '" << expected << "'"; +} + +// WritableFile that simply concats into string. +class StringWritableFile : public WritableFile { + public: + explicit StringWritableFile(std::string* str) : str_(*str) {} + + absl::Status Append(absl::string_view data) override { + absl::StrAppend(&str_, data); + return absl::OkStatus(); + } + + absl::Status Close() override { return absl::OkStatus(); } + + absl::Status Flush() override { return absl::OkStatus(); } + + absl::Status Name(absl::string_view* result) const override { + *result = "(string)"; + return absl::OkStatus(); + } + + absl::Status Sync() override { return absl::OkStatus(); } + + absl::Status Tell(int64_t* position) override { + return errors::Unimplemented("Stream not seekable"); + } + + private: + std::string& str_; +}; + +TEST(Dump, TextualIrToFileSuccess) { + Graph graph(OpRegistry::Global()); + Node* node; + CHECK_OK(NodeBuilder("A", "NoOp").Finalize(&graph, &node)); + + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + UseMlirForGraphDump(MlirDumpConfig()); + std::string ret = DumpGraphToFile("tir", graph); + ASSERT_EQ(ret, io::JoinPath(testing::TmpDir(), "tir.mlir")); + + std::string actual; + TF_ASSERT_OK(ReadFileToString(Env::Default(), ret, &actual)); +} + +TEST(Dump, TextualIrWithOptions) { + Graph graph(OpRegistry::Global()); + Node* node; + TF_ASSERT_OK(NodeBuilder("A", "Placeholder") + .Attr("dtype", DT_FLOAT) + .Finalize(&graph, &node)); + + std::string actual; + StringWritableFile file(&actual); + TF_ASSERT_OK(DumpTextualIRToFile(MlirDumpConfig().emit_location_information(), + graph, /*flib_def=*/nullptr, &file)); + + std::string expected_substr = R"(loc(#loc))"; + ExpectHasSubstr(actual, expected_substr); +} + +TEST(Dump, DumpToTFG) { + Graph graph(OpRegistry::Global()); + Node* node; + CHECK_OK(NodeBuilder("A", "NoOp").Finalize(&graph, &node)); + + std::string actual; + StringWritableFile file(&actual); + + TF_ASSERT_OK(DumpTextualIRToFile( + MlirDumpConfig().emit_dialect(MlirDumpConfig::Dialect::kTFG), graph, + /*flib_def=*/nullptr, &file)); + + std::string expected_substr("tfg.graph"); + ExpectHasSubstr(actual, expected_substr); + + std::string not_expected_substr("tf_executor.island"); + ExpectHasNoSubstr(actual, not_expected_substr); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..ca66f6b0d90ba14d7ae02de09cf3b07190b8b0f1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc @@ -0,0 +1,343 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" + +#include +#include +#include +#include +#include + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "xla/tsl/lib/io/buffered_file.h" +#include "tensorflow/core/platform/crash_analysis.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/path.h" + +using llvm::raw_ostream; + +namespace tensorflow { +namespace { + +struct NameCounts { + mutex counts_mutex; + llvm::StringMap counts; +}; + +std::string MakeUniqueFilename(std::string name) { + static NameCounts& instance = *new NameCounts; + + // Remove illegal characters from `name`. + for (int i = 0, e = name.size(); i < e; ++i) { + char ch = name[i]; + if (ch == '/' || ch == '[' || ch == ']' || ch == '*' || ch == '?' || + ch == '\\') { + name[i] = '_'; + } + } + + int count; + { + mutex_lock lock(instance.counts_mutex); + count = instance.counts[name]++; + } + + std::string filename = name; + if (count > 0) { + filename = llvm::formatv("{0}_{1}", filename, count).str(); + } + filename = llvm::Twine(filename).concat(".mlir").str(); + return filename; +} + +// Simple raw_ostream that prints to stderr. +struct LogInfoRawStream : public llvm::raw_ostream { + LogInfoRawStream() { SetUnbuffered(); } + ~LogInfoRawStream() override = default; + uint64_t current_pos() const override { return 0; } + + void write_impl(const char* ptr, size_t size) override { + fprintf(stderr, "%.*s", static_cast(size), ptr); + } +}; + +// Simple raw_ostream that prints to a file. +struct WritableFileRawStream : public llvm::raw_ostream { + explicit WritableFileRawStream(std::unique_ptr file) + : file(std::move(file)) { + SetUnbuffered(); + } + ~WritableFileRawStream() override = default; + + uint64_t current_pos() const override { + int64_t position; + if (file->Tell(&position).ok()) { + return position; + } else { + // MLIR uses os.tell() to determine whether something was written by + // a subroutine or not, so it's important we have a working current_pos(). + LOG(WARNING) + << "Couldn't query file position. Stream might be malformed.\n"; + return -1; + } + } + + void write_impl(const char* ptr, size_t size) override { + // Write the file if it is still valid. If the write fails, null out the + // file to avoid encountering another error. + if (file && !file->Append(absl::string_view(ptr, size)).ok()) { + file = nullptr; + } + } + + // The file being written to. + std::unique_ptr file; +}; + +struct CrashReproducerStream : public mlir::ReproducerStream { + CrashReproducerStream(llvm::StringRef name, + std::unique_ptr file) + : name(name), ostream(std::move(file)) {} + + llvm::StringRef description() override { return name; } + raw_ostream& os() override { return *ostream; } + + private: + std::string name; + std::unique_ptr ostream; +}; + +// MLIR crash reproducer which reports failures to the crash analysis system. +struct CrashAnalysisCrashReproducerStream : public mlir::ReproducerStream { + public: + CrashAnalysisCrashReproducerStream() + : internal_str(""), string_stream(internal_str) {} + + ~CrashAnalysisCrashReproducerStream() override { + crash_analysis::ReportEvent( + "mlir_crash_reproducer.mlir", + "Pass pipeline failure; crash reproducer attached", + string_stream.str()); + } + + llvm::StringRef description() override { return "mlir_crash_reproducer"; } + raw_ostream& os() override { return string_stream; } + + private: + std::string internal_str; + llvm::raw_string_ostream string_stream; +}; + +} // namespace + +absl::Status CreateFileForDumping(llvm::StringRef name, + std::unique_ptr* os, + std::string* filepath, + llvm::StringRef dirname) { + std::string dir; + if (!dirname.empty()) + dir = std::string(dirname); + else + dir = GetDumpDirFromEnvVar(); + + if (dir.empty()) { + return absl::Status(absl::StatusCode::kInvalidArgument, + "(TF_DUMP_GRAPH_PREFIX not specified)"); + } + + if (dir == kCrashReproducerStdErr) { + *os = std::make_unique(); + *filepath = + llvm::formatv("(stderr; requested filename: '{0}')", name).str(); + return absl::Status(); + } + + // Get a valid file path to dump with. + Env* env = Env::Default(); + absl::Status status = env->RecursivelyCreateDir(dir); + if (!status.ok()) { + LOG(WARNING) << "Failed to create '" << dir + << "' directory for dumping: " << status; + return absl::Status(absl::StatusCode::kUnavailable, "(unavailable)"); + } + *filepath = io::JoinPath(dir, MakeUniqueFilename(std::string(name))); + + // Try to open the file and generate a raw_ostream. + std::unique_ptr file; + status = env->NewWritableFile(*filepath, &file); + if (!status.ok()) { + LOG(WARNING) << "Failed to create file '" << filepath << "': " << status; + return absl::Status(absl::StatusCode::kUnavailable, "(unavailable)"); + } + file = std::make_unique(std::move(file)); + *os = std::make_unique(std::move(file)); + return absl::Status(); +} + +// Prints the pass pipeline of `pass_manager` to `os`. +void PrintPassPipeline(const mlir::PassManager& pass_manager, + mlir::Operation* op, llvm::raw_ostream& os) { + std::string str; + llvm::raw_string_ostream passOS(str); + llvm::interleaveComma( + pass_manager.getPasses(), passOS, + [&](mlir::Pass& pass) { pass.printAsTextualPipeline(passOS); }); + os << "{-# external_resources: { mlir_reproducer: { pipeline: " + "\"builtin.module(" + << passOS.str() << ")\", "; + os << "disable_threading: true, "; + os << "verify_each: true } } #-}"; + os << "\n\n"; +} + +std::string DumpMlirOpToFile(llvm::StringRef name, mlir::Operation* op, + llvm::StringRef dirname, + const mlir::PassManager* pass_manager) { + std::unique_ptr os; + std::string filepath; + absl::Status result = CreateFileForDumping(name, &os, &filepath, dirname); + if (!result.ok()) return std::string(result.message()); + + LOG(INFO) << "Dumping MLIR operation '" << op->getName().getStringRef().str() + << "' to '" << filepath << "'"; + if (pass_manager) PrintPassPipeline(*pass_manager, op, *os); + op->print(*os, mlir::OpPrintingFlags().useLocalScope()); + return filepath; +} + +std::string GetDumpDirFromEnvVar() { + const char* prefix_env = getenv("TF_DUMP_GRAPH_PREFIX"); + if (!prefix_env) { + LOG(WARNING) + << "Failed to dump MLIR module because dump location is not " + << "specified through TF_DUMP_GRAPH_PREFIX environment variable."; + return ""; + } + + std::string result = prefix_env; + + if (absl::EqualsIgnoreCase(result, "sponge") && + !io::GetTestUndeclaredOutputsDir(&result)) { + LOG(WARNING) << "TF_DUMP_GRAPH_PREFIX=sponge but " + "TEST_UNDECLARED_OUTPUT_DIRS is not set"; + return ""; + } + return result; +} + +std::string DumpRawStringToFile(llvm::StringRef name, llvm::StringRef content, + llvm::StringRef dirname) { + std::unique_ptr os; + std::string filepath; + absl::Status result = CreateFileForDumping(name, &os, &filepath, dirname); + if (!result.ok()) return std::string(result.message()); + + (*os) << content; + LOG(INFO) << "Outputted requested string to '" << filepath << "'"; + return filepath; +} + +void SetCrashReproducer(mlir::PassManager& pm, llvm::StringRef dir_path) { + std::string path = dir_path.str(); + if (path.empty() || path == kCrashReproducerCrashAnalysis) { + if (getenv("MLIR_CRASH_REPRODUCER_DIRECTORY")) + path = getenv("MLIR_CRASH_REPRODUCER_DIRECTORY"); + else if (getenv("TEST_UNDECLARED_OUTPUTS_DIR")) + path = "sponge"; + } + if (path.empty()) { + LOG_FIRST_N(INFO, 1) << "disabling MLIR crash reproducer, set env var " + "`MLIR_CRASH_REPRODUCER_DIRECTORY` to enable."; + return; + } + + // Output dirs "sponge" (case-insensitive) have a special meaning: Dump into + // the directory specified by the environment variable + // TEST_UNDECLARED_OUTPUTS_DIR. + std::string lower_path = absl::AsciiStrToLower(path); + if (lower_path == "sponge") { + if (!tensorflow::io::GetTestUndeclaredOutputsDir(&path)) { + LOG(ERROR) << "MLIR crash reproducer is set to '" << dir_path.str() + << "', but environment variable TEST_UNDECLARED_OUTPUTS_DIR " + "is not set, so cannot dump anywhere."; + return; + } + } + + // kCrashReproducerStdErr and kCrashReproducerCrashAnalysis settings do not + // require explicit file creation. + if (path != kCrashReproducerStdErr && path != kCrashReproducerCrashAnalysis) { + auto* env = tensorflow::Env::Default(); + auto status = env->RecursivelyCreateDir(path); + if (!status.ok()) { + LOG(WARNING) << "cannot create directory '" << path + << "': " << status.message(); + return; + } + + path += "/mlir_reproducer_"; + + if (!tensorflow::Env::Default()->CreateUniqueFileName(&path, ".mlir")) { + LOG(WARNING) << "cannot create unique filename, won't enable MLIR crash " + "reproducer."; + return; + } + } + + mlir::ReproducerStreamFactory factory = + [path](std::string& error) -> std::unique_ptr { + if (path == kCrashReproducerStdErr) + return std::make_unique( + "(stderr)", std::make_unique()); + if (path == kCrashReproducerCrashAnalysis) { + return std::make_unique(); + } + + // Try to open the file and generate a raw_ostream. + std::unique_ptr file; + absl::Status status = + tensorflow::Env::Default()->NewWritableFile(path, &file); + file = std::make_unique(std::move(file)); + + if (!status.ok()) { + error = absl::StrCat("Failed to create file '", path, + "': ", status.message()); + return nullptr; + } + return std::make_unique( + path, std::make_unique(std::move(file))); + }; + pm.enableCrashReproducerGeneration(factory, /*genLocalReproducer=*/false); +} + +void applyTensorflowAndCLOptions(mlir::PassManager& pm, + llvm::StringRef dir_path) { + mlir::registerPassManagerCLOptions(); + if (!mlir::succeeded(mlir::applyPassManagerCLOptions(pm))) { + LOG(ERROR) << "cannot apply MLIR pass manager CL options"; + return; + } + SetCrashReproducer(pm, dir_path); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h new file mode 100644 index 0000000000000000000000000000000000000000..a683bee779fd2a1f9201db454646e9bba3139a1c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h @@ -0,0 +1,105 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_MLIR_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_MLIR_UTIL_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/core/platform/status.h" + +namespace tensorflow { + +inline constexpr absl::string_view kCrashReproducerStdErr = "-"; +inline constexpr absl::string_view kCrashReproducerCrashAnalysis = + "crash_analysis"; + +// Creates a file to use for dumping and returns success if a file could be +// created. The opened file is placed in 'os' and the path of the file used is +// placed in 'filepath'. +// +// If the TF_DUMP_GRAPH_PREFIX environment variable is kCrashReproducerStdErr, +// then the LOG(INFO) macro is used instead. +// +// This will create a file name via prefixing `name` with the value of the +// TF_DUMP_GRAPH_PREFIX environment variable if `dirname` is empty and +// suffixing `name` with ".mlir". +absl::Status CreateFileForDumping(llvm::StringRef name, + std::unique_ptr* os, + std::string* filepath, + llvm::StringRef dirname = ""); + +// Dumps MLIR operation to a file and returns the file name used. +// +// If the TF_DUMP_GRAPH_PREFIX environment variable is kCrashReproducerStdErr, +// then the MLIR operation will be logged (using the LOG(INFO) macro) instead. +// +// This will create a file name via prefixing `name` with the value of the +// TF_DUMP_GRAPH_PREFIX environment variable if `dirname` is empty and +// suffixing `name` with ".mlir". +// If `pass_manager` is provided, prints a header with the pass pipeline. +std::string DumpMlirOpToFile(llvm::StringRef name, mlir::Operation* op, + llvm::StringRef dirname = "", + const mlir::PassManager* pass_manager = nullptr); + +// Reads the directory to dump the MLIR module from environment variables. +// Default is reading from TF_DUMP_GRAPH_PREFIX, and if the string is 'sponge' +// read from TEST_UNDECLARED_OUTPUTS_DIR. Returns nullptr if the directory +// cannot be determined and generates a warning message. +std::string GetDumpDirFromEnvVar(); + +// Dumps a raw string to a file and returns the file name used. +// +// This will create a file name via prefixing `name` with the value of the +// TF_DUMP_GRAPH_PREFIX environment variable if `dirname` is empty and +// suffixing `name` with ".mlir". +std::string DumpRawStringToFile(llvm::StringRef name, llvm::StringRef content, + llvm::StringRef dirname = ""); + +// Enable the crash reproducer on the provided PassManager to the provided +// directory path. +// If the provided path is empty, it is retrieved from the +// environment variable `MLIR_CRASH_REPRODUCER_DIRECTORY`. +// If the provided path is the string "sponge", the file will be included +// in the sponge "Output Files" by looking up the environment to infer +// the directory path. +// If the provided path is the string kCrashReproducerStdErr, the data is +// dumped into the stderr. +// If the provided path is the string kCrashReproducerCrashAnalysis, the data +// is dumped to the crash analysis system. Note, environment var +// `MLIR_CRASH_REPRODUCER_DIRECTORY` can be used to override +// kCrashReproducerCrashAnalysis settings. +void SetCrashReproducer(mlir::PassManager& pm, llvm::StringRef dir_path = ""); + +// This applies both the PassManagerCLOptions provided by MLIR along with any +// tensorflow specific options. +// +// Note that this function should be in a more appropriate file, but it is +// unclear what a proper file would be as no other functions would currently be +// in the file also. +void applyTensorflowAndCLOptions(mlir::PassManager& pm, + llvm::StringRef dir_path = ""); + +// Prints the pass pipeline of `pass_manager` to `os`. +void PrintPassPipeline(const mlir::PassManager& pass_manager, + mlir::Operation* op, llvm::raw_ostream& os); +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DUMP_MLIR_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a3e8eeef6b888ec0bb200438ee3aa185b78c18f8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util_test.cc @@ -0,0 +1,151 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" + +#include + +#include +#include +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/InitAllPasses.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "mlir/Tools/mlir-opt/MlirOptMain.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/bridge.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/core/framework/device.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +using ::testing::IsNull; + +TEST(DumpMlirModuleTest, NoEnvPrefix) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + unsetenv("TF_DUMP_GRAPH_PREFIX"); + + std::string filepath = DumpMlirOpToFile("module", module_ref.get()); + EXPECT_EQ(filepath, "(TF_DUMP_GRAPH_PREFIX not specified)"); +} + +TEST(DumpMlirModuleTest, LogInfo) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + setenv("TF_DUMP_GRAPH_PREFIX", "-", 1); + + std::string filepath = DumpMlirOpToFile("module", module_ref.get()); + EXPECT_EQ(filepath, "(stderr; requested filename: 'module')"); +} + +TEST(DumpMlirModuleTest, Valid) { + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + std::string expected_txt_module; + { + llvm::raw_string_ostream os(expected_txt_module); + module_ref->getOperation()->print(os, + mlir::OpPrintingFlags().useLocalScope()); + os.flush(); + } + + std::string filepath = DumpMlirOpToFile("module", module_ref.get()); + ASSERT_NE(filepath, "(TF_DUMP_GRAPH_PREFIX not specified)"); + ASSERT_NE(filepath, "LOG(INFO)"); + ASSERT_NE(filepath, "(unavailable)"); + + Env* env = Env::Default(); + std::string file_txt_module; + TF_ASSERT_OK(ReadFileToString(env, filepath, &file_txt_module)); + EXPECT_EQ(file_txt_module, expected_txt_module); +} + +TEST(DumpCrashReproducerTest, RoundtripDumpAndReadValid) { + mlir::registerPassManagerCLOptions(); + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + std::string filepath = + testing::TmpDir() + "/" + mlir::TF::kStandardPipelineBefore + ".mlir"; + + std::string output_dump = testing::TmpDir() + "/" + "output_dump.txt"; + + TF_ASSERT_OK(mlir::TF::RunBridgeWithStandardPipeline( + module_ref.get(), + /*enable_logging=*/true, /*enable_inliner=*/false)); + + std::string errorMessage; + auto input_file = mlir::openInputFile(filepath, &errorMessage); + EXPECT_THAT(input_file, Not(IsNull())); + + auto output_stream = mlir::openOutputFile(output_dump, &errorMessage); + EXPECT_THAT(output_stream, Not(IsNull())); + + mlir::PassPipelineCLParser passPipeline( + /*arg=*/"", /*description=*/"Compiler passes to run", /*alias=*/"p"); + mlir::DialectRegistry registry; + mlir::registerAllDialects(registry); + mlir::RegisterAllTensorFlowDialects(registry); + + mlir::registerAllPasses(); + mlir::registerTensorFlowPasses(); + + EXPECT_TRUE(mlir::MlirOptMain(output_stream->os(), std::move(input_file), + registry, + mlir::MlirOptMainConfig{} + .splitInputFile("") + .verifyPasses(false) + .allowUnregisteredDialects(false) + .setPassPipelineParser(passPipeline)) + .succeeded()); +} + +TEST(DumpRawStringToFileTest, Valid) { + llvm::StringRef example = "module {\n}"; + setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1); + + std::string filepath = DumpRawStringToFile("example", example); + ASSERT_NE(filepath, "(TF_DUMP_GRAPH_PREFIX not specified)"); + ASSERT_NE(filepath, "LOG(INFO)"); + ASSERT_NE(filepath, "(unavailable)"); + + Env* env = Env::Default(); + std::string file_txt_module; + TF_ASSERT_OK(ReadFileToString(env, filepath, &file_txt_module)); + EXPECT_EQ(file_txt_module, example); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..0f64d66c3d1bfed119782c50026ffffbaa382d41 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.cc @@ -0,0 +1,42 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" + +#include "llvm/ADT/SmallVector.h" + +namespace tensorflow { + +llvm::SmallVector ConvertTFShapeToMlir( + llvm::ArrayRef shapes) { + return llvm::to_vector(llvm::map_range(shapes, [](int64_t shape) { + return shape == kTFDynamicSize ? mlir::ShapedType::kDynamic : shape; + })); +} + +llvm::SmallVector ConvertMlirShapeToTF( + llvm::ArrayRef shapes) { + return llvm::to_vector(llvm::map_range(shapes, [](int64_t shape) { + return mlir::ShapedType::isDynamic(shape) ? kTFDynamicSize : shape; + })); +} + +mlir::RankedTensorType GetTypeFromTFTensorShape(llvm::ArrayRef shape, + mlir::Type elementType, + mlir::Attribute encoding) { + return mlir::RankedTensorType::get(ConvertTFShapeToMlir(shape), elementType, + encoding); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..0266bc952ca7a49db36d344c8be7b0f1f0f14fc4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h @@ -0,0 +1,34 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DYNAMIC_SHAPE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DYNAMIC_SHAPE_UTILS_H_ + +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project + +namespace tensorflow { + +llvm::SmallVector ConvertTFShapeToMlir(llvm::ArrayRef shapes); + +llvm::SmallVector ConvertMlirShapeToTF(llvm::ArrayRef shape); + +static constexpr int64_t kTFDynamicSize = -1; +mlir::RankedTensorType GetTypeFromTFTensorShape(llvm::ArrayRef shape, + mlir::Type elementType, + mlir::Attribute encoding = {}); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_DYNAMIC_SHAPE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..14052b3f14a56825b603e2b22d3883725fc50abf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.cc @@ -0,0 +1,60 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +#include + +#include "absl/status/status.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/util/managed_stack_trace.h" + +namespace mlir { + +StatusScopedDiagnosticHandler::StatusScopedDiagnosticHandler( + MLIRContext* context, bool propagate, bool filter_stack) + : BaseScopedDiagnosticHandler(context, propagate, filter_stack) { + if (filter_stack) { + this->shouldShowLocFn = [](Location loc) -> bool { + // For a Location to be surfaced in the stack, it must evaluate to true. + // For any Location that is a FileLineColLoc: + if (FileLineColLoc fileLoc = mlir::dyn_cast(loc)) { + return !tensorflow::IsInternalFrameForFilename( + fileLoc.getFilename().str()); + } else { + // If this is a non-FileLineColLoc, go ahead and include it. + return true; + } + }; + } + + setHandler([this](Diagnostic& diag) { return this->handler(&diag); }); +} + +Status StatusScopedDiagnosticHandler::ConsumeStatus() { + return BaseScopedDiagnosticHandler::ConsumeStatus(); +} + +Status StatusScopedDiagnosticHandler::Combine(Status status) { + absl::Status absl_s = BaseScopedDiagnosticHandler::Combine(status); + + return absl_s; +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.h new file mode 100644 index 0000000000000000000000000000000000000000..a4e4508d80d2451bd9df389986be3df7de8e45c6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util.h @@ -0,0 +1,63 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ERROR_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ERROR_UTIL_H_ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "xla/mlir/utils/error_util.h" +#include "tensorflow/core/platform/status.h" + +// Error utilities for MLIR when interacting with code using Status returns. +namespace mlir { + +// TensorFlow's Status is used for error reporting back to callers. +using ::tensorflow::Status; + +// TF customized diagnostic handler that collects all the diagnostics reported +// and can produce a Status to return to callers. This is for the case where +// MLIR functions are called from a function that will return a Status: MLIR +// code still uses the default error reporting, and the final return function +// can return the Status constructed from the diagnostics collected. +// todo: [b/253331656]. Note ConsumeStatus() and Combine() are wrappers +// of what is inherited from the BaseScopedDiagnosticHandler to +// support cases where tensorflow::Status is still being used (base class uses +// absl::Status) +class StatusScopedDiagnosticHandler : public BaseScopedDiagnosticHandler { + public: + // Constructs a diagnostic handler in a context. If propagate is true, then + // diagnostics reported are also propagated back to the original diagnostic + // handler. If filter_stack is true, a reduced stack will be produced. + + explicit StatusScopedDiagnosticHandler(MLIRContext* context, + bool propagate = false, + bool filter_stack = false); + + ~StatusScopedDiagnosticHandler() = default; + // Returns Status corresponding to the diagnostics reported. This consumes + // the diagnostics reported and returns a Status of type Unknown. It is + // required to consume the error status, if there is one, before destroying + // the object. + Status ConsumeStatus(); + + // Returns the combination of the passed in status and consumed diagnostics. + // This consumes the diagnostics reported and either appends the diagnostics + // to the error message of 'status' (if 'status' is already an error state), + // or returns an Unknown status (if diagnostics reported), otherwise OK. + Status Combine(Status status); +}; +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_ERROR_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..9bddcd1a180e17be5ae392731ea74c902ddfe17f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/error_util_test.cc @@ -0,0 +1,174 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "xla/hlo/testlib/test.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" + +namespace mlir { +namespace { + +using ::testing::HasSubstr; + +class ErrorUtilTest : public ::testing::Test { + protected: + ErrorUtilTest() + : id_(StringAttr::get(&context_, "//tensorflow/python/test.py")), + loc_(FileLineColLoc::get(&context_, id_, 0, 0)) {} + + MLIRContext context_; + StringAttr id_; + FileLineColLoc loc_; +}; + +using StatusScopedDiagnosticHandlerTest = ErrorUtilTest; + +TEST_F(StatusScopedDiagnosticHandlerTest, + OkWithoutDiagnosticGetsPassedThrough) { + TF_ASSERT_OK( + StatusScopedDiagnosticHandler(&context_).Combine(absl::OkStatus())); +} + +TEST_F(StatusScopedDiagnosticHandlerTest, + VerifyDiagnosticsAreCapturedAsUnknownStatus) { + StatusScopedDiagnosticHandler handler(&context_); + emitError(loc_) << "Diagnostic message"; + ASSERT_TRUE(absl::IsUnknown(handler.ConsumeStatus())); +} + +TEST_F(StatusScopedDiagnosticHandlerTest, VerifyPassedInErrorsArePropagated) { + const Status err = tensorflow::errors::Internal("Passed in error"); + ASSERT_TRUE( + absl::IsInternal(StatusScopedDiagnosticHandler(&context_).Combine(err))); +} + +TEST_F(StatusScopedDiagnosticHandlerTest, + VerifyThatReportedDiagnosticsAreAppendedToPassedInError) { + StatusScopedDiagnosticHandler ssdh(&context_); + emitError(loc_) << "Diagnostic message reported"; + emitError(loc_) << "Second diagnostic message reported"; + const Status s = + ssdh.Combine(tensorflow::errors::Internal("Passed in error")); + ASSERT_TRUE(absl::IsInternal(s)); + EXPECT_THAT(s.message(), HasSubstr("Passed in error")); + EXPECT_THAT(s.message(), HasSubstr("Diagnostic message reported")); + EXPECT_THAT(s.message(), HasSubstr("Second diagnostic message reported")); +} + +TEST_F(StatusScopedDiagnosticHandlerTest, VerifyThatWarningsAreIgnored) { + // Note: this logic is actually implemented in BaseScopedDiagnosticHandler's + // handler() function, but only StatusScopedDiagnosticHandler uses it. + StatusScopedDiagnosticHandler handler(&context_); + emitWarning(loc_) << "Warning message"; + TF_EXPECT_OK(handler.ConsumeStatus()); +} + +TEST(ErrorUtilTest, StatusScopedDiagnosticHandlerWithFilter) { + // Filtering logic is based on tensorflow::IsInternalFrameForFilename() + // Note we are surfacing the locations that are NOT internal frames + // so locations that fail IsInternalFrameForFilename() evaluation pass the + // filter. + + // These locations will fail the IsInternalFrameForFilename() check so will + // pass the filter. + MLIRContext context; + auto id = + StringAttr::get(&context, "//tensorflow/python/keras/keras_file.py"); + auto loc = FileLineColLoc::get(&context, id, 0, 0); + auto id2 = + StringAttr::get(&context, "//tensorflow/python/something/my_test.py"); + auto loc2 = FileLineColLoc::get(&context, id2, 0, 0); + auto id3 = StringAttr::get(&context, "python/tensorflow/show_file.py"); + auto loc3 = FileLineColLoc::get(&context, id3, 0, 0); + + // These locations will be evalauted as internal frames, passing the + // IsInternalFramesForFilenames() check so will be filtered out. + auto id_filtered = + StringAttr::get(&context, "//tensorflow/python/dir/filtered_file_A.py"); + auto loc_filtered = FileLineColLoc::get(&context, id_filtered, 0, 0); + auto id_filtered2 = + StringAttr::get(&context, "dir/tensorflow/python/filtered_file_B.py"); + auto loc_filtered2 = FileLineColLoc::get(&context, id_filtered2, 0, 0); + + // Build a small stack for each error; the MLIR diagnostic filtering will + // surface a location that would otherwise be filtered if it is the only + // location associated with an error; therefore we need a combinatination of + // locations to test. + auto callsite_loc = mlir::CallSiteLoc::get(loc, loc_filtered); + auto callsite_loc2 = mlir::CallSiteLoc::get(loc2, loc_filtered2); + auto callsite_loc3 = mlir::CallSiteLoc::get(loc_filtered2, loc3); + + // Test with filter on. + StatusScopedDiagnosticHandler ssdh_filter(&context, false, true); + emitError(callsite_loc) << "Error 1"; + emitError(callsite_loc2) << "Error 2"; + emitError(callsite_loc3) << "Error 3"; + Status s_filtered = ssdh_filter.ConsumeStatus(); + // Check for the files that should not be filtered. + EXPECT_THAT(s_filtered.message(), HasSubstr("keras")); + EXPECT_THAT(s_filtered.message(), HasSubstr("test.py")); + EXPECT_THAT(s_filtered.message(), HasSubstr("show_file")); + // Verify the filtered files are not present. + EXPECT_THAT(s_filtered.message(), Not(HasSubstr("filtered_file"))); +} + +TEST(ErrorUtilTest, StatusScopedDiagnosticHandlerWithoutFilter) { + // Filtering logic should be off so all files should 'pass'. + MLIRContext context; + // This file would pass the filter if it was on. + auto id = + StringAttr::get(&context, "//tensorflow/python/keras/keras_file.py"); + auto loc = FileLineColLoc::get(&context, id, 0, 0); + + // The '_filtered' locations would be evaluated as internal frames, so would + // not pass the filter if it was on. + auto id_filtered = + StringAttr::get(&context, "//tensorflow/python/dir/filtered_file_A.py"); + auto loc_filtered = FileLineColLoc::get(&context, id_filtered, 0, 0); + auto id_filtered2 = + StringAttr::get(&context, "dir/tensorflow/python/filtered_file_B.py"); + auto loc_filtered2 = FileLineColLoc::get(&context, id_filtered2, 0, 0); + auto id_filtered3 = + StringAttr::get(&context, "//tensorflow/python/something/my_op.py"); + auto loc_filtered3 = FileLineColLoc::get(&context, id_filtered3, 0, 0); + + // Build a small stack for each error; the MLIR diagnostic filtering will + // surface a location that would otherwise be filtered if it is the only + // location associated with an error; therefore we need a combinatination of + // locations to test. + auto callsite_loc = mlir::CallSiteLoc::get(loc, loc_filtered); + auto callsite_loc2 = mlir::CallSiteLoc::get(loc_filtered3, loc_filtered2); + + // Test with filter off. + StatusScopedDiagnosticHandler ssdh_no_filter(&context, false, false); + emitError(callsite_loc) << "Error 1"; + emitError(callsite_loc2) << "Error 2"; + Status s_no_filter = ssdh_no_filter.ConsumeStatus(); + // All files should be present, especially the 'filtered' ones. + EXPECT_THAT(s_no_filter.message(), HasSubstr("keras")); + EXPECT_THAT(s_no_filter.message(), HasSubstr("my_op")); + EXPECT_THAT(s_no_filter.message(), HasSubstr("filtered_file_A")); + EXPECT_THAT(s_no_filter.message(), HasSubstr("filtered_file_B")); +} + +} // namespace +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..c10bb7b5df68b12c5a717bb45351209645e19b23 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.cc @@ -0,0 +1,154 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/eval_util.h" + +#include +#include + +#include "absl/container/inlined_vector.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/c/eager/c_api_internal.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +using gtl::MakeCleanup; + +#define RETURN_FAILURE_IF_ERROR(expr) \ + if (!IsOk(expr)) { \ + return mlir::failure(); \ + } + +static bool IsOk(const TF_Status* s) { + if (TF_GetCode(s) == TF_OK) return true; + VLOG(2) << TF_Message(s); + return false; +} + +static bool IsOk(const absl::Status& s) { + if (s.ok()) return true; + VLOG(2) << s.message(); + return false; +} + +mlir::LogicalResult EvaluateOperation( + mlir::Operation* inst, llvm::ArrayRef operands, + TFE_Context* context, llvm::SmallVectorImpl* results) { + if (!context) { + VLOG(1) << "Can't evaluate with null context."; + return mlir::failure(); + } + // If any operand is nullptr returns true for a failure. + // TODO(b/120678030): remove this constraint if we find operators can be + // evaluated with some unknown operands. + if (std::any_of(operands.begin(), operands.end(), + [](mlir::Attribute operand) { return !operand; })) { + VLOG(1) << "Can't evaluate since not all operands are constant."; + return mlir::failure(); + } + + TF_Status* status = TF_NewStatus(); + auto clean_status = MakeCleanup([status] { TF_DeleteStatus(status); }); + + // Builds TF operation and sets all the attributes. + std::string node_name = "unnamed"; + if (auto attr = inst->getAttrOfType("name")) { + node_name = std::string(attr.getValue()); + } + auto node_def_or = ConvertTFDialectOpToNodeDef( + inst, node_name.c_str(), /*ignore_unregistered_attrs=*/true); + RETURN_FAILURE_IF_ERROR(node_def_or.status()); + const auto& node_def = node_def_or.value(); + + TFE_Op* op = TFE_NewOp(context, node_def->op().c_str(), status); + RETURN_FAILURE_IF_ERROR(status); + auto clean_op = MakeCleanup([op] { TFE_DeleteOp(op); }); + + // Explicitly set device to Host CPU instead of the device present in device + // attribute of the MLIR op. The assigned device might be remote, not + // available during compilation or compilation only device for on demand + // execution which may create a recursion if used for constant folding. + constexpr char kHostCpu[] = "/job:localhost/replica:0/task:0/CPU:0"; + TFE_OpSetDevice(op, kHostCpu, status); + RETURN_FAILURE_IF_ERROR(status); + for (const auto& attr : node_def->attr()) { + SetOpAttrValueScalar(context, op, attr.second, attr.first.c_str(), status); + RETURN_FAILURE_IF_ERROR(status); + } + + VLOG(1) << "Start to evaluate node: " << *node_def; + + // Adds inputs to the TF operation. + for (const auto operand : operands) { + Tensor tensor; + RETURN_FAILURE_IF_ERROR(ConvertToTensor(operand, &tensor)); + TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status->status); + RETURN_FAILURE_IF_ERROR(status); + auto clean_tensor = + MakeCleanup([tf_tensor] { TF_DeleteTensor(tf_tensor); }); + TFE_TensorHandle* input_handle = TFE_NewTensorHandle(tf_tensor, status); + RETURN_FAILURE_IF_ERROR(status); + auto clean_input_handle = + MakeCleanup([input_handle] { TFE_DeleteTensorHandle(input_handle); }); + TFE_OpAddInput(op, input_handle, status); + RETURN_FAILURE_IF_ERROR(status); + } + + // Executes the TF operation. + int num_outputs = inst->getNumResults(); + absl::InlinedVector outputs(num_outputs); + TFE_Execute(op, outputs.data(), &num_outputs, status); + RETURN_FAILURE_IF_ERROR(status); + auto clean_outputs = MakeCleanup([&outputs] { + for (TFE_TensorHandle* tensor_handle : outputs) { + TFE_DeleteTensorHandle(tensor_handle); + } + }); + + // Converts the outputs to MLIR attributes. + mlir::Builder builder(inst->getContext()); + for (TFE_TensorHandle* tensor_handle : outputs) { + TF_Tensor* tf_tensor = TFE_TensorHandleResolve(tensor_handle, status); + RETURN_FAILURE_IF_ERROR(status); + auto clean_tensor = + MakeCleanup([tf_tensor] { TF_DeleteTensor(tf_tensor); }); + Tensor tensor; + RETURN_FAILURE_IF_ERROR(TF_TensorToTensor(tf_tensor, &tensor)); + auto attr_or = ConvertTensor(tensor, &builder); + RETURN_FAILURE_IF_ERROR(attr_or.status()); + results->push_back(attr_or.value()); + } + + VLOG(1) << "Evaluate node " << node_name << " successfully!"; + + return mlir::success(); +} + +#undef RETURN_FAILURE_IF_ERROR +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.h new file mode 100644 index 0000000000000000000000000000000000000000..dc01fce76773aa3a7c86bb2a09ddaf5cae8767b9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/eval_util.h @@ -0,0 +1,39 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EVAL_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EVAL_UTIL_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/c/eager/c_api.h" + +namespace tensorflow { + +// Attempts to evaluates an MLIR Operation in TensorFlow eager mode with the +// specified operands. The op is always executed on the local host CPU +// irrespective of the device attribute of the given op. If there is a CPU +// kernel registered for the op and is executed successfully, this fills in the +// results vector. If not, results vector is unspecified. +// +mlir::LogicalResult EvaluateOperation( + mlir::Operation* inst, llvm::ArrayRef operands, + TFE_Context* context, llvm::SmallVectorImpl* results); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EVAL_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..c5a2d5ac3a066b750b421eb04ed3d09e419d4162 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc @@ -0,0 +1,487 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/export_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/location_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "xla/status_macros.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_to_functiondef.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace { +// static TensorFlow op prefix set. +std::set* GlobalOpPrefixes() { + static std::set* global_op_prefixes = [] { + std::set* result = new std::set; + result->insert("tf."); + result->insert("tf_executor."); + return result; + }(); + return global_op_prefixes; +} + +// Converts a location to the debug information for the node def. +absl::Status ConvertLocation(mlir::Location inst_loc, llvm::StringRef node_name, + NodeDef::ExperimentalDebugInfo* debug_info) { + mlir::Location unwrapped_inst_loc = GetLocationWithoutOpType(inst_loc); + + if (auto call_site = mlir::dyn_cast(unwrapped_inst_loc)) { + if (auto name_loc = mlir::dyn_cast( + GetLocationWithoutOpType(call_site.getCallee()))) { + llvm::StringRef original_node_name, original_func_name; + std::tie(original_node_name, original_func_name) = + name_loc.getName().strref().split('@'); + // The location points to the current node def. + if (node_name == original_node_name && original_func_name.empty()) { + return absl::OkStatus(); + } + debug_info->add_original_node_names(original_node_name.str()); + if (!original_func_name.empty()) { + debug_info->add_original_func_names(original_func_name.str()); + } + } + } else if (auto fused = mlir::dyn_cast(unwrapped_inst_loc)) { + auto locations = fused.getLocations(); + if (locations.size() <= 1) + return errors::InvalidArgument("expected experimental debuf info."); + // skip the first one, which is the name of the node_def. + for (int i = 0, end = locations.size() - 1; i < end; ++i) { + TF_RETURN_IF_ERROR(ConvertLocation(locations[i], node_name, debug_info)); + } + } + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::BoolAttr& attr, AttrValue* value) { + value->set_b(attr.getValue()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::IntegerAttr& attr, AttrValue* value) { + value->set_i(attr.getInt()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::FloatAttr& attr, AttrValue* value) { + value->set_f(attr.getValueAsDouble()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::ElementsAttr& attr, + AttrValue* value) { + return ConvertToTensorProto(attr, value->mutable_tensor()); +} + +absl::Status ConvertAttribute(const mlir::TF::PlaceholderAttr& attr, + AttrValue* value) { + value->set_placeholder(attr.getValue().str()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::TF::ShapeAttr& attr, + AttrValue* value) { + SetTensorShapeProto(attr, value->mutable_shape()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::FlatSymbolRefAttr& attr, + AttrValue* value) { + value->mutable_func()->set_name(attr.getValue().str()); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::TF::FuncAttr& attr, + bool remove_ref_type, AttrValue* value) { + TF_RETURN_IF_ERROR(ConvertAttribute( + mlir::cast(attr.getName()), value)); + TF_RETURN_IF_ERROR(ConvertAttributes(attr.getAttrs().getValue(), + /*attrs_to_ignore=*/{}, remove_ref_type, + value->mutable_func()->mutable_attr())); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::StringAttr& attr, AttrValue* value) { + absl::string_view attr_value(attr.getValue().data(), attr.getValue().size()); + switch (mangling_util::GetMangledKind(attr_value)) { + case mangling_util::MangledKind::kUnknown: { + value->set_s(attr_value); + return absl::OkStatus(); + } + case mangling_util::MangledKind::kDataType: { + DataType dtype; + TF_RETURN_IF_ERROR(mangling_util::DemangleDataType(attr_value, &dtype)); + value->set_type(dtype); + return absl::OkStatus(); + } + case mangling_util::MangledKind::kTensorShape: + TF_RETURN_IF_ERROR( + mangling_util::DemangleShape(attr_value, value->mutable_shape())); + return absl::OkStatus(); + default: + return errors::Unimplemented("Mangled string couldn't be handled!"); + } + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(mlir::Type type, bool remove_ref_type, + AttrValue* value) { + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(type, &dtype)); + if (tensorflow::IsRefType(dtype)) dtype = tensorflow::RemoveRefType(dtype); + value->set_type(dtype); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::TypeAttr& type, bool remove_ref_type, + AttrValue* value) { + return ConvertAttribute(type.getValue(), remove_ref_type, value); +} + +absl::Status ConvertAttribute(const mlir::UnitAttr& attr, AttrValue* value) { + value->clear_value(); + return absl::OkStatus(); +} + +absl::Status ConvertAttribute(const mlir::ArrayAttr& attr, bool remove_ref_type, + AttrValue* value) { + auto* list = value->mutable_list(); + for (mlir::Attribute a : attr.getValue()) { + if (auto attr = mlir::dyn_cast(a)) { + list->add_b(attr.getValue()); + } else if (auto attr = mlir::dyn_cast(a)) { + list->add_i(attr.getInt()); + } else if (auto attr = mlir::dyn_cast(a)) { + list->add_f(attr.getValueAsDouble()); + } else if (auto attr = mlir::dyn_cast(a)) { + AttrValue nested_value; + TF_RETURN_IF_ERROR(ConvertAttribute(attr, &nested_value)); + switch (nested_value.value_case()) { + case AttrValue::kS: + list->add_s(nested_value.s()); + break; + case AttrValue::kType: + list->add_type(nested_value.type()); + break; + case AttrValue::kShape: + *list->add_shape() = nested_value.shape(); + break; + default: + return errors::Unimplemented("Unhandled nested attribute!"); + } + } else if (auto attr = mlir::dyn_cast(a)) { + TensorProto tensor; + TF_RETURN_IF_ERROR(ConvertToTensorProto(attr, &tensor)); + *list->add_tensor() = tensor; + } else if (auto attr = mlir::dyn_cast(a)) { + AttrValue attr_val; + TF_RETURN_IF_ERROR(ConvertAttribute(attr, &attr_val)); + *list->add_func() = attr_val.func(); + } else if (auto attr = mlir::dyn_cast(a)) { + AttrValue attr_val; + // For type attributes, we only propagate the element type. + mlir::Type elt_type = attr.getValue(); + if (auto shaped_type = mlir::dyn_cast(elt_type)) { + elt_type = shaped_type.getElementType(); + } + TF_RETURN_IF_ERROR( + ConvertAttribute(elt_type, remove_ref_type, &attr_val)); + list->add_type(attr_val.type()); + } else if (auto attr = mlir::dyn_cast(a)) { + AttrValue attr_val; + TF_RETURN_IF_ERROR(ConvertAttribute(attr, &attr_val)); + *list->add_shape() = attr_val.shape(); + } else if (auto attr = mlir::dyn_cast(a)) { + std::vector vals; + for (mlir::Attribute a : attr.getValue()) { + auto i = mlir::dyn_cast(a); + if (!i) + return errors::Unimplemented( + "Expected 64-bit integer array attributes!"); + vals.push_back(i.getInt()); + } + mlir::OpBuilder builder(attr.getContext()); + mlir::TensorType ty = + mlir::RankedTensorType::get(vals.size(), builder.getIntegerType(64)); + TensorProto tensor; + TF_RETURN_IF_ERROR(ConvertToTensorProto( + mlir::DenseIntElementsAttr::get(ty, vals), &tensor)); + *list->add_tensor() = tensor; + } else { + return errors::Unimplemented("Unhandled attribute!"); + } + } + return absl::OkStatus(); +} + +// Returns true if the executor/control dialect op should map to Ref node in +// TensorFlow Graph. For control dialect NextIteration it uses the 1st operand +// type. For executor dialect NextIteration it uses the 2nd operand type. For +// all others (Enter/Exit/Merge/Switch), if the output type is ref, they +// correspond to the Ref equivalent op in TF Graph. +static bool IsRefTypeControlOp(mlir::Operation* op) { + if (auto next_iter_sink = + llvm::dyn_cast(op)) + return mlir::isa( + mlir::getElementTypeOrSelf(next_iter_sink.getInput().getType())); + + auto op_name_or_status = GetTensorFlowOpName(op->getName().getStringRef()); + if (!op_name_or_status.ok()) return false; + + auto op_name = std::move(op_name_or_status).value(); + if (op_name == "NextIteration") + return mlir::isa( + mlir::getElementTypeOrSelf(op->getOperand(0).getType())); + + if (op_name == "Enter" || op_name == "Exit" || op_name == "Switch" || + op_name == "Merge") { + return mlir::isa( + getElementTypeOrSelf(op->getResult(0).getType())); + } + return false; +} + +} // anonymous namespace + +absl::StatusOr GetTensorFlowOpName(llvm::StringRef op_name) { + // When being converted to MLIR, some prefixes and suffixes are added to the + // operation types, and we have to remove them when converting the + // operations back to a graph: + // - "tf." or "tf_executor." : every operation type has this prefix. + // - ".sink" or ".Sink": only the NextIteration operation has this suffix. We + // don't need to consider ".source"/".Source" because the nodes with this + // suffix are skipped by the caller and will not be added to the graph. + auto prefixes = GlobalOpPrefixes(); + if (std::none_of(prefixes->begin(), prefixes->end(), [&](std::string prefix) { + return op_name.consume_front(prefix); + })) { + return errors::FailedPrecondition("op node '", op_name.str(), + "' was not a TF op!"); + } + // Control dialect NextIteration sink ends with ".sink" and Executor dialect + // NextIteration sink ends with ".Sink". + if (!op_name.consume_back(".sink")) op_name.consume_back(".Sink"); + return op_name; +} + +absl::StatusOr> GetOperationNodeDef( + mlir::Operation* inst, llvm::StringRef name) { + auto node_def = std::make_unique(); + // Note: we do not use NodeBuilder or NodeDefBuilder as that would require + // mapping back from the inputs to the input arguments. + + llvm::SmallString<64> op_name; + if (IsLegacyCallInstruction(inst)) { + // The op_name is the name of the function. + op_name.append(inst->getAttrOfType("f") + .getLeafReference() + .getValue()); + // Remove the attribute from the instruction as it is already converted to + // op_name. + auto attr_id = mlir::StringAttr::get(inst->getContext(), "f"); + inst->removeAttr(attr_id); + } else { + // Some control flow ops in TensorFlow Graph have their respective "Ref" ops + // as well. For example there is Enter and RefEnter op. RefEnter forwards + // the input ref buffer to output. However both Enter and RefEnter are + // mapped to tf_executor::EnterOp during import. Check if it is a Ref op to + // correctly map to the TensorFlow Graph op. + if (IsRefTypeControlOp(inst)) op_name = "Ref"; + TF_ASSIGN_OR_RETURN(auto tf_name, + GetTensorFlowOpName(inst->getName().getStringRef())); + op_name.append(tf_name); + } + + node_def->set_name(name.str()); + node_def->set_op(std::string(op_name.str())); + + // Update NodeDef constructed out of an MLIR Case/If/While op to map it to + // either TensorFlow StatelessX or X op depending on the additional attribute. + if (llvm::isa(inst)) { + auto stateless = inst->getAttrOfType("is_stateless"); + if (stateless && stateless.getValue()) + *node_def->mutable_op() = "Stateless" + node_def->op(); + } + + // Add inputs to the NodeDef based on the number of operands. This is required + // as later when edges are added to the Node using Graph::AddEdge the + // associated NodeDef is not updated. + for (int i = 0, e = inst->getNumOperands(); i < e; ++i) { + node_def->add_input(); + } + if (auto attr = inst->getAttrOfType("device")) { + node_def->set_device(std::string(attr.getValue())); + } + + // Add the node debug info. + TF_RETURN_IF_ERROR(ConvertLocation( + inst->getLoc(), name, node_def->mutable_experimental_debug_info())); + + return node_def; +} + +absl::Status ConvertAttributes( + const llvm::ArrayRef attrs, + const absl::flat_hash_set& attrs_to_ignore, + bool remove_ref_type, AttrValueMap* values) { + AttrValueMap func_call_attrs; + for (const mlir::NamedAttribute& named_attr : attrs) { + auto name_strref = named_attr.getName().str(); + auto attr = named_attr.getValue(); + absl::string_view name(name_strref.data(), name_strref.size()); + if (name == "name" || name == "device" || attrs_to_ignore.contains(name)) { + // The name, device spec of a TF op or function are not stored as + // AttrValue inside NodeDef, but we model them using attribute inside + // MLIR. So we need to ignore them when going back to AttrValue here. + continue; + } + if (mangling_util::IsMangledAttributeName(name)) { + // In MLIR, attributes for functions requires dialect prefix. We need to + // remove TF dialect prefix before converting to AttrValue. + name = mangling_util::DemangleAttributeName(name); + } + AttrValue value; + if (auto symbol_ref = mlir::dyn_cast(attr)) { + TF_RETURN_IF_ERROR(ConvertAttribute( + mlir::cast(symbol_ref), &value)); + func_call_attrs[std::string(name)] = std::move(value); + continue; + } + if (auto func_attr = mlir::dyn_cast(attr)) { + TF_RETURN_IF_ERROR(ConvertAttribute(func_attr, remove_ref_type, &value)); + func_call_attrs[std::string(name)] = std::move(value); + continue; + } + if (mlir::isa(attr)) { + // AffineMapAttr is not implemented. + return errors::Unimplemented("AffineMap attribute (needed for '", + name_strref, "') unimplemented"); + } + TF_RETURN_IF_ERROR( + llvm::TypeSwitch(attr) + .Case( + [&](auto derived_attr) { + return ConvertAttribute(derived_attr, &value); + }) + .Case([&](auto derived_attr) { + return ConvertAttribute(derived_attr, remove_ref_type, &value); + }) + .Default([&](mlir::Attribute) { + return errors::Unimplemented( + "Unhandled attribute kind for attribute '", name_strref, + '\''); + })); + + // According to the NodeDef proto definition, an attribute name from the + // input TensorFlow GraphDef shouldn't contain '.'. If it does appear in + // the attribute from MLIR, it is treated as an attribute from function + // calls. + std::vector name_tokens = + absl::StrSplit(name, '.', absl::SkipEmpty()); + TF_RET_CHECK(name_tokens.size() <= 2); + auto it = func_call_attrs.find(name_tokens[0]); + if (it == func_call_attrs.end()) { + (*values)[std::string(name)] = std::move(value); + } else { + (*it->second.mutable_func()->mutable_attr())[name_tokens[1]] = + std::move(value); + } + } + for (auto& it : func_call_attrs) { + (*values)[it.first] = std::move(it.second); + } + return absl::OkStatus(); +} + +absl::Status SetShapeAttribute(absl::string_view name, + mlir::ShapedType shaped_type, + AttrValueMap* values) { + AttrValue value; + SetTensorShapeProto(shaped_type, value.mutable_list()->add_shape()); + + auto result = values->insert({std::string(name), value}); + if (!result.second) { + // This should be extremely rare as it means we are adding the same + // attribute multiple times/have some redundancy in representing this + // attribute. + TensorShapeProto actual_shape = result.first->second.shape(); + // Just check via string output as we shouldn't get here and if we do they + // should be trivially the same, else fail. + std::string new_shape_string = value.list().shape(0).ShortDebugString(); + if (actual_shape.ShortDebugString() != new_shape_string) { + return errors::InvalidArgument("Expected ", new_shape_string, " '", name, + "' attribute but found ", + actual_shape.ShortDebugString()); + } + } + return absl::OkStatus(); +} + +bool IsLegacyCallInstruction(mlir::Operation* inst) { + return llvm::dyn_cast(inst); +} + +absl::Status AddTensorFlowOpPrefix(std::string prefix) { + GlobalOpPrefixes()->insert(prefix); + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..5a2bd4ccec6b531f8727bd0b926812b7e4dc75a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h @@ -0,0 +1,96 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EXPORT_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EXPORT_UTILS_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/lib/core/status.h" + +namespace mlir { +class ShapedType; +} // namespace mlir + +namespace tensorflow { + +using tsl::StatusOr; + +// Add custom op prefix for TensorFlow dialects. +absl::Status AddTensorFlowOpPrefix(std::string); + +// Maps an MLIR op name in the TensorFlow dialect or the TensorFlow control +// dialect back into a TensorFlow valid op name. +absl::StatusOr GetTensorFlowOpName(llvm::StringRef); + +// Converts an MLIR operation to TensorFlow NodeDef with given node name. This +// name should be unique to the graph it is being inserted into. +absl::StatusOr> GetOperationNodeDef( + mlir::Operation* inst, llvm::StringRef name); + +// Converts MLIR attributes with values to their tensorflow equivalent. +// "name" and "device" attributes are ignored by default. Use attrs_to_ignore to +// specify any other attributes that should be ignored. +absl::Status ConvertAttributes( + llvm::ArrayRef attrs, + const absl::flat_hash_set& attrs_to_ignore, + bool remove_ref_type, AttrValueMap* values); + +// Fill in the contents of TensorShapeProto for the given shape. +// ShapeContainerT is any type with the following methods: +// bool hasRank() +// ArrayRef getShape() +// This includes mlir::TF::ShapeAttr and mlir::ShapedType. +template +void SetTensorShapeProto(ShapeContainerT shape, TensorShapeProto* proto) { + if (shape.hasRank()) { + for (int64_t dim : shape.getShape()) { + proto->add_dim()->set_size(mlir::ShapedType::isDynamic(dim) ? -1 : dim); + } + } else { + proto->set_unknown_rank(true); + } +} + +// Sets shape attribute with the given name. If the attribute already exists +// with a different value, returns an error. +absl::Status SetShapeAttribute(absl::string_view name, mlir::ShapedType shape, + AttrValueMap* values); + +// Returns true if the given instruction is an mlir::TF::LegacyCallOp or the +// result of such an operation transformed by the +// ExecutorToControlDialectConversion pass. +// +// TODO(b/145706023): When the ExecutorToControlDialectConversion pass runs +// before the exporter, it mutates an mlir::TF::LegacyCallOp instruction to +// an instruction with a different operation name. As such, this routine checks +// both forms of a LegacyCall instruction. We only need to check for +// mlir::TF::LegacyCallOp when the ticket is resolved. +bool IsLegacyCallInstruction(mlir::Operation* inst); +} // namespace tensorflow +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_EXPORTER_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.cc new file mode 100644 index 0000000000000000000000000000000000000000..e7fc9579b0819142781623a3ccb2dc051b8f25de --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.cc @@ -0,0 +1,197 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/fake_session.h" + +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "llvm/Support/CommandLine.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/device_attributes.pb.h" +#include "tensorflow/core/framework/device_factory.h" +#include "tensorflow/core/framework/resource_handle.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/resource_var.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/threadpool_options.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" +#include "tensorflow/core/public/session_options.h" + +namespace mlir { +namespace TF { +namespace test_util { +namespace { +using ::tensorflow::Status; +using ::tensorflow::Tensor; + +const char kDeviceNamePrefix[] = "/job:worker/replica:0/task:1"; +const char kDeviceName[] = "/job:worker/replica:0/task:1/device:CPU:0"; + +// Struct holding options for FakeSession which are configuered through +// command line flags. +struct FakeSessionOptions { + llvm::cl::opt fail_to_fetch_local_device_manager{ + "fail-to-fetch-local-device-manager", + llvm::cl::desc("Fail to fetch local device manager."), + llvm::cl::init(false)}; +}; +FakeSessionOptions* kSessionOptions = []() { return new FakeSessionOptions; }(); +} // namespace + +FakeSession::FakeSession() { + // We don't initialize devices in constructor as it causes some + // global initialization fiasco between tests and code in TF. +} + +void FakeSession::Initialize() { + if (initialized_) return; + BuildDeviceManager(); + InitVariables(); + initialized_ = true; +} + +void FakeSession::BuildDeviceManager() { + auto device = + tensorflow::DeviceFactory::NewDevice("CPU", {}, kDeviceNamePrefix); + device_mgr_ = + std::make_unique(std::move(device)); +} + +void FakeSession::InitVariables() { + tensorflow::Device* device = nullptr; + auto status = device_mgr_->LookupDevice(kDeviceName, &device); + if (status != absl::OkStatus()) return; + auto container = device->resource_manager()->default_container(); + + // Create 2 resources and initialize them with dummy values. + CHECK_OK(device->resource_manager()->Create( + container, "var1", new tensorflow::Var(tensorflow::DataType::DT_FLOAT))); + CHECK_OK(device->resource_manager()->Create( + container, "var2", new tensorflow::Var(tensorflow::DataType::DT_FLOAT))); +} + +Status FakeSession::Create(const tensorflow::GraphDef& graph) { + return tensorflow::errors::Unimplemented("not available"); +} +Status FakeSession::Extend(const tensorflow::GraphDef& graph) { + return tensorflow::errors::Unimplemented("not available"); +} + +Status FakeSession::Close() { + return tensorflow::errors::Unimplemented("not available"); +} + +Status FakeSession::ListDevices( + std::vector* response) { + return tensorflow::errors::Unimplemented("not available"); +} + +Status FakeSession::LocalDeviceManager( + const tensorflow::DeviceMgr** deviceMgrPtr) { + Initialize(); + if (kSessionOptions->fail_to_fetch_local_device_manager) + return Status(absl::StatusCode::kUnknown, "No Local Device Manager"); + *deviceMgrPtr = device_mgr_.get(); + return absl::OkStatus(); +} + +Status FakeSession::Run( + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs) { + tensorflow::RunMetadata run_metadata; + return Run(tensorflow::RunOptions(), inputs, output_names, target_nodes, + outputs, &run_metadata); +} + +Status FakeSession::Run( + const tensorflow::RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, std::vector* outputs, + tensorflow::RunMetadata* run_metadata) { + return Run(run_options, inputs, output_names, target_nodes, outputs, + run_metadata, tensorflow::thread::ThreadPoolOptions()); +} + +Status FakeSession::Run( + const tensorflow::RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, std::vector* outputs, + tensorflow::RunMetadata* run_metadata, + const tensorflow::thread::ThreadPoolOptions& thread_pool_options) { + Initialize(); + for (const std::string& output_name : output_names) { + Tensor output; + if (output_name == "dense/bias") { + Tensor t = Tensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({50})); + t.flat().setZero(); + outputs->push_back(t); + } else if (output_name == "dense/kernel") { + Tensor t = + Tensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({100, 50})); + t.flat().setZero(); + outputs->push_back(t); + } else if (output_name == "var1") { + Tensor t = Tensor(tensorflow::DT_RESOURCE, tensorflow::TensorShape({1})); + t.scalar()().set_name("var1"); + t.scalar()().set_device(kDeviceName); + + outputs->push_back(t); + } else if (output_name == "var2") { + Tensor t = Tensor(tensorflow::DT_RESOURCE, tensorflow::TensorShape({1})); + t.scalar()().set_name("var2"); + t.scalar()().set_device(kDeviceName); + + outputs->push_back(t); + } else if (output_name == "var3") { + Tensor t = Tensor(tensorflow::DT_RESOURCE, tensorflow::TensorShape({1})); + t.scalar()().set_name("var3"); + t.scalar()().set_device(kDeviceName); + + outputs->push_back(t); + } else if (output_name == "invalid_var") { + Tensor t = Tensor(tensorflow::DT_RESOURCE, tensorflow::TensorShape({1})); + t.scalar()().set_name("invalid_var"); + t.scalar()().set_device("invalid_device"); + + outputs->push_back(t); + } else if (absl::StartsWith(output_name, "var")) { + return Status(absl::StatusCode::kNotFound, + "Can't find variable " + output_name + " in session"); + } else { + // Create a scalar float tensor. + Tensor t = Tensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({})); + t.flat()(0) = 1.0f; + outputs->push_back(t); + } + } + return absl::OkStatus(); +} + +} // namespace test_util +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.h new file mode 100644 index 0000000000000000000000000000000000000000..1ab9f70cd90bd9d3e68193200edf3e822e801767 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/fake_session.h @@ -0,0 +1,85 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_FAKE_SESSION_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_FAKE_SESSION_H_ + +#include +#include +#include +#include + +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace TF { +namespace test_util { +// FakeSession is for testing only. +class FakeSession : public tensorflow::Session { + public: + FakeSession(); + + absl::Status Create(const tensorflow::GraphDef& graph) override; + absl::Status Extend(const tensorflow::GraphDef& graph) override; + + absl::Status Close() override; + + absl::Status ListDevices( + std::vector* response) override; + + absl::Status LocalDeviceManager( + const tensorflow::DeviceMgr** deviceMgrPtr) override; + + absl::Status Run( + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector<::tensorflow::Tensor>* outputs) override; + + absl::Status Run( + const tensorflow::RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector<::tensorflow::Tensor>* outputs, + tensorflow::RunMetadata* run_metadata) override; + + absl::Status Run( + const tensorflow::RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector<::tensorflow::Tensor>* outputs, + tensorflow::RunMetadata* run_metadata, + const tensorflow::thread::ThreadPoolOptions& thread_pool_options) + override; + + private: + void InitVariables(); + void BuildDeviceManager(); + void Initialize(); + + std::unique_ptr device_mgr_; + bool initialized_ = false; +}; + +} // namespace test_util +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_FAKE_SESSION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..59fbd75b5549260a84b30e75684a376b134facad --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.cc @@ -0,0 +1,81 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/import_utils.h" + +#include + +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { +namespace { +inline llvm::StringRef StringViewToRef(absl::string_view view) { + return {view.data(), view.size()}; +} +} // namespace + +absl::Status LoadProtoFromBuffer(absl::string_view input, + protobuf::Message* proto) { + // Attempt to parse as text. + if (ParseTextProto(input, "", proto).ok()) return absl::OkStatus(); + + // Else attempt to parse as binary. + return LoadProtoFromBuffer(input, static_cast(proto)); +} + +absl::Status LoadProtoFromBuffer(absl::string_view input, + protobuf::MessageLite* proto) { + // Attempt to parse as binary. + protobuf::io::ArrayInputStream binary_stream(input.data(), input.size()); + if (proto->ParseFromZeroCopyStream(&binary_stream)) return absl::OkStatus(); + + LOG(ERROR) << "Error parsing Protobuf: " << proto->GetTypeName(); + return errors::InvalidArgument("Could not parse input proto"); +} + +template +absl::Status LoadProtoFromFileImpl(absl::string_view input_filename, T* proto) { + const auto file_or_err = + llvm::MemoryBuffer::getFileOrSTDIN(StringViewToRef(input_filename)); + if (std::error_code error = file_or_err.getError()) { + return errors::InvalidArgument( + "Could not open input file ", + std::string(input_filename.data(), input_filename.size()).c_str()); + } + + const auto& input_file = *file_or_err; + absl::string_view content(input_file->getBufferStart(), + input_file->getBufferSize()); + return LoadProtoFromBuffer(content, proto); +} + +absl::Status LoadProtoFromFile(absl::string_view input_filename, + protobuf::Message* proto) { + return LoadProtoFromFileImpl(input_filename, proto); +} + +absl::Status LoadProtoFromFile(absl::string_view input_filename, + protobuf::MessageLite* proto) { + return LoadProtoFromFileImpl(input_filename, proto); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..99028ebd1c5919eac9e884f29c5f65adabfd7537 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/import_utils.h @@ -0,0 +1,45 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_IMPORT_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_IMPORT_UTILS_H_ + +#include "absl/strings/string_view.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +// Reads text (.pbtext) or binary (.pb) format of a proto message from the given +// buffer. Returns error status of the file is not found or malformed proto. +// Note that text protos can only be parsed when full protobuf::Message protos +// are used, and will fail for protobuf::MessageLite protos. +absl::Status LoadProtoFromBuffer(absl::string_view input, + protobuf::Message* proto); +absl::Status LoadProtoFromBuffer(absl::string_view input, + protobuf::MessageLite* proto); + +// Reads text (.pbtext) or binary (.pb) format of a proto message from the given +// file path. Returns error status of the file is not found or malformed proto. +// Note that text protos can only be parsed when full protobuf::Message protos +// are used, and will fail for protobuf::MessageLite protos. +absl::Status LoadProtoFromFile(absl::string_view input_filename, + protobuf::Message* proto); +absl::Status LoadProtoFromFile(absl::string_view input_filename, + protobuf::MessageLite* proto); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_IMPORT_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..828945f0af5938c966868a532adcfc3fce957e42 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.cc @@ -0,0 +1,44 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/location_utils.h" + +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace tensorflow { + +mlir::Location GetLocationWithoutOpType(mlir::Location loc) { + if (auto fused_loc = mlir::dyn_cast(loc)) { + auto locations = fused_loc.getLocations(); + if (!locations.empty()) { + // Skip locations for propagating op_type metadata. + if (auto name_loc = mlir::dyn_cast(locations[0])) { + if (name_loc.getName().strref().ends_with(":")) { + if (locations.size() == 2) + return locations[1]; + else if (locations.size() > 2) + return mlir::FusedLoc::get( + fused_loc.getContext(), + {locations.begin() + 1, locations.end()}); + } + } + } + } + return loc; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..8bddad9cb79d22b978599e3decdf30a9b1873168 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/location_utils.h @@ -0,0 +1,27 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_LOCATION_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_LOCATION_UTILS_H_ + +#include "mlir/IR/Location.h" // from @llvm-project + +namespace tensorflow { + +mlir::Location GetLocationWithoutOpType(mlir::Location loc); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_IMPORT_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..4b8683154624f29425ea82282016294b105a751b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.cc @@ -0,0 +1,100 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" + +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/ir/importexport/mangling.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { +namespace mangling_util { +namespace { + +using ::mlir::tfg::mangling_util::PrintShortTextProto; + +const char kAttributePrefix[] = "tf."; +const char kDataTypePrefix[] = "tfdtype$"; +const char kTensorShapePrefix[] = "tfshape$"; +const char kTensorPrefix[] = "tftensor$"; + +} // namespace + +std::string MangleAttributeName(absl::string_view str) { + return absl::StrCat(kAttributePrefix, str); +} + +bool IsMangledAttributeName(absl::string_view str) { + return absl::StartsWith(str, kAttributePrefix); +} + +absl::string_view DemangleAttributeName(absl::string_view str) { + DCHECK(IsMangledAttributeName(str)); + return str.substr(std::strlen(kAttributePrefix)); +} + +MangledKind GetMangledKind(absl::string_view str) { + if (absl::StartsWith(str, kDataTypePrefix)) { + return MangledKind::kDataType; + } else if (absl::StartsWith(str, kTensorShapePrefix)) { + return MangledKind::kTensorShape; + } else if (absl::StartsWith(str, kTensorPrefix)) { + return MangledKind::kTensor; + } else { + return MangledKind::kUnknown; + } +} + +std::string MangleShape(const TensorShapeProto& shape) { + return absl::StrCat(kTensorShapePrefix, PrintShortTextProto(shape)); +} + +absl::Status DemangleShape(absl::string_view str, TensorShapeProto* proto) { + return ParseTextProto(str, kTensorShapePrefix, proto); +} + +std::string MangleTensor(const TensorProto& tensor) { + return absl::StrCat(kTensorPrefix, PrintShortTextProto(tensor)); +} + +absl::Status DemangleTensor(absl::string_view str, TensorProto* proto) { + return ParseTextProto(str, kTensorPrefix, proto); +} + +std::string MangleDataType(const DataType& dtype) { + return absl::StrCat(kDataTypePrefix, DataType_Name(dtype)); +} + +absl::Status DemangleDataType(absl::string_view str, DataType* proto) { + absl::string_view pbtxt; + TF_RETURN_IF_ERROR(ConsumePrefix(str, kDataTypePrefix, &pbtxt)); + if (!DataType_Parse(pbtxt, proto)) { + return errors::FailedPrecondition( + "Could not parse TFDataType mangled proto"); + } + return absl::OkStatus(); +} + +} // namespace mangling_util +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h new file mode 100644 index 0000000000000000000000000000000000000000..6e6c8908ac17b4fdca8284762e9660ac76c0df3e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h @@ -0,0 +1,61 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MANGLING_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MANGLING_UTIL_H_ + +#include "absl/strings/string_view.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace mangling_util { +// The type of a mangled string. +enum class MangledKind { kUnknown, kDataType, kTensorShape, kTensor }; + +// Mangles an attribute name, marking the attribute as a TensorFlow attribute. +std::string MangleAttributeName(absl::string_view str); + +// Returns true if 'str' was mangled with MangleAttributeName. +bool IsMangledAttributeName(absl::string_view str); + +// Demangles an attribute name that was manged with MangleAttributeName. +// REQUIRES: IsMangledAttributeName returns true. +absl::string_view DemangleAttributeName(absl::string_view str); + +// Returns the type of a mangled string, or kUnknown. +MangledKind GetMangledKind(absl::string_view str); + +// Return a TensorShapeProto mangled as a string. +std::string MangleShape(const TensorShapeProto& shape); +// Demangle a string mangled with MangleShape. +absl::Status DemangleShape(absl::string_view str, TensorShapeProto* proto); + +// Return a TensorProto mangled as a string. +std::string MangleTensor(const TensorProto& tensor); +// Demangle a string mangled with MangleTensor. +absl::Status DemangleTensor(absl::string_view str, TensorProto* proto); + +// Return a DataType mangled as a string. +std::string MangleDataType(const DataType& dtype); +// Demangle a string mangled with MangleDataType. +absl::Status DemangleDataType(absl::string_view str, DataType* proto); + +} // namespace mangling_util +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MANGLING_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..f69c0ae5e857428747e8faa14bdccb4b68feeeb5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.cc @@ -0,0 +1,41 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.h" + +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/mlprogram.h" + +namespace tensorflow { + +void RegisterMlProgramPasses() { + mlir::registerPassPipeline( + "tf-lower-to-mlprogram-and-hlo", "Lower TF to ml_program + mhlo", + [](mlir::OpPassManager& pm, llvm::StringRef options, + llvm::function_ref + errorHandler) { + tensorflow::PopulateLowerToMlProgramAndHloPipeline(pm); + return mlir::success(); + }, + [](llvm::function_ref) {}); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.h new file mode 100644 index 0000000000000000000000000000000000000000..00c4cce0c244c5f9cf6c32f3a221223e38024195 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.h @@ -0,0 +1,31 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MLPROGRAM_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MLPROGRAM_UTIL_H_ + +#include +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace tensorflow { + +void RegisterMlProgramPasses(); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_MLPROGRAM_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..1712a4fa6d298e434ce6ae5704adb63c93e555b9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.cc @@ -0,0 +1,62 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h" + +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" + +namespace mlir { +namespace TF { + +tf_device::ParallelExecuteOp BuildParallelExecuteOp( + tf_device::ClusterFuncOp cluster_func, OpBuilder* builder) { + const auto output_types = cluster_func.getResultTypes(); + builder->setInsertionPoint(cluster_func); + auto parallel_execute = tf_device::ParallelExecuteOp::create( + *builder, cluster_func.getLoc(), 1, output_types); + cluster_func->remove(); + auto& block = parallel_execute.GetRegionBlockWithIndex(0); + builder->setInsertionPointToEnd(&block); + builder->insert(cluster_func); + cluster_func.replaceAllUsesWith(parallel_execute); + tf_device::ReturnOp::create(*builder, block.getParent()->getLoc(), + cluster_func.getResults()); + return parallel_execute; +} + +LogicalResult RemoveSingletonParallelExecuteOp( + tf_device::ParallelExecuteOp parallel_execute, OpBuilder* builder) { + if (parallel_execute.getRegions().size() == 1) { + builder->setInsertionPoint(parallel_execute); + auto& block = parallel_execute.GetRegionBlockWithIndex(0); + llvm::SmallVector ops_move; + for (Operation& op : block) { + ops_move.push_back(&op); + } + if (ops_move.size() != 2) { + parallel_execute.emitError() << "Expected 2 ops in parallel_execute."; + return failure(); + } + ops_move[0]->remove(); + builder->insert(ops_move[0]); + parallel_execute.replaceAllUsesWith(ops_move[0]); + parallel_execute.erase(); + } + return success(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h new file mode 100644 index 0000000000000000000000000000000000000000..f93b520208940c9f1f40999b47ef1866f1d320e5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parallel_execute_util.h @@ -0,0 +1,41 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARALLEL_EXECUTE_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARALLEL_EXECUTE_UTIL_H_ + +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" + +namespace mlir { +namespace TF { + +// TODO(b/243076653): Once the ParallelExecute is added do not remove it. This +// means BuildSingletonParallelExecuteOp will be used in one location, and +// RemoveSingletonParallelExecuteOp can be removed. + +// Wrap `cluster_func` in a `ParallelExecute` with only one child. This +// can be used to canonicalize IR, so there is always one `ParallelExecute`. +tf_device::ParallelExecuteOp BuildParallelExecuteOp( + tf_device::ClusterFuncOp cluster_func, OpBuilder* builder); + +// Unwrap `parallel_execute`'s contents if it only has one child. +LogicalResult RemoveSingletonParallelExecuteOp( + tf_device::ParallelExecuteOp parallel_execute, OpBuilder* builder); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARALLEL_EXECUTE_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.cc new file mode 100644 index 0000000000000000000000000000000000000000..dd65043c4df6374de8777fc813108d9a0421eb2b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.cc @@ -0,0 +1,71 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h" + +#include + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/io/tokenizer.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/casts.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +namespace { +// Error collector that simply ignores errors reported. +class NoOpErrorCollector : public protobuf::io::ErrorCollector { + public: + void RecordError(int line, tsl::protobuf::io::ColumnNumber column, + absl::string_view message) override {} +}; +} // namespace + +absl::Status ConsumePrefix(absl::string_view str, absl::string_view prefix, + absl::string_view* output) { + if (absl::StartsWith(str, prefix)) { + *output = str.substr(prefix.size()); + return absl::OkStatus(); + } + return errors::NotFound("No prefix \"", prefix, "\" in \"", str, "\""); +} + +absl::Status ParseTextProto(absl::string_view text_proto, + absl::string_view prefix_to_strip, + protobuf::Message* parsed_proto) { + protobuf::TextFormat::Parser parser; + // Don't produce errors when attempting to parse text format as it would fail + // when the input is actually a binary file. + NoOpErrorCollector collector; + parser.RecordErrorsTo(&collector); + // Attempt to parse as text. + absl::string_view text_proto_without_prefix = text_proto; + if (!prefix_to_strip.empty()) { + TF_RETURN_IF_ERROR( + ConsumePrefix(text_proto, prefix_to_strip, &text_proto_without_prefix)); + } + protobuf::io::ArrayInputStream input_stream(text_proto_without_prefix.data(), + text_proto_without_prefix.size()); + if (parser.Parse(&input_stream, parsed_proto)) { + return absl::OkStatus(); + } + parsed_proto->Clear(); + return errors::InvalidArgument("Could not parse text proto: ", text_proto); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h new file mode 100644 index 0000000000000000000000000000000000000000..7e492e75371c8ca142b6fab7704ffdc9bdaa9ec5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/parse_text_proto.h @@ -0,0 +1,44 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARSE_TEXT_PROTO_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARSE_TEXT_PROTO_H_ + +#include "absl/strings/string_view.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { + +// Sets output to the given input with `prefix` stripped, or returns an error if +// the prefix doesn't exist. +absl::Status ConsumePrefix(absl::string_view str, absl::string_view prefix, + absl::string_view* output); + +// Strips `prefix_to_strip` from `text_proto`, parses, and returns the parsed +// proto. +absl::Status ParseTextProto(absl::string_view text_proto, + absl::string_view prefix_to_strip, + protobuf::Message* parsed_proto); +inline absl::Status ParseTextProto(absl::string_view /* text_proto */, + absl::string_view /* prefix_to_strip */, + protobuf::MessageLite* /* parsed_proto */) { + return errors::Unavailable("Cannot parse text protos on mobile."); +} + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_PARSE_TEXT_PROTO_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..3bba61b8af968ceef34d993678c54a0a606e958b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.cc @@ -0,0 +1,102 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Bytecode/BytecodeWriter.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "xla/tsl/lib/io/zlib_compression_options.h" +#include "xla/tsl/lib/io/zlib_outputbuffer.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/file_system.h" + +namespace tensorflow { +namespace { +class WritableStringFile : public tsl::WritableFile { + public: + explicit WritableStringFile(std::string* data) : data_(data) {}; + ~WritableStringFile() override = default; + + absl::Status Append(absl::string_view data) override { + absl::StrAppend(data_, data); + return absl::OkStatus(); + } + + absl::Status Close() override { return absl::OkStatus(); } + absl::Status Flush() override { return absl::OkStatus(); } + absl::Status Sync() override { return absl::OkStatus(); } + + private: + std::string* data_; +}; +} // namespace + +absl::StatusOr SerializeMlirModuleToCompressedBytecode( + mlir::ModuleOp module_op) { + std::string bytecode; + llvm::raw_string_ostream os(bytecode); + mlir::BytecodeWriterConfig config; + if (mlir::failed(mlir::writeBytecodeToFile(module_op, os, config))) { + return absl::InternalError("Failed to serialize MLIR module to bytecode."); + } + std::string compressed_bytecode; + WritableStringFile f(&compressed_bytecode); + + tsl::io::ZlibCompressionOptions options = + tsl::io::ZlibCompressionOptions::GZIP(); + tsl::io::ZlibOutputBuffer buffer(&f, options.input_buffer_size, + options.output_buffer_size, options); + TF_RETURN_IF_ERROR(buffer.Init()); + TF_RETURN_IF_ERROR(buffer.Append(bytecode)); + TF_RETURN_IF_ERROR(buffer.Close()); + return compressed_bytecode; +} + +std::string SerializeMlirModule(mlir::ModuleOp module_op) { + if (GetMlirCommonFlags()->tf_serialize_mlir_to_compressed_bytecode) { + auto compressed_bytecode = + SerializeMlirModuleToCompressedBytecode(module_op); + if (compressed_bytecode.ok()) { + return compressed_bytecode.value(); + } + LOG_IF(ERROR, !compressed_bytecode.ok()) + << "Failed to serialize MLIR module to " + "compressed bytecode." + << compressed_bytecode.status(); + return ""; + } + std::string serialized_mlir_module; + llvm::raw_string_ostream os(serialized_mlir_module); + mlir::OpPrintingFlags print_flags; + if (GetMlirCommonFlags()->tf_mlir_enable_debug_info_serialization) { + print_flags.enableDebugInfo(); + } + module_op.print(os, print_flags); + return std::move(os.str()); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..a731cc3dcd6234eaa1480a393c6a2e37059b3b8c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h @@ -0,0 +1,33 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SERIALIZE_MLIR_MODULE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SERIALIZE_MLIR_MODULE_UTILS_H_ + +#include + +#include "absl/status/statusor.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project + +namespace tensorflow { +// Serializes a MLIR module `module_op` to a compressed bytecode string. +absl::StatusOr SerializeMlirModuleToCompressedBytecode( + mlir::ModuleOp module_op); +// Prints a MLIR module `module_op` and returns it as a string. +std::string SerializeMlirModule(mlir::ModuleOp module_op); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SERIALIZE_MLIR_MODULE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..f1e785a0f9340256fed3219ee436e564fd2e78d0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils_test.cc @@ -0,0 +1,59 @@ +/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" + +#include + +#include +#include "absl/strings/match.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +TEST(SerializeMlirModuleUtilsTest, DebugInfoSerialization) { + mlir::MLIRContext context; + mlir::OwningOpRef mlir_module = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + + GetMlirCommonFlags()->tf_mlir_enable_debug_info_serialization = true; + std::string serialized_module = SerializeMlirModule(*mlir_module); + EXPECT_TRUE(absl::StrContains(serialized_module, "loc(")); + + GetMlirCommonFlags()->tf_mlir_enable_debug_info_serialization = false; + serialized_module = SerializeMlirModule(*mlir_module); + EXPECT_FALSE(absl::StrContains(serialized_module, "loc(")); +} + +TEST(SerializeMlirModuleUtilsTest, CompressedBytecodeSerializationRoundTrip) { + GetMlirCommonFlags()->tf_serialize_mlir_to_compressed_bytecode = true; + mlir::MLIRContext context; + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + std::string mlir_module_str = tensorflow::SerializeMlirModule(*module_ref); + mlir::OwningOpRef deserialized_module; + EXPECT_TRUE(tensorflow::DeserializeMlirModule(mlir_module_str, &context, + &deserialized_module) + .ok()); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..1eb759e2bb40bb70fae97d1bab6f5152f972322a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.cc @@ -0,0 +1,82 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/session_utils.h" + +#include +#include + +#include "absl/status/status.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/utils/string_container_utils.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/device.h" +#include "tensorflow/core/framework/resource_var.h" + +namespace mlir { +namespace tf_saved_model { + +std::string GetVariableName(TF::VarHandleOp var_handle_op) { + // In some cases the shared_name attribute doesn't have the same + // tensor name in the model, so we first try to use the location + // then fallback to shared_name attribute. + if (auto loc = mlir::dyn_cast(var_handle_op->getLoc())) + return loc.getName().str(); + return var_handle_op.getSharedName().str(); +} + +std::vector GetVariableNames( + llvm::ArrayRef var_handle_ops) { + std::vector names; + names.reserve(var_handle_ops.size()); + for (auto var_handle_op : var_handle_ops) + names.push_back(GetVariableName(var_handle_op)); + return names; +} + +tensorflow::Var* GetVariableFromSession(mlir::TF::VarHandleOp var_handle_op, + llvm::StringRef device_name, + const tensorflow::DeviceMgr* mgr) { + tensorflow::Device* device = nullptr; + if (!mgr || !mgr->LookupDevice(StringRefToView(device_name), &device).ok()) + return nullptr; + tensorflow::Var* var_ptr = nullptr; + const auto& container = var_handle_op.getContainer().str(); + auto status = device->resource_manager()->Lookup( + (container.empty() ? device->resource_manager()->default_container() + : container), + var_handle_op.getSharedName().str(), &var_ptr); + if (!device || !status.ok()) return nullptr; + return var_ptr; +} + +absl::StatusOr> GetResourcesFromSession( + llvm::ArrayRef var_handle_ops, + tensorflow::Session* session) { + if (!session) + return absl::Status(absl::StatusCode::kInvalidArgument, + "Null Session provided."); + std::vector resource_tensors; + auto variable_names = GetVariableNames(var_handle_ops); + if (variable_names.empty()) return resource_tensors; + + auto status = session->Run({}, variable_names, {}, &resource_tensors); + if (!status.ok()) + return absl::Status(absl::StatusCode::kInternal, status.message()); + return resource_tensors; +} +} // namespace tf_saved_model +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..8c11a28c2daf404bfc1feb4d0979a6571694567d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/session_utils.h @@ -0,0 +1,50 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SESSION_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SESSION_UTILS_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace tf_saved_model { + +// Returns the variable for the provided 'var_handle_op'. +std::string GetVariableName(TF::VarHandleOp var_handle_op); + +// Returns pointer to the variable from 'session' that 'var_handle_op' +// refers to which is in 'device_name' device. If failed to fetch the value null +// will be returned. +// Note, caller is responsible for Unref the variable. +tensorflow::Var* GetVariableFromSession(mlir::TF::VarHandleOp var_handle_op, + llvm::StringRef device_name, + const tensorflow::DeviceMgr* mgr); + +// Returns resource tensors from session for all variables in 'module'. +absl::StatusOr> GetResourcesFromSession( + llvm::ArrayRef var_handle_ops, + tensorflow::Session* session); + +} // namespace tf_saved_model +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SESSION_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..bf365114552630047b6be52e3e9c7574d3ff7a40 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.cc @@ -0,0 +1,48 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.h" + +#include + +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/Interfaces/InferTypeOpInterface.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/core/ir/utils/shape_inference_utils.h" + +#define DEBUG_TYPE "tf-shape-inference-utils" + +namespace mlir { +namespace TF { + +LogicalResult InferReturnTypeComponentsForTFOp( + std::optional location, Operation* op, int64_t graph_version, + tfg::OperandAsConstantFn operand_as_constant_fn, + tfg::OpResultAsShapeFn op_result_as_shape_fn, + tfg::ResultElementTypeFn result_element_type_fn, + SmallVectorImpl& inferred_return_shapes) { + assert(op->getName().getDialectNamespace() == + TensorFlowDialect::getDialectNamespace()); + return tfg::InferReturnTypeComponentsForTFOp( + location, op, op->getOperands(), graph_version, operand_as_constant_fn, + op_result_as_shape_fn, result_element_type_fn, + tensorflow::GetAttrValuesFromOperation, inferred_return_shapes); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..bbfab215cefec5f8f1f497ddc8ae9f910fc736b6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/shape_inference_utils.h @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SHAPE_INFERENCE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SHAPE_INFERENCE_UTILS_H_ + +#include + +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/Interfaces/InferTypeOpInterface.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/core/ir/utils/shape_inference_utils.h" + +namespace mlir { + +class Operation; + +namespace TF { + +// Runs TensorFlow shape inference associated to the op type registered in the +// TensorFlow op registry based on the Graph version, operands, and attributes. +// Invoking this shape function will create conversions of parameters to the +// TensorFlow Graph equivalent data structures and back to MLIR equivalent data +// structures. This does not use a natively implemented shape inference in MLIR, +// and instead is temporary until shape functions are reimplemented/migrated to +// being in MLIR instead of the TensorFlow op registry. +LogicalResult InferReturnTypeComponentsForTFOp( + std::optional location, Operation* op, int64_t graph_version, + tfg::OperandAsConstantFn operand_as_constant_fn, + tfg::OpResultAsShapeFn op_result_as_shape_fn, + tfg::ResultElementTypeFn result_element_type_fn, + SmallVectorImpl& inferred_return_shapes); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SHAPE_INFERENCE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..3de43572ac31113dbaae78833de50c9db6dd46d1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.cc @@ -0,0 +1,66 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.h" + +#include + +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_side_effects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TF { + +std::string GetDeviceAttrAsResourceInstanceStr(mlir::Operation* op) { + auto device_attr = op->getAttrOfType("device"); + // Treat missing device attribute like unspecified (= empty string) attribute. + // Note that different op instances with the same string (including empty + // string) are seen as dependent (same resource instance). + if (!device_attr) return ""; + return device_attr.str(); +} + +void MarkResourceAsReadAndWrite( + OpOperand& op_operand, + SmallVectorImpl>& + effects) { + if (llvm::isa(llvm::cast(op_operand.get().getType()) + .getElementType())) { + effects.emplace_back(MemoryEffects::Read::get(), &op_operand, + ResourceEffects::Variable::get()); + effects.emplace_back(MemoryEffects::Write::get(), &op_operand, + ResourceEffects::Variable::get()); + } +} + +void MarkResourceAsReadOnly( + OpOperand& op_operand, + SmallVectorImpl>& + effects) { + if (llvm::isa(llvm::cast(op_operand.get().getType()) + .getElementType())) { + effects.emplace_back(MemoryEffects::Read::get(), &op_operand, + ResourceEffects::Variable::get()); + } +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.h new file mode 100644 index 0000000000000000000000000000000000000000..ac944708073226d369dcb0dfe63c7adb3ccb2821 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/side_effect_analysis_util.h @@ -0,0 +1,46 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SIDE_EFFECT_ANALYSIS_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SIDE_EFFECT_ANALYSIS_UTIL_H_ + +#include + +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace mlir { +namespace TF { + +std::string GetDeviceAttrAsResourceInstanceStr(Operation* op); + +void MarkResourceAsReadAndWrite( + OpOperand& op_operand, + SmallVectorImpl>& + effect); + +void MarkResourceAsReadOnly( + OpOperand& op_operand, + SmallVectorImpl>& + effect); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_SIDE_EFFECT_ANALYSIS_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.cc new file mode 100644 index 0000000000000000000000000000000000000000..483b82f297624ccb10e991248dd1a6418e8d8f68 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.cc @@ -0,0 +1,78 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h" + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo + +namespace mlir { +namespace TF { +namespace { + +// jax2tf sets `stablehlo.custom_call`'s target name as `tf.call_tf_function` +// to represent calling a TF host callback function. +constexpr llvm::StringRef kTfTargetName = "tf.call_tf_function"; + +// `tf.backend_config` is a DictionaryAttr, JAX2TF sets the value of its +// string attribute `caller_name` to the TF host callback function's name. +constexpr llvm::StringRef kTfBackendConfigAttrName = "tf.backend_config"; +constexpr llvm::StringRef kCalledFuncAttrName = "called_func"; + +} // namespace + +bool IsTfFuncCustomCall(stablehlo::CustomCallOp op) { + return op.getCallTargetName() == kTfTargetName; +} + +DictionaryAttr GetTfBackendConfig(stablehlo::CustomCallOp op) { + return op->getAttrOfType(kTfBackendConfigAttrName); +} + +FailureOr GetTfFuncCustomCallFuncName( + stablehlo::CustomCallOp op) { + if (!IsTfFuncCustomCall(op)) { + return success(nullptr); + } + + auto config = GetTfBackendConfig(op); + if (config == nullptr) { + op.emitOpError() << "does not have dictionary attribute '" + << kTfBackendConfigAttrName << "'"; + return failure(); + } + + auto f = config.get(kCalledFuncAttrName); + if (f == nullptr) { + op.emitOpError() << "does not have attribute '" << kCalledFuncAttrName + << "' in its dictionary attribute '" + << kTfBackendConfigAttrName << "'"; + return failure(); + } + + if (auto attr = mlir::dyn_cast(f)) { + return attr; + } + + op.emitOpError() << "'s attribute '" << kCalledFuncAttrName + << "' is neither StringAttr nor FlatSymbolRefAttr"; + return failure(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h new file mode 100644 index 0000000000000000000000000000000000000000..0d33174cfcec95e76821f18a414bbefeb6b4d32b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/stablehlo_custom_call.h @@ -0,0 +1,40 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STABLEHLO_CUSTOM_CALL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STABLEHLO_CUSTOM_CALL_H_ + +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo + +namespace mlir { +namespace TF { + +// Returns whether the custom call op represents a TF function call. +bool IsTfFuncCustomCall(stablehlo::CustomCallOp op); + +// Returns the `called_func` symbol ref attribute in the `tf.backend_config` +// dictionary attribute. +// +// If the op does not represent a TF function call, returns nullptr. +// Otherwise, if the op does not have `caller_name`, returns failure. +FailureOr GetTfFuncCustomCallFuncName( + stablehlo::CustomCallOp op); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STABLEHLO_CUSTOM_CALL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..105f09184512e680072bd07042609b7b79bc2bfc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.cc @@ -0,0 +1,58 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/string_util.h" + +#include +#include + +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project + +namespace tensorflow { + +// Return a string form of `op` including debug information. +std::string OpAsString(mlir::Operation& op) { + std::string out; + llvm::raw_string_ostream op_stream(out); + op.print(op_stream, mlir::OpPrintingFlags() + .elideLargeElementsAttrs() + .assumeVerified() + .skipRegions() + .printGenericOpForm()); + return out; +} + +std::string AttrAsString(mlir::Attribute& attr) { + std::string out; + llvm::raw_string_ostream attr_stream(out); + attr.print(attr_stream); + return out; +} + +std::ostream& operator<<(std::ostream& o, const LoggableOperation& op) { + return o << OpAsString(op.v); +} + +std::ostream& operator<<(std::ostream& o, const LoggableAttribute& attr) { + return o << AttrAsString(attr.v); +} + +std::ostream& operator<<(std::ostream& o, const LoggableStringRef& ref) { + return o << ref.v.str(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.h new file mode 100644 index 0000000000000000000000000000000000000000..601db2db19b7af5ebdb3b46e9257c4bcfaef12f9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/string_util.h @@ -0,0 +1,60 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STRING_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STRING_UTIL_H_ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project + +// Utility functions for dumping operations/attributes as strings and ostream +// bindings. + +namespace tensorflow { +std::string OpAsString(mlir::Operation& op); +std::string AttrAsString(mlir::Attribute& attr); + +// b/281863212 enable automatic without Op/AttrAsString. +// We add logging via a wrapper struct in order to respect ODS and avoid +// multiple symbol definitions if MLIR or someone else decides to add ostream +// definitions for the MLIR symbols. +struct LoggableOperation { + mlir::Operation& v; + // NOLINTNEXTLINE(google-explicit-constructor) + LoggableOperation(mlir::Operation& v) : v(v) {} +}; +std::ostream& operator<<(std::ostream& o, const LoggableOperation& op); + +struct LoggableAttribute { + mlir::Attribute& v; + // NOLINTNEXTLINE(google-explicit-constructor) + LoggableAttribute(mlir::Attribute& v) : v(v) {} +}; +std::ostream& operator<<(std::ostream& o, const LoggableAttribute& attr); + +struct LoggableStringRef { + const llvm::StringRef& v; + // NOLINTNEXTLINE(google-explicit-constructor) + LoggableStringRef(const llvm::StringRef& v) : v(v) {} +}; +std::ostream& operator<<(std::ostream& o, const LoggableStringRef& ref); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_STRING_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tf_xla_mlir_translate.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tf_xla_mlir_translate.cc new file mode 100644 index 0000000000000000000000000000000000000000..96ac21cd71b410b89f6cd7e6ec1fdb04ec6e8f45 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tf_xla_mlir_translate.cc @@ -0,0 +1,479 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/AsmParser/AsmParser.h" // from @llvm-project +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/Quant.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Tools/mlir-translate/Translation.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/parsers.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/compiler/mlir/tools/tf_mlir_translate_cl.h" +#include "tensorflow/compiler/mlir/utils/string_container_utils.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_argument.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/hlo/builder/xla_builder.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/hlo/ir/hlo_module.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/service/hlo.pb.h" +#include "xla/service/hlo_module_config.h" +#include "xla/shape.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tsl/platform/errors.h" +#include "tsl/platform/statusor.h" + +namespace { + +// NOLINTNEXTLINE +llvm::cl::opt input_types( + "tf-xla-input-types", + llvm::cl::desc("XLA input argument types (kinds), separated by ','. " + "Supported types include ['parameter', 'resource']. If " + "empty, all arguments are assumed to be parameters."), + llvm::cl::init("")); +// NOLINTNEXTLINE +llvm::cl::opt emit_use_tuple_arg( + "tf-xla-emit-use-tuple-args", + llvm::cl::desc( + "Emit HLO modules using tuples as args for the entry computation"), + llvm::cl::init(false)); +// NOLINTNEXTLINE +llvm::cl::opt emit_return_tuple( + "tf-xla-emit-return-tuple", + llvm::cl::desc("Emit HLO modules with entry computations returning tuple"), + llvm::cl::init(false)); +} // namespace + +namespace tensorflow { + +namespace { + +mlir::LogicalResult PrintHloModuleText( + const XlaCompilationResult& compilation_result, llvm::raw_ostream& output) { + const xla::HloModuleConfig module_config( + compilation_result.computation->GetProgramShape().value()); + auto status_or_hlo_module = xla::HloModule::CreateFromProto( + compilation_result.computation->proto(), module_config); + if (!status_or_hlo_module.ok()) { + LOG(ERROR) << "Conversion to HLO module failed: " + << status_or_hlo_module.status(); + return mlir::failure(); + } + + xla::HloModule* hlo_module = status_or_hlo_module.value().get(); + + output << hlo_module->ToString(xla::HloPrintOptions() + .set_print_large_constants(false) + .set_print_operand_shape(true)); + + if (!compilation_result.input_mapping.empty()) + output << "// InputMapping {" + << absl::StrJoin(compilation_result.input_mapping, ", ") << "}\n"; + + for (const auto& xla_input_shape : compilation_result.xla_input_shapes) + output << "// XlaInputShape " << xla_input_shape.ToString() << '\n'; + + output << "// XlaOutputShape " + << compilation_result.xla_output_shape.ToString() << '\n'; + + for (const auto& xla_output_description : compilation_result.outputs) { + output << "// XlaOutputDescription type=" + << DataTypeString(xla_output_description.type) << " shape=(" + << absl::StrJoin(xla_output_description.shape.dim_sizes(), ", ") + << ')'; + if (xla_output_description.input_index >= 0) + output << " input_index=" << xla_output_description.input_index; + if (xla_output_description.is_constant) output << " constant"; + if (xla_output_description.is_tensor_list) output << " tensor_list"; + output << '\n'; + } + + for (const auto& resource_update : compilation_result.resource_updates) { + output << "// ResourceUpdate input_index=" << resource_update.input_index + << " type=" << DataTypeString(resource_update.type) << " shape=(" + << absl::StrJoin(resource_update.shape.dim_sizes(), " ") << ')'; + if (resource_update.modified) output << " modified"; + output << '\n'; + } + + return mlir::success(); +} + +absl::Status ParseArgumentShapes( + absl::string_view input_shapes_str, + llvm::SmallVectorImpl& arg_shapes) { + arg_shapes.clear(); + std::vector>> input_shapes_vector; + TF_RETURN_IF_ERROR(ParseNodeShapes(input_shapes_str, input_shapes_vector)); + arg_shapes.resize(input_shapes_vector.size()); + for (const auto& shape : llvm::enumerate(input_shapes_vector)) { + if (!shape.value().has_value()) { + TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape( + static_cast(nullptr), 0, &arg_shapes[shape.index()].shape)); + continue; + } + TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape( + *shape.value(), &arg_shapes[shape.index()].shape)); + } + + return absl::OkStatus(); +} + +absl::Status ParseDataTypes(absl::string_view data_types_str, + llvm::SmallVectorImpl& data_types) { + data_types.clear(); + std::vector input_dtypes_vector; + TF_RETURN_IF_ERROR(ParseNodeDataTypes(data_types_str, input_dtypes_vector)); + data_types.resize(input_dtypes_vector.size(), DT_INVALID); + for (auto data_type : llvm::enumerate(input_dtypes_vector)) { + if (!DataType_Parse(data_type.value(), &data_types[data_type.index()])) + return errors::InvalidArgument("Invalid dtype at index ", + data_type.index(), ": ", + data_type.value()); + const auto& resolved_dtype = data_types[data_type.index()]; + if (resolved_dtype == DT_INVALID || resolved_dtype == DT_STRING || + resolved_dtype == DT_RESOURCE || resolved_dtype == DT_VARIANT || + IsRefType(resolved_dtype)) + return errors::InvalidArgument("Unsupported dtype at index ", + data_type.index(), ": ", + data_type.value()); + } + + return absl::OkStatus(); +} + +absl::Status ParseArgumentKinds( + absl::string_view input_types_str, + llvm::SmallVectorImpl& argument_kinds) { + argument_kinds.clear(); + if (input_types_str.empty()) return absl::OkStatus(); + + std::vector argument_kind_strs = + absl::StrSplit(input_types_str, ','); + argument_kinds.reserve(argument_kind_strs.size()); + for (const auto& argument_kind_str : llvm::enumerate(argument_kind_strs)) { + const auto& value = argument_kind_str.value(); + if (value == "parameter") { + argument_kinds.push_back(XlaArgument::Kind::kParameter); + } else if (value == "resource") { + argument_kinds.push_back(XlaArgument::Kind::kResource); + } else { + return errors::InvalidArgument( + "Unsupported TF/XLA argument kind at index ", + argument_kind_str.index(), ": ", value); + } + } + + return absl::OkStatus(); +} + +absl::Status ParseXlaArguments( + absl::string_view input_shapes_str, absl::string_view input_dtypes_str, + absl::string_view arg_kinds_str, + llvm::SmallVectorImpl& xla_arguments) { + xla_arguments.clear(); + std::vector>> input_shapes_vector; + TF_RETURN_IF_ERROR( + tensorflow::ParseNodeShapes(input_shapes_str, input_shapes_vector)); + llvm::SmallVector dtypes_vector; + TF_RETURN_IF_ERROR(ParseDataTypes(input_dtypes_str, dtypes_vector)); + llvm::SmallVector arg_kinds_vector; + TF_RETURN_IF_ERROR(ParseArgumentKinds(arg_kinds_str, arg_kinds_vector)); + + if (input_shapes_vector.empty()) + input_shapes_vector.resize(dtypes_vector.size()); + + if (arg_kinds_vector.empty()) + arg_kinds_vector.resize(input_shapes_vector.size(), + XlaArgument::Kind::kParameter); + + if (input_shapes_vector.size() != dtypes_vector.size() || + input_shapes_vector.size() != arg_kinds_vector.size()) + return errors::InvalidArgument( + "Input shapes, dtypes, and types/kinds must be of the same " + "length, but got ", + input_shapes_vector.size(), ", ", dtypes_vector.size(), ", and ", + arg_kinds_vector.size(), " respectively"); + + xla_arguments.resize(input_shapes_vector.size()); + for (const auto& arg_components : + llvm::zip(xla_arguments, input_shapes_vector, dtypes_vector, + arg_kinds_vector)) { + XlaArgument& arg = std::get<0>(arg_components); + TensorShape shape; + auto input_shapes = std::get<1>(arg_components); + if (input_shapes.has_value()) { + TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape(*input_shapes, &shape)); + } else { + TF_RETURN_IF_ERROR( + TensorShapeUtils::MakeShape(static_cast(nullptr), 0, &shape)); + } + arg.shape = std::move(shape); + arg.type = std::get<2>(arg_components); + arg.kind = std::get<3>(arg_components); + } + + return absl::OkStatus(); +} + +} // anonymous namespace + +// Test BuildHloFromTf. BuildHloFromTf only performs part of the conversion, so +// to make this test comparable to other compile tests, the test implements +// the remaining parts of the conversion. +absl::Status CompileMlirToXlaHloViaBuilder( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes) { + // This call to RefineShapes is redundant with the call in BuildHloFromTf. + // It's here so xla::Parameters that are created form block.getArguments will + // have the proper shapes. + TF_RETURN_IF_ERROR(RefineShapes(arg_shapes, module_op)); + + mlir::func::FuncOp main = module_op.lookupSymbol("main"); + mlir::Block& block = main.getRegion().front(); + xla::XlaBuilder builder("main"); + + // Create xla_params. + std::vector xla_params; + for (mlir::BlockArgument& arg : block.getArguments()) { + auto num = arg.getArgNumber(); + xla::Shape shape = xla::TypeToShape(arg.getType()); + xla::XlaOp argop = + xla::Parameter(&builder, num, shape, absl::StrCat("Arg_", num)); + xla_params.push_back(argop); + } + + std::vector returns(1); + TF_RETURN_IF_ERROR(BuildHloFromTf(module_op, builder, xla_params, returns, + arg_shapes, device_type, + custom_legalization_passes)); + + xla::XlaOp return_value; + if (returns.size() == 1) + return_value = returns[0]; + else + return_value = xla::Tuple(&builder, returns); + + TF_ASSIGN_OR_RETURN( + xla::XlaComputation computation, + return_value.valid() ? builder.Build(return_value) : builder.Build()); + auto hlo_module = computation.proto(); + xla::HloProto hlo_proto; + hlo_proto.mutable_hlo_module()->Swap(&hlo_module); + + compilation_result->computation = std::make_shared(); + xla::XlaComputation* xla_computation = compilation_result->computation.get(); + *xla_computation = xla::XlaComputation(hlo_proto.hlo_module()); + + XlaHelpers::ShapeRepresentationFn shape_representation_fn = + IdentityShapeRepresentationFn(); + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns{ + UseNoPreferenceLayoutFn(), IdentityShapeRepresentationFn()}; + return PopulateResultIOInfo(module_op, arg_shapes, /*use_tuple_args=*/false, + /*use_resource_updates_for_aliases=*/false, + shape_determination_fns, compilation_result); +} + +static mlir::LogicalResult MlirTfToHloTextTranslateFunctionImpl( + mlir::ModuleOp module_op, llvm::raw_ostream& output, bool via_builder) { + if (!module_op) return mlir::failure(); + + llvm::SmallVector arg_shapes; + auto args_status = + ParseArgumentShapes(mlir::StringRefToView(input_shapes), arg_shapes); + if (!args_status.ok()) { + LOG(ERROR) << args_status; + return mlir::failure(); + } + + auto device_type = "XLA_CPU_JIT"; + llvm::MutableArrayRef> + custom_legalization_passes{}; + XlaCompilationResult compilation_result; + auto compilation_status = + via_builder ? CompileMlirToXlaHloViaBuilder( + module_op, arg_shapes, device_type, &compilation_result, + custom_legalization_passes) + : CompileMlirToXlaHloAndSerialize( + module_op, arg_shapes, device_type, emit_use_tuple_arg, + /*enable_op_fallback=*/false, emit_return_tuple, + /*use_resource_updates_for_aliases=*/true, + /*shape_determination_fns=*/{}, &compilation_result, + custom_legalization_passes) + .status(); + if (!compilation_status.ok()) { + LOG(ERROR) << "TF/XLA compilation failed: " << compilation_status; + return mlir::failure(); + } + + return PrintHloModuleText(compilation_result, output); +} + +static mlir::LogicalResult MlirTfGraphToHloTextTranslateFunction( + mlir::ModuleOp module_op, llvm::raw_ostream& output) { + if (!module_op) return mlir::failure(); + + llvm::SmallVector xla_arguments; + auto args_status = ParseXlaArguments( + mlir::StringRefToView(input_shapes), mlir::StringRefToView(input_dtypes), + mlir::StringRefToView(input_types), xla_arguments); + if (!args_status.ok()) { + LOG(ERROR) << args_status; + return mlir::failure(); + } + + XlaCompilationResult compilation_result; + auto compilation_status = + CompileGraphToXlaHlo(module_op, xla_arguments, + /*device_type=*/"XLA_CPU_JIT", emit_use_tuple_arg, + /*analyse_graph=*/false, emit_return_tuple, + /*shape_determination_fns=*/{}, &compilation_result, + /*custom_legalization_passes=*/{}); + if (!compilation_status.ok()) { + LOG(ERROR) << "TF/XLA compilation failed: " << compilation_status; + return mlir::failure(); + } + + return PrintHloModuleText(compilation_result, output); +} + +static void RegisterMlirInputDialects(mlir::DialectRegistry& registry) { + // TODO(b/259459405): Remove support for stablehlo as an input. + registry + .insert(); + mlir::func::registerAllExtensions(registry); +} + +static void RegisterGraphInputDialects(mlir::DialectRegistry& registry) { + RegisterMlirInputDialects(registry); + registry.insert(); +} + +static mlir::OwningOpRef +SerializedMlirStringAttrToMlirModuleTranslate(llvm::StringRef input, + mlir::MLIRContext* context) { + // When the parser doesn't read all the input chars, it issues an error unless + // an output parameter is provided for returning the number of chars read. + size_t numRead; + mlir::Attribute attr = mlir::parseAttribute(input, context, {}, &numRead); + if (!attr || !mlir::isa(attr)) { + LOG(ERROR) << "Input is not parsable as a MLIR StringAttr."; + return nullptr; + } + auto str_attr = mlir::cast(attr); + + mlir::DialectRegistry registry; + RegisterMlirInputDialects(registry); + context->appendDialectRegistry(registry); + mlir::OwningOpRef module_ref; + auto status = + DeserializeMlirModule(str_attr.getValue().str(), context, &module_ref); + if (!status.ok()) { + LOG(ERROR) << status; + return nullptr; + } + + return module_ref; +} + +static mlir::LogicalResult MlirModuleToSerializedMlirStringAttrTranslate( + mlir::ModuleOp module_op, llvm::raw_ostream& output) { + output << "\""; + std::string serialized_module = SerializeMlirModule(module_op); + llvm::printEscapedString(serialized_module, output); + output << "\""; + return mlir::success(); +} + +static mlir::LogicalResult MlirTfToHloTextTranslateFunction( + mlir::ModuleOp module_op, llvm::raw_ostream& output) { + return MlirTfToHloTextTranslateFunctionImpl(module_op, output, false); +} + +static mlir::LogicalResult MlirTfToHloTextViaBuilderTranslateFunction( + mlir::ModuleOp module_op, llvm::raw_ostream& output) { + return MlirTfToHloTextTranslateFunctionImpl(module_op, output, true); +} + +} // namespace tensorflow + +static mlir::TranslateFromMLIRRegistration MlirTfToHloTextTranslate( + "mlir-tf-to-hlo-text", "mlir-tf-to-hlo-text", + tensorflow::MlirTfToHloTextTranslateFunction, + tensorflow::RegisterMlirInputDialects); + +static mlir::TranslateFromMLIRRegistration MlirTfToHloTextViaBuilderTranslate( + "mlir-tf-to-hlo-text-via-builder", "mlir-tf-to-hlo-text-via-builder", + tensorflow::MlirTfToHloTextViaBuilderTranslateFunction, + tensorflow::RegisterMlirInputDialects); + +static mlir::TranslateFromMLIRRegistration MlirTfGraphToHloTextTranslate( + "mlir-tf-graph-to-hlo-text", "mlir-tf-graph-to-hlo-text", + tensorflow::MlirTfGraphToHloTextTranslateFunction, + tensorflow::RegisterGraphInputDialects); + +static mlir::TranslateToMLIRRegistration SerializedMlirStringAttrToMlirModule( + "mlir-tf-str-attr-to-mlir", "mlir-tf-str-attr-to-mlir", + tensorflow::SerializedMlirStringAttrToMlirModuleTranslate); + +static mlir::TranslateFromMLIRRegistration MlirModuleToSerializedMlirStringAttr( + "mlir-tf-mlir-to-str-attr", "mlir-tf-mlir-to-str-attr", + tensorflow::MlirModuleToSerializedMlirStringAttrTranslate, + tensorflow::RegisterMlirInputDialects); diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.cc new file mode 100644 index 0000000000000000000000000000000000000000..d27386235d9628e2e3c60316f9b5496ae108974d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.cc @@ -0,0 +1,163 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h" + +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace mlir { +namespace TF { + +ExtraDependenciesFunction no_extra_dependencies = nullptr; + +std::vector SortBlockTopologically( + Block& block, PriorityFunction priorityFunction, + ExtraDependenciesFunction extraDependencies) { + llvm::DenseMap remaining_incoming_data_edges; + llvm::DenseMap remaining_incoming_ctrl_edges; + llvm::DenseMap position; + llvm::DenseMap ancestor; + SmallVector ready; + + llvm::SmallVector empty_op_set; + auto ctrlPredecessors = + [&](Operation* op) -> llvm::SmallVector const& { + if (extraDependencies) { + return extraDependencies(op, /*incoming=*/true); + } else { + return empty_op_set; + } + }; + auto ctrlSuccessors = + [&](Operation* op) -> llvm::SmallVector const& { + if (extraDependencies) { + return extraDependencies(op, /*incoming=*/false); + } else { + return empty_op_set; + } + }; + + int i = 0; + for (Operation& op : block.getOperations()) { + int incoming_ctrl_edges = 0; + int incoming_data_edges = 0; + op.walk([&](Operation* child) { + ancestor[child] = &op; + for (Operation* predecessor : ctrlPredecessors(child)) { + if (predecessor->getBlock() == &block) { + incoming_ctrl_edges++; + } + } + for (Value v : child->getOperands()) { + if (v.getParentBlock() == &block) { + incoming_data_edges++; + } + } + }); + remaining_incoming_data_edges[&op] = incoming_data_edges; + remaining_incoming_ctrl_edges[&op] = incoming_ctrl_edges; + if (incoming_data_edges == 0 && incoming_ctrl_edges == 0) { + ready.push_back(&op); + } + position[&op] = i++; + } + + std::queue todo; + for (Value value : block.getArguments()) { + todo.push(value); + } + + std::vector result; + Operation* previous_op = nullptr; + while (!todo.empty() || !ready.empty()) { + while (!todo.empty()) { + Value value = todo.front(); + todo.pop(); + // All operations that have all their inputs available are good to go. + // Uses, not Users, in case getUsers ever dedups. + for (OpOperand& operand : value.getUses()) { + Operation* user = ancestor[operand.getOwner()]; + remaining_incoming_data_edges[user]--; + if (remaining_incoming_data_edges[user] == 0 && + remaining_incoming_ctrl_edges[user] == 0) { + ready.push_back(user); + } + } + } + + // Find the "best" operation to emit. We + // (a) emit the terminator last. + // (b) honor the priority function (as far as possible). + // (c) preserve order within the ops of one dialect. + auto better = [&](Operation* a, Operation* b) { + if (a->hasTrait() != + b->hasTrait()) { + return b->hasTrait(); + } + int a_priority = priorityFunction(previous_op, a); + int b_priority = priorityFunction(previous_op, b); + if (a_priority != b_priority) { + return a_priority > b_priority; + } else { + return position[a] < position[b]; // preserve order + } + }; + + Operation* best = nullptr; + for (Operation* op : ready) { + if (best == nullptr || better(op, best)) { + best = op; + } + } + + if (!best) { + assert(ready.empty()); + return result; // happens for unused results for ops in the todo list + } + + // Consider this operation emitted, and make its results available. + ready.erase(std::find(ready.begin(), ready.end(), best)); + previous_op = best; + for (Value result : best->getResults()) { + todo.push(result); + } + for (Operation* successor : ctrlSuccessors(best)) { + if (ancestor.find(successor) != ancestor.end()) { + successor = ancestor[successor]; + remaining_incoming_ctrl_edges[successor]--; + if (remaining_incoming_ctrl_edges[successor] == 0 && + remaining_incoming_data_edges[successor] == 0) { + ready.push_back(successor); + } + } + } + result.push_back(best); + } + return result; +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h new file mode 100644 index 0000000000000000000000000000000000000000..733f712c80e489818549b6ce7e5f908410655c10 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/topological_sort.h @@ -0,0 +1,74 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TOPOLOGICAL_SORT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TOPOLOGICAL_SORT_H_ + +#include +#include + +#include "absl/types/span.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// A function that determines which op to emit next in the case of ties. +// The predecessor (which can be null) is the last op we emitted, +// and op is the candidate we're considering. A larger returned integer +// means the op has a higher chance of being emitted first. +typedef int (*PriorityFunction)(Operation *predecessor, Operation *op); + +// A function that returns extra dependencies for each op. These might +// e.g. be known side-effects (or control dependencies) between ops. +// If "incoming" is true, then the list of (extra) predecessors of the +// op should be returned. If "incoming" is false, the list of successors. +// The algorithm assumes that these are consistent which each other. So +// if (and only if) op1 is in extra_dependencies(op2, true), then op2 +// must also be in extra_dependencies(op1, false). +// This function is called multiple times during the topological sort, +// so the implementation should preferably be constant-time. +typedef llvm::function_ref const &( + Operation *, bool incoming)> + ExtraDependenciesFunction; + +// Convenience function if there are no extra dependencies to declare. +// (Unlike nullptr, this also works inside the ternary operator) +extern ExtraDependenciesFunction no_extra_dependencies; + +// Sort a block topologically, so that for all ops, all operands are +// available at the time of execution. This is similar to MLIR's topological +// sort (lib/Transforms/TopologicalSort.cpp) but also takes a priority +// function to determine the next op to emit in the case of ambiguity. This +// makes it possible to group operations by certain attributes. For example, +// the order_by_dialect pass uses this function to group by dialect. +// Only the operations nested directly under the block will be reordered. +// Nested blocks will be left alone. +// Also takes a list of control dependencies (vector of operation pairs, +// from->to) that will be honored when ordering the ops together with the +// data dependencies given through (the ops/results of) the operations +// themselves. +std::vector SortBlockTopologically( + Block &block, PriorityFunction priorityFunction, + ExtraDependenciesFunction extraDependencies = no_extra_dependencies); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TOPOLOGICAL_SORT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..153ae5af347c1f31d5524bbecc7a2aab52419a59 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.cc @@ -0,0 +1,128 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "mlir/Analysis/CallGraph.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace mlir { +namespace TFTPU { + +namespace { +mlir::LogicalResult WalkReachableFromTpuCluster( + bool pass_host_device, ModuleOp module, + std::function)> + callback) { + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(module, &devices))) return failure(); + const CallGraph call_graph(module); + // symbol_table caches callees in the CallGraph. + SymbolTableCollection symbol_table; + // List pending nodes to traverse with their root TPU cluster. + llvm::SmallVector> + pending_call_nodes; + // Cache the host device for each TPU cluster. + std::unordered_map> cluster_to_host; + + auto insert_pending_op = [&](Operation* op, + tf_device::ClusterOp tpu_cluster) { + // Add callee nodes to pending_call_nodes. + if (CallOpInterface call = dyn_cast(op)) { + CallGraphNode* node = call_graph.resolveCallable(call, symbol_table); + pending_call_nodes.emplace_back(node, tpu_cluster); + } + }; + + // Traverse ops in each TPU cluster. + auto result = module.walk([&](tf_device::ClusterOp tpu_cluster) { + std::optional host_device; + if (pass_host_device) { + std::string host_device_value; + if (failed(tensorflow::GetHostDeviceOutsideComputation( + devices, tpu_cluster, &host_device_value))) + return WalkResult::interrupt(); + host_device = host_device_value; + } + cluster_to_host[tpu_cluster.getOperation()] = host_device; + return tpu_cluster.walk([&](Operation* op) { + insert_pending_op(op, tpu_cluster); + return callback(op, tpu_cluster, + cluster_to_host[tpu_cluster.getOperation()]); + }); + }); + if (result.wasInterrupted()) return failure(); + + // Traverse ops that are reachable from some TPU cluster. + // node_to_cluster is used to avoid traversing the same node twice, and to + // check that no node is reachable from multiple TPU clusters. + std::unordered_map node_to_cluster; + while (!pending_call_nodes.empty()) { + auto pair = pending_call_nodes.back(); + pending_call_nodes.pop_back(); + CallGraphNode* node = pair.first; + tf_device::ClusterOp tpu_cluster = pair.second; + if (node_to_cluster.count(node)) { + if (node_to_cluster[node].getOperation() != tpu_cluster) { + node->getCallableRegion()->getParentOp()->emitOpError( + "The same function is reachable from multiple TPU Clusters."); + } + } else { + node_to_cluster[node] = tpu_cluster; + auto result = node->getCallableRegion()->walk([&](Operation* op) { + insert_pending_op(op, tpu_cluster); + return callback(op, tpu_cluster, + cluster_to_host[tpu_cluster.getOperation()]); + }); + if (result.wasInterrupted()) return failure(); + } + } + + return success(); +} +} // namespace + +mlir::LogicalResult WalkReachableFromTpuCluster( + ModuleOp module, std::function)> + callback) { + return WalkReachableFromTpuCluster(true, module, callback); +} + +mlir::LogicalResult WalkReachableFromTpuCluster( + ModuleOp module, + std::function callback) { + auto with_host_op = [&](Operation* op, tf_device::ClusterOp tpu_cluster, + std::optional host_device) { + return callback(op, tpu_cluster); + }; + return WalkReachableFromTpuCluster(false, module, with_host_op); +} + +} // namespace TFTPU +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.h new file mode 100644 index 0000000000000000000000000000000000000000..cfabae85a057973731beaf9d29607a585ced930e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_cluster_util.h @@ -0,0 +1,51 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_CLUSTER_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_CLUSTER_UTIL_H_ + +#include +#include +#include + +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace mlir { +namespace TFTPU { + +// For each TPU cluster in `module`, walk over all ops inside the cluster +// and reachable in the call graph from the cluster. +// For each op walked, `callback` is applied to the op, the root cluster, and +// the root cluster's host device. `callback` returning WasInterrupted +// indicates failure. +// The host device is null when the tpu_cluster HasModelParallelism: The +// HasModelParallelism case is currently unsupported in combination with +// outside compilation. +mlir::LogicalResult WalkReachableFromTpuCluster( + ModuleOp module, std::function)> + callback); + +// Like above, except TPU clusters are not required to have a host device, and +// no host device is passed to `callback`. +mlir::LogicalResult WalkReachableFromTpuCluster( + ModuleOp module, + std::function callback); + +} // namespace TFTPU +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_CLUSTER_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..46a911eb9ea94172030488f326fcda97067ddd82 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.cc @@ -0,0 +1,872 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/utils/string_container_utils.h" +#include "xla/array4d.h" +#include "xla/service/computation_placer.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { + +// Device coordinates are defined as (x, y, z, core), thus resulting in a rank 4 +// topology. +constexpr int kTPUTopologyRank = 4; + +constexpr char kDeviceTPUSystem[] = "TPU_SYSTEM"; +constexpr char kDeviceTPU[] = "TPU"; +constexpr char kTPUReplicatedCore[] = "TPU_REPLICATED_CORE"; +constexpr char kTPUReplicatedHost[] = "TPU_REPLICATED_HOST"; +constexpr char kBadIntArrayElementMsg[] = + "bad '{0}' attribute at index {1}, not an int"; + +using ParsedDevice = DeviceNameUtils::ParsedName; +using ParsedDevices = llvm::ArrayRef; + +namespace { +// Find matching devices in `devices` based on pattern `spec`. +llvm::SmallVector FindMatchingDevices( + ParsedDevices devices, const ParsedDevice& spec) { + llvm::SmallVector matching_devices; + for (const auto& device : devices) { + if (DeviceNameUtils::IsCompleteSpecification(spec, device)) { + matching_devices.push_back(device); + } + } + return matching_devices; +} + +// Create error message for a conflicting attribute of a device. +template +absl::Status MismatchedTPUSystemAttributeErr(absl::string_view attribute, T a, + T b) { + return absl::InvalidArgumentError( + absl::StrCat("found ", kDeviceTPUSystem, " devices with conflicting ", + attribute, "s '", a, "' and '", b, "'")); +} + +// Find TPU_SYSTEM:0 devices in `devices`. If multiple TPU_SYSTEM devices are +// found, the first one lexicographically is returned. If no TPU_SYSTEM device +// is found or if there are multiple TPU_SYSTEM devices with different jobs or +// replicas, a failure will be returned. +absl::StatusOr> GetTPUSystemDevices( + ParsedDevices devices) { + ParsedDevice spec; + spec.type = kDeviceTPUSystem; + spec.has_type = true; + spec.id = 0; + spec.has_id = true; + + llvm::SmallVector system_devices = + FindMatchingDevices(devices, spec); + if (system_devices.empty()) + return absl::InvalidArgumentError( + absl::StrCat("no ", kDeviceTPUSystem, " devices found")); + + // Check that all system devices are part of the same job. + const auto& job = system_devices[0].job; + auto replica = system_devices[0].replica; + for (const auto& device : llvm::make_range(std::next(system_devices.begin()), + system_devices.end())) { + if (device.job != job) + return MismatchedTPUSystemAttributeErr("job", job, device.job); + + if (device.replica != replica) + return MismatchedTPUSystemAttributeErr("replica", replica, + device.replica); + } + + // Sort by task to be deterministic. + std::sort(system_devices.begin(), system_devices.end(), + [](const ParsedDevice& a, const ParsedDevice& b) { + return a.task < b.task; + }); + + return system_devices; +} + +// Find TPU devices associated to system device based on spec (e.g. from +// GetTPUSystemDevices). If the number of TPU devices per host do not match for +// every host, a failure will be returned. +absl::StatusOr, 8>> +GetTPUDevices(ParsedDevices devices, + llvm::ArrayRef system_devices) { + llvm::SmallVector, 8> tpu_devices; + tpu_devices.reserve(system_devices.size()); + + auto lookup = [&devices](ParsedDevice device_spec) { + device_spec.has_type = true; + device_spec.type = kDeviceTPU; + // Enumerate all the available TPUs. + device_spec.has_id = false; + + llvm::SmallVector host_tpu_devices = + FindMatchingDevices(devices, device_spec); + + // Sort devices by id. + std::sort(host_tpu_devices.begin(), host_tpu_devices.end(), + [](const ParsedDevice& i, const ParsedDevice& j) { + return i.id < j.id; + }); + return host_tpu_devices; + }; + + int num_tpus_per_host = 0; + { + const auto& device = system_devices[0]; + auto host_tpu_devices = lookup(device); + num_tpus_per_host = host_tpu_devices.size(); + tpu_devices.push_back(std::move(host_tpu_devices)); + } + + for (const auto& device_spec : llvm::make_range( + std::next(system_devices.begin()), system_devices.end())) { + auto host_tpu_devices = lookup(device_spec); + // Check number of TPU devices per host all match. + const int64_t host_tpu_devices_size = host_tpu_devices.size(); + if (num_tpus_per_host != host_tpu_devices_size) + return absl::InvalidArgumentError( + absl::StrCat("expected the number of TPU devices per host to be ", + num_tpus_per_host, ", got ", host_tpu_devices.size())); + + tpu_devices.push_back(std::move(host_tpu_devices)); + } + + return tpu_devices; +} + +// Find the compilation device from system device with `DEVICE_CPU` as its +// type. +std::string GetTPUCompilationDevice(ParsedDevice system_device) { + // TODO(b/110910013) GetTPUSystemDevices parses the spec and returns the + // TPU_SYSTEM device, which we replace with the CPU device. We do this + // replacement because we want to place the `tf._TPUCompileMlir` explicitly on + // CPU devices of the same job as the TPU_SYSTEM device. + system_device.type = tensorflow::DEVICE_CPU; + return DeviceNameUtils::ParsedNameToString(system_device); +} + +// Find the host CPU device for a given TPU device with `DEVICE_CPU` as its +// type. If multiple local cpu devices are disabled, always assign id 0. If +// set, use the same id as the tpu device. +absl::StatusOr GetCPUHostDeviceForTPUDevice( + ParsedDevice tpu_device, ParsedDevices devices) { + tpu_device.type = DEVICE_CPU; + bool enable_multiple_local_cpu_devices = + tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_multiple_local_cpu_devices; + if (!enable_multiple_local_cpu_devices) { + tpu_device.id = 0; + } + if (FindMatchingDevices(devices, tpu_device).empty()) { + return absl::InvalidArgumentError(absl::StrCat( + "Can't find device: ", DeviceNameUtils::ParsedNameToString(tpu_device), + " in the devices list.")); + } + return DeviceNameUtils::ParsedNameToString(tpu_device); +} + +// Determine execution devices when topology and device assignment are not +// defined. This is a special case where a single core computation is replicated +// to every core in the mesh. TPU devices are simply added to +// `execution_devices` of one replica. `num_replicas` must be 1 or the total +// number of TPU devices available, and `num_cores_per_replica` must be 1. +absl::StatusOr GetFullMeshTPUExecutionDeviceAssignment( + int num_replicas, int num_cores_per_replica, + llvm::ArrayRef> tpu_devices, + ParsedDevices devices) { + const int num_tasks = tpu_devices.size(); + const int num_tpus_per_task = tpu_devices[0].size(); + const int num_tpu_devices = num_tasks * num_tpus_per_task; + + if (num_replicas != 1 && num_replicas != num_tpu_devices) + return absl::InvalidArgumentError( + absl::StrCat("'num_replicas' must be equal to 1 or ", num_tpu_devices, + ", got ", num_replicas)); + + if (num_cores_per_replica != 1) + return absl::InvalidArgumentError( + absl::StrCat("'num_cores_per_replica' must be equal to 1, got ", + num_cores_per_replica)); + + TPUDevicesAndHosts devices_and_hosts; + devices_and_hosts.reserve(num_replicas); + for (int i = 0; i < num_replicas; ++i) { + const int task = i / num_tpus_per_task; + const int device = i % num_tpus_per_task; + const auto& tpu_device = tpu_devices[task][device]; + devices_and_hosts.push_back({TPUDeviceAndHost( + /*device=*/tensorflow::DeviceNameUtils::ParsedNameToString(tpu_device), + /*host=*/*GetCPUHostDeviceForTPUDevice(tpu_device, devices))}); + } + + return devices_and_hosts; +} + +// Helper struct for keeping track of task and device for an associated TPU +// device coordinate. +struct TaskAndDevice { + TaskAndDevice() = default; + TaskAndDevice(int task, int device) : task(task), device(device) {} + + int task = -1; + int device = -1; +}; + +// Check if device coordinate is outside of topology mesh shape bounds. +bool DeviceCoordinateOutOfBound(int x, int y, int z, int core, int bound_x, + int bound_y, int bound_z, int bound_core) { + return x < 0 || x >= bound_x || y < 0 || y >= bound_y || z < 0 || + z >= bound_z || core < 0 || core >= bound_core; +} + +// Create error message for an out of bound device coordinate. +absl::Status DeviceCoordinateErrorMsg(absl::string_view attribute, int x, int y, + int z, int core, int bound_x, int bound_y, + int bound_z, int bound_core) { + return absl::InvalidArgumentError( + absl::StrCat("device coordinate (", x, ", ", y, ", ", z, ", ", core, + ") in '", attribute, "' is outside of mesh shape (", bound_x, + ", ", bound_y, ", ", bound_z, ", ", bound_core, ")")); +} + +// Create error message for a duplicate device coordinate. +absl::Status DuplicateCoordinateErrorMsg(absl::string_view attribute, int x, + int y, int z, int core) { + return absl::InvalidArgumentError( + absl::StrCat("'", attribute, "' has duplicate device coordinate (", x, + ", ", y, ", ", z, ", ", core, ")")); +} + +// Parse and validate topology (serialized string of TopologyProto), and maps +// device coordinate (x, y, z, core) to task and device (of available TPUs). +// Topology attribute device coordinates are ordered by task then device (major +// to minor). +// +// A valid TopologyProto must have: +// - a valid mesh shape (rank 4 with positive dimensions) +// - `num_tasks` and `num_tpu_devices_per_task` must match the number of +// available TPU hosts and devices per host +// - device coordinates within the mesh shape +// - no duplicate device coordinates +// - number of device coordinates (in tuple 3) match number of availabe TPUs +absl::StatusOr> ParseTopologyAttr( + llvm::StringRef topology_attr, int num_tasks, int num_tpus_per_task) { + tpu::TopologyProto topology_proto; + if (!topology_proto.ParseFromString(topology_attr.str())) + return absl::InvalidArgumentError(absl::StrCat( + "failed to parse '", kTopologyAttr, "' attribute to TopologyProto")); + + if (topology_proto.mesh_shape_size() != kTPUTopologyRank) + return absl::InvalidArgumentError(absl::StrCat( + "'", kTopologyAttr, "' 'mesh_shape' must be rank ", kTPUTopologyRank, + ", got rank ", topology_proto.mesh_shape_size())); + + for (auto mesh_shape_dim : llvm::enumerate(topology_proto.mesh_shape())) + if (mesh_shape_dim.value() <= 0) + return absl::InvalidArgumentError( + absl::StrCat("'", kTopologyAttr, "' 'mesh_shape' dimension ", + mesh_shape_dim.index(), " must be positive, got ", + mesh_shape_dim.value())); + + if (topology_proto.num_tasks() != num_tasks) + return absl::InvalidArgumentError(absl::StrCat( + "number of tasks from available TPU devices must be 'num_tasks' in '", + kTopologyAttr, "' (", topology_proto.num_tasks(), "), got ", + num_tasks)); + + if (topology_proto.num_tpu_devices_per_task() != num_tpus_per_task) + return absl::InvalidArgumentError(absl::StrCat( + "number of TPU devices available per task must be " + "'num_tpu_devices_per_task' in '", + kTopologyAttr, "' (", topology_proto.num_tpu_devices_per_task(), + "), got ", num_tpus_per_task)); + + const int expected_device_coordinates_size = + num_tasks * num_tpus_per_task * kTPUTopologyRank; + if (topology_proto.device_coordinates_size() != + expected_device_coordinates_size) + return absl::InvalidArgumentError(absl::StrCat( + "length of 'device_coordinates' in '", kTopologyAttr, + "' must be 'num_tasks' * 'num_tpus_per_task' * ", kTPUTopologyRank, + " (", num_tasks, " * ", num_tpus_per_task, " * ", kTPUTopologyRank, + "), got ", topology_proto.device_coordinates_size())); + + const int bound_x = topology_proto.mesh_shape(0); + const int bound_y = topology_proto.mesh_shape(1); + const int bound_z = topology_proto.mesh_shape(2); + const int bound_core = topology_proto.mesh_shape(3); + + xla::Array4D topology(bound_x, bound_y, bound_z, bound_core); + int pos = 0; + for (int task = 0; task < num_tasks; ++task) { + for (int device = 0; device < num_tpus_per_task; ++device) { + int x = topology_proto.device_coordinates(pos++); + int y = topology_proto.device_coordinates(pos++); + int z = topology_proto.device_coordinates(pos++); + int core = topology_proto.device_coordinates(pos++); + if (DeviceCoordinateOutOfBound(x, y, z, core, bound_x, bound_y, bound_z, + bound_core)) + return DeviceCoordinateErrorMsg(kTopologyAttr, x, y, z, core, bound_x, + bound_y, bound_z, bound_core); + + auto& task_and_device = topology(x, y, z, core); + if (task_and_device.task != -1) + return DuplicateCoordinateErrorMsg(kTopologyAttr, x, y, z, core); + + task_and_device = {task, device}; + } + } + + return topology; +} + +// Determine execution devices when topology and device assignment are defined. +// With a topology device coordinate to task and device mapping, device +// assignment device coordinates can then be mapped to task and device for TPU +// devices. The device assignment array is also validated. +// +// A valid device assignment array must have: +// - device coordinates within the topology mesh shape +// - no duplicate device coordinates +// - number of device coordinates (in tuple 3) match number 'num_replicas' * +// 'num_cores_per_replica' +// - a TPU device associated with each device coordinate +absl::StatusOr> +GetGeneralTPUExecutionDeviceAssignment( + int num_replicas, int num_cores_per_replica, + llvm::ArrayRef> tpu_devices, + ParsedDevices devices, llvm::StringRef topology_attr, + llvm::ArrayRef device_assignment_attr) { + const int num_tasks = tpu_devices.size(); + const int num_tpus_per_task = tpu_devices[0].size(); + + TF_ASSIGN_OR_RETURN(auto topology, ParseTopologyAttr(topology_attr, num_tasks, + num_tpus_per_task)); + + const int expected_device_assignment_size = + num_replicas * num_cores_per_replica * kTPUTopologyRank; + const int device_assignment_attr_size = device_assignment_attr.size(); + if (device_assignment_attr_size != expected_device_assignment_size) + return absl::InvalidArgumentError(absl::StrCat( + "length of '", kDeviceAssignmentAttr, + "' must be 'num_replicas' * 'num_cores_per_replica' * ", + kTPUTopologyRank, " (", num_replicas, " * ", num_cores_per_replica, + " * ", kTPUTopologyRank, "), got ", device_assignment_attr.size())); + + const int bound_x = topology.n1(); + const int bound_y = topology.n2(); + const int bound_z = topology.n3(); + const int bound_core = topology.n4(); + + // TPU XLA device ID is determined by its device coordinate, from major to + // minor coordinates (z, y, x, core). + auto location_to_id = [&](int x, int y, int z, int core) { + return (x + bound_x * (y + bound_y * z)) * bound_core + core; + }; + + std::vector used_device_ids(bound_x * bound_y * bound_z * bound_core, + false); + TPUDevicesAndHosts devices_and_hosts( + num_replicas, llvm::SmallVector( + num_cores_per_replica, TPUDeviceAndHost())); + xla::DeviceAssignment device_assignment(num_replicas, num_cores_per_replica); + int pos = 0; + for (int replica = 0; replica < num_replicas; ++replica) { + for (int logical_core = 0; logical_core < num_cores_per_replica; + ++logical_core) { + int x = device_assignment_attr[pos++]; + int y = device_assignment_attr[pos++]; + int z = device_assignment_attr[pos++]; + int core = device_assignment_attr[pos++]; + if (DeviceCoordinateOutOfBound(x, y, z, core, bound_x, bound_y, bound_z, + bound_core)) + return DeviceCoordinateErrorMsg(kDeviceAssignmentAttr, x, y, z, core, + bound_x, bound_y, bound_z, bound_core); + + TaskAndDevice task_and_device = topology(x, y, z, core); + const int task = task_and_device.task; + const int device = task_and_device.device; + if (task == -1 || device == -1) + return absl::InvalidArgumentError(absl::StrCat( + "no TPU device found for '", kDeviceAssignmentAttr, + "' device coordinate (", x, ", ", y, ", ", z, ", ", core, ")")); + + const int device_id = location_to_id(x, y, z, core); + if (used_device_ids[device_id]) + return DuplicateCoordinateErrorMsg(kDeviceAssignmentAttr, x, y, z, + core); + + used_device_ids[device_id] = true; + device_assignment(replica, logical_core) = device_id; + auto& device_and_host = devices_and_hosts[replica][logical_core]; + const auto& tpu_device = tpu_devices[task][device]; + device_and_host.device = DeviceNameUtils::ParsedNameToString(tpu_device); + device_and_host.host = *GetCPUHostDeviceForTPUDevice(tpu_device, devices); + } + } + + xla::DeviceAssignmentProto device_assignment_proto; + device_assignment.Serialize(&device_assignment_proto); + + return std::pair( + std::move(devices_and_hosts), std::move(device_assignment_proto)); +} + +mlir::LogicalResult GetTopology(mlir::tf_device::ClusterOp cluster, + std::string& topology) { + mlir::StringAttr topology_attr = + cluster->getAttrOfType(tensorflow::kTopologyAttr); + if (topology_attr) { + topology = topology_attr.getValue(); + return mlir::success(); + } else { + return cluster.emitOpError( + llvm::formatv("requires attribute '{0}'", tensorflow::kTopologyAttr) + .str()); + } +} + +mlir::LogicalResult GetDeviceAssignmentCoordinates( + mlir::tf_device::ClusterOp cluster, + llvm::SmallVector& device_coordinates) { + mlir::ArrayAttr device_assignment_attr = + cluster->getAttrOfType( + tensorflow::kDeviceAssignmentAttr); + if (!device_assignment_attr) + return cluster.emitOpError(llvm::formatv("requires attribute '{0}'", + tensorflow::kDeviceAssignmentAttr) + .str()); + if (absl::StatusOr> fetched_device_coordinates = + tensorflow::GetDeviceCoordinates(device_assignment_attr); + fetched_device_coordinates.ok()) { + device_coordinates = *fetched_device_coordinates; + return mlir::success(); + } else { + return cluster.emitError() << "error in fetching tpu device coordinates: " + << fetched_device_coordinates.status().message(); + } +} + +int GetNumCoresPerReplica(mlir::tf_device::ClusterOp cluster) { + mlir::IntegerAttr num_cores_per_replica_attr = + cluster->getAttrOfType(kNumCoresPerReplicaAttr); + if (num_cores_per_replica_attr) { + return num_cores_per_replica_attr.getInt(); + } else { + return 1; + } +} + +// Get the TPUDevicesAndHosts for a cluster that is not replicated. +mlir::LogicalResult GetTPUDevicesAndHostsNotReplicated( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + tensorflow::TPUDevicesAndHosts& devices_and_hosts) { + std::string topology; + if (failed(GetTopology(cluster, topology))) { + return mlir::failure(); + } + + llvm::SmallVector device_coordinates; + if (failed(GetDeviceAssignmentCoordinates(cluster, device_coordinates))) { + return mlir::failure(); + } + + // Determine compilation and execution devices. + if (absl::StatusOr tpu_device_assignment = + tensorflow::GetTPUCompilationAndExecutionDevices( + devices.device_names(), /*num_replicas=*/1, + GetNumCoresPerReplica(cluster), topology, device_coordinates); + tpu_device_assignment.ok()) { + devices_and_hosts = tpu_device_assignment->tpu_devices; + return mlir::success(); + } else { + return cluster.emitError() + << "error in fetching TPU compilation/execution devices: " + << tpu_device_assignment.status().message(); + } +} + +mlir::LogicalResult GetHostDeviceOCInTPUPipeline( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + std::string& host_device) { + mlir::tf_device::ReplicateOp replicate = + cluster->getParentOfType(); + if (replicate) { + host_device = GetDeviceAliasForHostOfLogicalCore(0); + return mlir::success(); + } + + tensorflow::TPUDevicesAndHosts devices_and_hosts; + if (failed(GetTPUDevicesAndHostsNotReplicated(devices, cluster, + devices_and_hosts))) { + return mlir::failure(); + } else { + host_device = devices_and_hosts[0][0].host; + return mlir::success(); + } +} + +// Get the map from `core` to `TPU_REPLICATED_HOST_{core}` for a replicated +// TPU cluster. +// TPU_REPLICATED_HOST_{core} is the host that corresponds to the TPU core. +// Different TPU_REPLICATED_HOST_*s can map to the same physical host within the +// same replica. Also, TPU_REPLICATE_HOST_{core} in different replicas can map +// to the same physical host. For example, if there are 2 hosts, num_replicas=8, +// and num_cores_per_replica=2, then all cores in the first 4 replicas will map +// to the first host and all cores in the second 4 replicas will map to the +// second host. +llvm::SmallVector GetTPUToHostMapReplicated( + mlir::tf_device::ClusterOp cluster) { + int num_cores_per_replica = GetNumCoresPerReplica(cluster); + llvm::SmallVector core_to_host; + core_to_host.reserve(num_cores_per_replica); + for (int core = 0; core < num_cores_per_replica; ++core) { + core_to_host.push_back(GetDeviceAliasForHostOfLogicalCore(core)); + } + return core_to_host; +} + +// Get the map from `core` to host device for a non-replicated TPU cluster. +mlir::LogicalResult GetTPUToHostMapNotReplicated( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + llvm::SmallVector& core_to_host) { + tensorflow::TPUDevicesAndHosts devices_and_hosts; + if (failed(GetTPUDevicesAndHostsNotReplicated(devices, cluster, + devices_and_hosts))) { + return mlir::failure(); + } + + // core_to_host is the list of hosts in replica 0, which is the only replica. + core_to_host.reserve(GetNumCoresPerReplica(cluster)); + for (const auto& device_and_host : devices_and_hosts[0]) { + core_to_host.push_back(device_and_host.host); + } + return mlir::success(); +} + +// Get the map from `core` to host device for a TPU cluster. +mlir::LogicalResult GetTPUToHostMap( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + llvm::SmallVector& core_to_host) { + if (cluster->getParentOfType()) { + core_to_host = GetTPUToHostMapReplicated(cluster); + return mlir::success(); + } + return GetTPUToHostMapNotReplicated(devices, cluster, core_to_host); +} + +} // anonymous namespace + +absl::StatusOr> GetDeviceCoordinates( + mlir::ArrayAttr device_assignment_attr) { + llvm::SmallVector device_coordinates; + device_coordinates.reserve(device_assignment_attr.size()); + + for (auto device_coordinate_and_idx : + llvm::enumerate(device_assignment_attr)) { + auto device_coordinate = + mlir::dyn_cast(device_coordinate_and_idx.value()); + if (!device_coordinate) + return absl::InvalidArgumentError( + llvm::formatv(kBadIntArrayElementMsg, kDeviceAssignmentAttr, + device_coordinate_and_idx.index()) + .str()); + + device_coordinates.push_back(device_coordinate.getInt()); + } + + return device_coordinates; +} + +absl::StatusOr GetXlaDeviceAssignmentProto( + llvm::StringRef topology_attr, int num_replicas, int num_cores_per_replica, + llvm::ArrayRef device_assignment_attr) { + tpu::TopologyProto topology_proto; + if (!topology_proto.ParseFromString(topology_attr.str())) + return absl::InvalidArgumentError(absl::StrCat( + "failed to parse '", kTopologyAttr, "' attribute to TopologyProto")); + + if (topology_proto.mesh_shape_size() < 4) { + return absl::InvalidArgumentError(absl::StrCat( + "The size of mesh_shape must be larger than or equal to 4, but got ", + topology_proto.mesh_shape_size())); + } + + const int bound_x = topology_proto.mesh_shape(0); + const int bound_y = topology_proto.mesh_shape(1); + const int bound_z = topology_proto.mesh_shape(2); + const int bound_core = topology_proto.mesh_shape(3); + + const int expected_device_assignment_size = + num_replicas * num_cores_per_replica * kTPUTopologyRank; + const int device_assignment_attr_size = device_assignment_attr.size(); + if (device_assignment_attr_size != expected_device_assignment_size) + return absl::InvalidArgumentError(absl::StrCat( + "length of '", kDeviceAssignmentAttr, + "' must be 'num_replicas' * 'num_cores_per_replica' * ", + kTPUTopologyRank, " (", num_replicas, " * ", num_cores_per_replica, + " * ", kTPUTopologyRank, "), got ", device_assignment_attr.size())); + + // TPU XLA device ID is determined by its device coordinate, from major to + // minor coordinates (z, y, x, core). + auto location_to_id = [&](int x, int y, int z, int core) { + return (x + bound_x * (y + bound_y * z)) * bound_core + core; + }; + + std::vector used_device_ids(bound_x * bound_y * bound_z * bound_core, + false); + + xla::DeviceAssignment device_assignment(num_replicas, num_cores_per_replica); + int pos = 0; + for (int replica = 0; replica < num_replicas; ++replica) { + for (int logical_core = 0; logical_core < num_cores_per_replica; + ++logical_core) { + int x = device_assignment_attr[pos++]; + int y = device_assignment_attr[pos++]; + int z = device_assignment_attr[pos++]; + int core = device_assignment_attr[pos++]; + if (DeviceCoordinateOutOfBound(x, y, z, core, bound_x, bound_y, bound_z, + bound_core)) + return DeviceCoordinateErrorMsg(kDeviceAssignmentAttr, x, y, z, core, + bound_x, bound_y, bound_z, bound_core); + const int device_id = location_to_id(x, y, z, core); + if (used_device_ids[device_id]) + return DuplicateCoordinateErrorMsg(kDeviceAssignmentAttr, x, y, z, + core); + + used_device_ids[device_id] = true; + device_assignment(replica, logical_core) = device_id; + } + } + xla::DeviceAssignmentProto device_assignment_proto; + device_assignment.Serialize(&device_assignment_proto); + return device_assignment_proto; +} + +absl::StatusOr GetTPUCompilationAndExecutionDevices( + ParsedDevices devices, int num_replicas, int num_cores_per_replica, + llvm::StringRef topology_attr, + llvm::ArrayRef device_assignment_attr) { + // Collect TPU_SYSTEM devices. + TF_ASSIGN_OR_RETURN(auto system_devices, GetTPUSystemDevices(devices)); + + // Collect TPU devices based on TPU_SYSTEM devices collected earlier. + TF_ASSIGN_OR_RETURN(auto tpu_devices, GetTPUDevices(devices, system_devices)); + + std::string compilation_device = GetTPUCompilationDevice(system_devices[0]); + + if (topology_attr.empty()) { + if (!device_assignment_attr.empty()) + return absl::InvalidArgumentError( + absl::StrCat("'", kDeviceAssignmentAttr, "' must not be set when '", + kTopologyAttr, "' is not set")); + + TF_ASSIGN_OR_RETURN( + auto execution_devices, + GetFullMeshTPUExecutionDeviceAssignment( + num_replicas, num_cores_per_replica, tpu_devices, devices)); + return TPUDeviceAssignment(compilation_device, + std::move(execution_devices)); + } + + TF_ASSIGN_OR_RETURN(auto devices_and_ids, + GetGeneralTPUExecutionDeviceAssignment( + num_replicas, num_cores_per_replica, tpu_devices, + devices, topology_attr, device_assignment_attr)); + return TPUDeviceAssignment(compilation_device, + std::move(devices_and_ids.first), + std::move(devices_and_ids.second)); +} + +std::string GetDeviceAliasForLogicalCore(const int core_index) { + return llvm::formatv("{0}_{1}", kTPUReplicatedCore, core_index).str(); +} + +std::string GetDeviceAliasForHostOfLogicalCore(const int core_index) { + return llvm::formatv("{0}_{1}", kTPUReplicatedHost, core_index).str(); +} + +bool HasModelParallelism(mlir::tf_device::ClusterOp cluster) { + mlir::IntegerAttr num_cores_per_replica_attr = + cluster->getAttrOfType( + tensorflow::kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) return false; + return num_cores_per_replica_attr.getInt() != 1; +} + +bool HasTPUDevice(const mlir::TF::RuntimeDevices& devices) { + for (const auto& device : devices.device_names()) { + if (device.has_type && device.type == "TPU") return true; + } + return false; +} + +mlir::LogicalResult GetHostDeviceOutsideCompilationInGenericPipeline( + mlir::TF::RuntimeDevices devices, std::string* host_device) { + for (const auto& device : devices.device_names()) { + if (device.has_type && device.type == "CPU" && device.id == 0) { + if (!host_device->empty()) { + // TODO(hanxiongwang): Remove this warning when TF API to bridge + // interface is understood. + LOG(WARNING) << "Found multiple CPU:0 host devices"; + if (device.job == "chief") + *host_device = + tensorflow::DeviceNameUtils::ParsedNameToString(device); + continue; + } + *host_device = tensorflow::DeviceNameUtils::ParsedNameToString(device); + } + } + if (host_device->empty()) { + LOG(ERROR) << "Did not find any CPU:0 host devices"; + return mlir::failure(); + } + return mlir::success(); +} + +mlir::LogicalResult GetHostDeviceOutsideComputation( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + std::string* host_device) { + if (HasTPUDevice(devices) || + cluster->getParentOfType()) { + return GetHostDeviceOCInTPUPipeline(devices, cluster, *host_device); + } else { + return GetHostDeviceOutsideCompilationInGenericPipeline(devices, + host_device); + } +} + +bool IsTPUDevice(llvm::StringRef device) { + ParsedDevice parsed_device; + if (!DeviceNameUtils::ParseFullName(mlir::StringRefToView(device), + &parsed_device)) + return false; + return parsed_device.has_type && parsed_device.type == kDeviceTPU; +} + +bool IsTPUReplicatedCore(llvm::StringRef device) { + ParsedDevice parsed_device; + if (!DeviceNameUtils::ParseFullName(mlir::StringRefToView(device), + &parsed_device)) + return false; + return parsed_device.has_type && parsed_device.type == kTPUReplicatedCore; +} + +bool TypeValidForXLA(const mlir::Type& type) { + const mlir::Type elem = getElementTypeOrSelf(type); + return !mlir::isa(elem) && + !mlir::isa(elem); +} + +mlir::LogicalResult GetDeviceToHostMap( + mlir::tf_device::ClusterOp cluster, + llvm::SmallVector& core_to_host) { + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp( + cluster->getParentOfType(), &devices))) { + return mlir::failure(); + } + + if (tensorflow::HasTPUDevice(devices) || + cluster->getParentOfType()) { + return GetTPUToHostMap(devices, cluster, core_to_host); + } + + std::string host_device; + if (failed(tensorflow::GetHostDeviceOutsideCompilationInGenericPipeline( + devices, &host_device))) { + return mlir::failure(); + } else { + core_to_host.push_back(host_device); + return mlir::success(); + } +} + +mlir::LogicalResult GetNonReplicatedTPU0(mlir::Operation* op, + std::string* tpu0_device) { + // Fetch the TPU devices. + mlir::ModuleOp moduleOp = op->getParentOfType(); + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(moduleOp, &devices))) + return moduleOp.emitOpError() << "No available devices."; + llvm::ArrayRef device_names = + devices.device_names(); + auto status_or_system_devices = GetTPUSystemDevices(device_names); + if (!status_or_system_devices.ok()) + return moduleOp.emitOpError() + << "error in fetching TPU_SYSTEM devices: " + << status_or_system_devices.status().message(); + auto status_or_tpu_devices = + GetTPUDevices(device_names, status_or_system_devices.value()); + if (!status_or_tpu_devices.ok()) + return moduleOp.emitOpError() << "error in fetching TPU devices: " + << status_or_tpu_devices.status().message(); + + // Select the first TPU device. + *tpu0_device = + DeviceNameUtils::ParsedNameToString(status_or_tpu_devices.value()[0][0]); + return mlir::success(); +} + +mlir::LogicalResult GetNonReplicatedCPU0(mlir::Operation* op, + std::string* cpu0_device) { + std::string tpu0_device; + if (failed(tensorflow::GetNonReplicatedTPU0(op, &tpu0_device))) + return mlir::failure(); + auto status = tensorflow::DeviceNameUtils::DeviceNameToCpuDeviceName( + tpu0_device, cpu0_device); + if (!status.ok()) + return op->emitError() + << "error in converting TPU0 to CPU0. The TPU device is " + << tpu0_device; + return mlir::success(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h new file mode 100644 index 0000000000000000000000000000000000000000..84156bda2b748112e1ca288910cc0b1bfb45383e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h @@ -0,0 +1,311 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_REWRITE_DEVICE_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_REWRITE_DEVICE_UTIL_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/statusor.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { +using tsl::StatusOr; + +inline constexpr absl::string_view kNumCoresPerReplicaAttr = + "num_cores_per_replica"; +inline constexpr absl::string_view kTopologyAttr = "topology"; +inline constexpr absl::string_view kDeviceAssignmentAttr = "device_assignment"; + +// A TPU device for execution alongside its associated host CPU device. +struct TPUDeviceAndHost { + TPUDeviceAndHost() = default; + TPUDeviceAndHost(llvm::StringRef device, llvm::StringRef host) + : device(device), host(host) {} + + std::string device; + std::string host; +}; + +// TPU devices to be used for execution (e.g. devices for TPUExecute ops) and +// their associated host CPU devices (for outside compilation). They are ordered +// by `num_replicas` followed by `num_cores_per_replica`. +using TPUDevicesAndHosts = + llvm::SmallVector, 8>; + +// TPU compilation device, execution and associated host devices, and optionally +// execution device IDs. Execution device IDs are populated if `topology` and +// `device_assignment` are provided. +struct TPUDeviceAssignment { + TPUDeviceAssignment(llvm::StringRef compilation_device, + TPUDevicesAndHosts&& tpu_devices) + : compilation_device(compilation_device), + tpu_devices(std::move(tpu_devices)) {} + + TPUDeviceAssignment(llvm::StringRef compilation_device, + TPUDevicesAndHosts&& tpu_devices, + xla::DeviceAssignmentProto&& xla_device_assignment) + : compilation_device(compilation_device), + tpu_devices(std::move(tpu_devices)), + xla_device_assignment(std::move(xla_device_assignment)) {} + + std::string compilation_device; + TPUDevicesAndHosts tpu_devices; + std::optional xla_device_assignment; +}; + +// Extracts device coordinates from a device assignment attribute on an op. +absl::StatusOr> GetDeviceCoordinates( + mlir::ArrayAttr device_assignment_attr); + +// Finds the TPU compilation device and execution devices from `devices` for a +// TPU computation subgraph. Compilation device is determined from looking up +// all TPU_SYSTEM:0 devices and choosing the CPU device associated to the first +// TPU_SYSTEM device sorted lexicographically by replica and task. Execution +// devices are determined by looking up all TPU devices associated with each +// TPU_SYSTEM:0 device found, alongside associated `topology_attr` and +// `device_assignment_attr`. If `topology_attr` not an empty string (parsable to +// TopologyProto), `device_assignment_attr` must not be empty also. When +// `topology_attr` and `device_assignment_attr` are not empty, a general device +// assignment based on those two attributes are used. Otherwise when +// `topology_attr` and `device_assignment_attr` are empty, a full mesh device +// assignment is used instead. A failure will be returned if it is not possible +// (e.g. invalid devices or invalid parameters). +// +// +// For example, for `devices`: +// { +// /job:localhost/replica:0/task:0/device:CPU:0, +// /job:worker/replica:0/task:0/device:CPU:0, +// /job:worker/replica:0/task:0/device:TPU_SYSTEM:0, +// /job:worker/replica:0/task:0/device:TPU:0, +// /job:worker/replica:0/task:0/device:TPU:1, +// /job:worker/replica:0/task:0/device:TPU:2, +// /job:worker/replica:0/task:0/device:TPU:3, +// /job:worker/replica:0/task:1/device:CPU:0, +// /job:worker/replica:0/task:1/device:TPU_SYSTEM:0, +// /job:worker/replica:0/task:1/device:TPU:0, +// /job:worker/replica:0/task:1/device:TPU:1, +// /job:worker/replica:0/task:1/device:TPU:2, +// /job:worker/replica:0/task:1/device:TPU:3 +// } +// +// +// With the following parameters (full mesh device assignment): +// `num_replicas` = 8 +// `num_cores_per_replica` = 1 +// `topology_attr` = "" +// `device_assignment_attr` = {} +// +// The `compilation_device` will be: +// /job:worker/replica:0/task:0/device:CPU:0 +// +// `execution_devices` will be: +// { +// { +// /job:worker/replica:0/task:0/device:TPU:0 +// }, +// { +// /job:worker/replica:0/task:0/device:TPU:1 +// }, +// { +// /job:worker/replica:0/task:0/device:TPU:2 +// }, +// { +// /job:worker/replica:0/task:0/device:TPU:3 +// }, +// { +// /job:worker/replica:0/task:1/device:TPU:0 +// }, +// { +// /job:worker/replica:0/task:1/device:TPU:1 +// }, +// { +// /job:worker/replica:0/task:1/device:TPU:2 +// }, +// { +// /job:worker/replica:0/task:1/device:TPU:3 +// } +// } +// +// and `xla_device_assignment` will not be set. +// +// +// With the following parameters (general device assignment): +// `num_replicas` = 4 +// `num_cores_per_replica` = 2 +// `topology_attr` (in proto debug string format) = +// { +// mesh_shape: 2 +// mesh_shape: 2 +// mesh_shape: 2 +// num_tasks: 2 +// num_tpu_devices_per_task: 4 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 1 +// device_coordinates: 1 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 1 +// device_coordinates: 1 +// device_coordinates: 0 +// device_coordinates: 0 +// device_coordinates: 1 +// } +// `device_assignment` = +// {0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1} +// +// The `compilation_device` will be: +// /job:worker/replica:0/task:0/device:CPU:0 +// +// `execution_devices` will be: +// { +// { +// "/job:worker/replica:0/task:0/device:TPU:0", +// "/job:worker/replica:0/task:1/device:TPU:3" +// }, +// { +// "/job:worker/replica:0/task:0/device:TPU:1", +// "/job:worker/replica:0/task:1/device:TPU:2" +// }, +// { +// "/job:worker/replica:0/task:0/device:TPU:3", +// "/job:worker/replica:0/task:1/device:TPU:0" +// }, +// { +// "/job:worker/replica:0/task:0/device:TPU:2", +// "/job:worker/replica:0/task:1/device:TPU:1" +// } +// } +// +// and `xla_device_assignment` will be: +// { +// replica_count: 4 +// computation_count: 2 +// computation_devices { +// replica_device_ids: 0 +// replica_device_ids: 4 +// replica_device_ids: 2 +// replica_device_ids: 6 +// } +// computation_devices { +// replica_device_ids: 1 +// replica_device_ids: 5 +// replica_device_ids: 3 +// replica_device_ids: 7 +// } +// } +absl::StatusOr GetTPUCompilationAndExecutionDevices( + llvm::ArrayRef devices, int num_replicas, + int num_cores_per_replica, llvm::StringRef topology_attr, + llvm::ArrayRef device_assignment_attr); + +// Converts a device assignment attribute to an XLA device assignment proto. +absl::StatusOr GetXlaDeviceAssignmentProto( + llvm::StringRef topology_attr, int num_replicas, int num_cores_per_replica, + llvm::ArrayRef device_assignment_attr); + +// Virtual device name of the passed logical core. The logical core is the index +// of a core within a replica. +std::string GetDeviceAliasForLogicalCore(int core_index); + +// Virtual device name of the host that is associated with the passed logical +// core. The logical core is the index of a core within a replica. +std::string GetDeviceAliasForHostOfLogicalCore(int core_index); + +// Returns true if cluster contains model parallelism based on +// `num_cores_per_replica_attribute`. Otherwise returns false. +bool HasModelParallelism(mlir::tf_device::ClusterOp cluster); + +// Returns true if the devices list contain any TPU devices +bool HasTPUDevice(const mlir::TF::RuntimeDevices& devices); + +// Returns the host device used for outside compilation in generic pipeline. +mlir::LogicalResult GetHostDeviceOutsideCompilationInGenericPipeline( + mlir::TF::RuntimeDevices devices, std::string* host_device); + +// Parses XLA compilation and execution devices from a tf_device.cluster and +// returns the host device for the head and tail computations. For TPU device, +// if the computation is replicated, GetDeviceAliasForHostOfLogicalCore(0) is +// returned instead. +mlir::LogicalResult GetHostDeviceOutsideComputation( + mlir::TF::RuntimeDevices devices, mlir::tf_device::ClusterOp cluster, + std::string* host_device); + +// Checks if a device string is a TPU device. +bool IsTPUDevice(llvm::StringRef device); + +// Checks if a device string is a TPU replicated core device. +bool IsTPUReplicatedCore(llvm::StringRef device); + +// Checks if `type` is allowed for XLA. String and resources are not XLA types. +// There are other TF types that are not XLA types which will be removed by +// successive passes in TF/XLA bridge phase 2. +bool TypeValidForXLA(const mlir::Type& type); + +// Returns the map from core to the host that is associated with the +// core. If `cluster` is not replicated then the core is a physical core index +// and the host is a physical host name. If `cluster` is replicated then the +// core with index `i` is a logical core (`TPU_REPLICATED_CORE_i`), and the host +// is the associated virtual device name (`TPU_REPLICATED_HOST_i`). +mlir::LogicalResult GetDeviceToHostMap( + mlir::tf_device::ClusterOp cluster, + llvm::SmallVector& core_to_host); + +// Returns the first TPU device, for use in the non-replicated case. The list of +// TPU devices is retrived from `op`'s module ancestor. +mlir::LogicalResult GetNonReplicatedTPU0(mlir::Operation* op, + std::string* tpu0_device); + +// Returns the CPU of the first TPU device, for use in the non-replicated case. +// The list of devices is retrived from `op`'s module ancestor. +mlir::LogicalResult GetNonReplicatedCPU0(mlir::Operation* op, + std::string* cpu0_device); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TPU_REWRITE_DEVICE_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..00259e5d44bdc8b9e2ad31f329a66d38bc4c8053 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util_test.cc @@ -0,0 +1,1200 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +#include +#include +#include +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/TypeRange.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/platform/status_matchers.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tsl/platform/statusor.h" + +namespace tensorflow { +namespace { + +absl::StatusOr> GetMlirModuleFromString( + llvm::StringRef string, mlir::MLIRContext* context) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + mlir::OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +using Device = DeviceNameUtils::ParsedName; + +bool DeviceNamesToParsedNames(llvm::ArrayRef device_names, + llvm::SmallVectorImpl* parsed_devices) { + parsed_devices->reserve(device_names.size()); + for (const auto& device_name : device_names) { + Device parsed_name; + if (!DeviceNameUtils::ParseFullName(device_name, &parsed_name)) + return false; + + parsed_devices->push_back(parsed_name); + } + return true; +} + +using DeviceNames = llvm::SmallVector; + +struct ParameterizedDeviceSetTest + : ::testing::TestWithParam> {}; + +TEST_P(ParameterizedDeviceSetTest, BadDeviceSet) { + llvm::SmallVector devices; + ASSERT_TRUE(DeviceNamesToParsedNames(std::get<0>(GetParam()), &devices)); + std::string topology_attr; + std::vector device_assignment_attr; + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, /*num_replicas=*/1, /*num_cores_per_replica=*/1, topology_attr, + device_assignment_attr); + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), std::get<1>(GetParam())); +} + +INSTANTIATE_TEST_SUITE_P( + BadDeviceSet, ParameterizedDeviceSetTest, + ::testing::Values( + std::make_tuple( + {"/job:localhost/replica:0/task:0/device:CPU:0"}, + "no TPU_SYSTEM devices found"), + std::make_tuple( + {"/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", + "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0"}, + "found TPU_SYSTEM devices with conflicting jobs 'localhost' and " + "'worker'"), + std::make_tuple( + {"/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", + "/job:localhost/replica:1/task:0/device:TPU_SYSTEM:0"}, + "found TPU_SYSTEM devices with conflicting replicas '0' and '1'"), + std::make_tuple( + {"/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", + "/job:localhost/replica:0/task:0/device:TPU:0", + "/job:localhost/replica:0/task:0/device:TPU:1", + "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0", + "/job:localhost/replica:0/task:1/device:TPU:0"}, + "expected the number of TPU devices per host to be 2, got 1"))); + +struct ParameterizedMetadataTest + : ::testing::TestWithParam, std::string>> { +}; + +TEST_P(ParameterizedMetadataTest, BadMetadata) { + llvm::SmallVector devices; + ASSERT_TRUE(DeviceNamesToParsedNames( + {"/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", + "/job:worker/replica:0/task:0/device:TPU:0", + "/job:worker/replica:0/task:0/device:CPU:0", + "/job:worker/replica:0/task:1/device:TPU_SYSTEM:0", + "/job:worker/replica:0/task:1/device:TPU:0", + "/job:worker/replica:0/task:1/device:CPU:0"}, + &devices)); + std::string compilation_device; + llvm::SmallVector, 8> execution_devices; + std::optional xla_device_assignment; + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, std::get<0>(GetParam()), std::get<1>(GetParam()), + std::get<2>(GetParam()), std::get<3>(GetParam())); + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), std::get<4>(GetParam())); +} + +std::string TopologyWithMeshShape(llvm::ArrayRef mesh_shape) { + tpu::TopologyProto topology_proto; + for (int mesh_dim : mesh_shape) topology_proto.add_mesh_shape(mesh_dim); + return topology_proto.SerializeAsString(); +} + +std::string TopologyWithMeshShapeAndTasks(llvm::ArrayRef mesh_shape, + int num_tasks, + int num_tpu_devices_per_task) { + tpu::TopologyProto topology_proto; + for (int mesh_dim : mesh_shape) topology_proto.add_mesh_shape(mesh_dim); + topology_proto.set_num_tasks(num_tasks); + topology_proto.set_num_tpu_devices_per_task(num_tpu_devices_per_task); + return topology_proto.SerializeAsString(); +} + +std::string TopologyWithDeviceCoordinates( + llvm::ArrayRef device_coordinates) { + tpu::TopologyProto topology_proto; + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(1); + topology_proto.set_num_tasks(2); + topology_proto.set_num_tpu_devices_per_task(1); + for (int device_coordinate : device_coordinates) + topology_proto.add_device_coordinates(device_coordinate); + return topology_proto.SerializeAsString(); +} + +INSTANTIATE_TEST_SUITE_P( + BadFullMeshMetadata, ParameterizedMetadataTest, + ::testing::Values( + std::make_tuple( + 2, 1, "", std::vector{0}, + "'device_assignment' must not be set when 'topology' is not set"), + std::make_tuple(8, 1, "", std::vector(), + "'num_replicas' must be equal to 1 or 2, got 8"), + std::make_tuple(2, 2, "", std::vector(), + "'num_cores_per_replica' must be equal to 1, got 2"))); + +INSTANTIATE_TEST_SUITE_P( + BadGeneralTopologyMetadata, ParameterizedMetadataTest, + ::testing::Values( + std::make_tuple( + 2, 1, "BAD_TOPOLOGY", std::vector(), + "failed to parse 'topology' attribute to TopologyProto"), + std::make_tuple(4, 2, TopologyWithMeshShape({0}), + std::vector(), + "'topology' 'mesh_shape' must be rank 4, got rank 1"), + std::make_tuple( + 2, 1, TopologyWithMeshShape({2, 0, 1, 2}), std::vector(), + "'topology' 'mesh_shape' dimension 1 must be positive, got 0"), + std::make_tuple(2, 1, TopologyWithMeshShapeAndTasks({1, 1, 1, 1}, 1, 1), + std::vector(), + "number of tasks from available TPU devices must be " + "'num_tasks' in 'topology' (1), got 2"), + std::make_tuple(2, 1, TopologyWithMeshShapeAndTasks({1, 1, 1, 1}, 2, 2), + std::vector(), + "number of TPU devices available per task must be " + "'num_tpu_devices_per_task' in 'topology' (2), got 1"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({}), std::vector(), + "length of 'device_coordinates' in 'topology' must be 'num_tasks' " + "* 'num_tpus_per_task' * 4 (2 * 1 * 4), got 0"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({-1, 0, 0, 0, 1, 0, 0, 0}), + std::vector(), + "device coordinate (-1, 0, 0, 0) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({2, 0, 0, 0, 1, 0, 0, 0}), + std::vector(), + "device coordinate (2, 0, 0, 0) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, -1, 0, 0, 1, 0, 0, 0}), + std::vector(), + "device coordinate (0, -1, 0, 0) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 1, 0, 0, 1, 0, 0, 0}), + std::vector(), + "device coordinate (0, 1, 0, 0) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, -1, 1, 0, 0, 0}), + std::vector(), + "device coordinate (0, 0, 0, -1) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 1, 1, 0, 0, 0}), + std::vector(), + "device coordinate (0, 0, 0, 1) in 'topology' is outside " + "of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 0, 0, 0, 0}), + std::vector(), + "'topology' has duplicate device coordinate (0, 0, 0, 0)"))); + +INSTANTIATE_TEST_SUITE_P( + BadGeneralDeviceAssignmentMetadata, ParameterizedMetadataTest, + ::testing::Values( + std::make_tuple(2, 1, + TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector(), + "length of 'device_assignment' must be 'num_replicas' " + "* 'num_cores_per_replica' * 4 (2 * 1 * 4), got 0"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{-1, 0, 0, 0, 0, 0, 0, 0}, + "device coordinate (-1, 0, 0, 0) in 'device_assignment' " + "is outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{2, 0, 0, 0, 0, 0, 0, 0}, + "device coordinate (2, 0, 0, 0) in 'device_assignment' is " + "outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{0, -1, 0, 0, 0, 0, 0, 0}, + "device coordinate (0, -1, 0, 0) in 'device_assignment' " + "is outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{0, 1, 0, 0, 0, 0, 0, 0}, + "device coordinate (0, 1, 0, 0) in 'device_assignment' is " + "outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{0, 0, 0, -1, 0, 0, 0, 0}, + "device coordinate (0, 0, 0, -1) in 'device_assignment' " + "is outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple( + 2, 1, TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{0, 0, 0, 1, 0, 0, 0, 0}, + "device coordinate (0, 0, 0, 1) in 'device_assignment' is " + "outside of mesh shape (2, 1, 1, 1)"), + std::make_tuple(2, 1, + TopologyWithDeviceCoordinates({0, 0, 0, 0, 1, 0, 0, 0}), + std::vector{0, 0, 0, 0, 0, 0, 0, 0}, + "'device_assignment' has duplicate device coordinate " + "(0, 0, 0, 0)"))); + +std::vector MakeDeviceSet(int num_tasks, + int num_devices_per_task) { + std::vector devices{ + "/job:localhost/replica:0/task:0/device:CPU:0"}; + devices.reserve(num_tasks * num_devices_per_task + num_tasks + 1); + + for (int task = 0; task < num_tasks; ++task) { + devices.push_back( + llvm::formatv("/job:worker/replica:0/task:{0}/device:CPU:0", task) + .str()); + devices.push_back( + llvm::formatv("/job:worker/replica:0/task:{0}/device:TPU_SYSTEM:0", + task) + .str()); + for (int device = 0; device < num_devices_per_task; ++device) + devices.push_back( + llvm::formatv("/job:worker/replica:0/task:{0}/device:TPU:{1}", task, + device) + .str()); + } + + return devices; +} + +TEST(TPURewriteDeviceUtilTest, + BadGeneralDeviceAssignmentMetadataMissingDevice) { + tpu::TopologyProto topology_proto; + { + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(1); + topology_proto.set_num_tasks(1); + topology_proto.set_num_tpu_devices_per_task(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + } + + std::string topology_attr = topology_proto.SerializeAsString(); + std::vector device_assignment_attr{1, 0, 0, 0}; + + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/1, /*num_devices_per_task=*/1); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, /*num_replicas=*/1, /*num_cores_per_replica=*/1, topology_attr, + device_assignment_attr); + + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), + "no TPU device found for 'device_assignment' device coordinate (1, " + "0, 0, 0)"); +} + +TEST(TPURewriteDeviceUtilTest, ValidFullMeshDeviceAssignment) { + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/2, /*num_devices_per_task=*/4); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + std::string topology_attr; + std::vector device_assignment_attr; + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, /*num_replicas=*/8, /*num_cores_per_replica=*/1, topology_attr, + device_assignment_attr); + + TF_ASSERT_OK(status_or.status()); + + const auto& tpu_device_assignment = status_or.value(); + EXPECT_EQ(tpu_device_assignment.compilation_device, + "/job:worker/replica:0/task:0/device:CPU:0"); + const auto& tpu_devices = tpu_device_assignment.tpu_devices; + ASSERT_EQ(tpu_devices.size(), 8); + for (const auto& replica_tpu_devices : tpu_devices) + ASSERT_EQ(replica_tpu_devices.size(), 1); + + EXPECT_EQ(tpu_devices[0][0].device, + "/job:worker/replica:0/task:0/device:TPU:0"); + EXPECT_EQ(tpu_devices[0][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][0].device, + "/job:worker/replica:0/task:0/device:TPU:1"); + EXPECT_EQ(tpu_devices[1][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[2][0].device, + "/job:worker/replica:0/task:0/device:TPU:2"); + EXPECT_EQ(tpu_devices[2][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[3][0].device, + "/job:worker/replica:0/task:0/device:TPU:3"); + EXPECT_EQ(tpu_devices[3][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[4][0].device, + "/job:worker/replica:0/task:1/device:TPU:0"); + EXPECT_EQ(tpu_devices[4][0].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[5][0].device, + "/job:worker/replica:0/task:1/device:TPU:1"); + EXPECT_EQ(tpu_devices[5][0].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[6][0].device, + "/job:worker/replica:0/task:1/device:TPU:2"); + EXPECT_EQ(tpu_devices[6][0].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[7][0].device, + "/job:worker/replica:0/task:1/device:TPU:3"); + EXPECT_EQ(tpu_devices[7][0].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + + EXPECT_FALSE(tpu_device_assignment.xla_device_assignment.has_value()); +} + +TEST(TPURewriteDeviceUtilTest, ValidGeneralDeviceAssignmentMesh2x2x2) { + tpu::TopologyProto topology_proto; + { + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(2); + topology_proto.set_num_tasks(2); + topology_proto.set_num_tpu_devices_per_task(4); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + } + + std::string topology_attr = topology_proto.SerializeAsString(); + std::vector device_assignment_attr{0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, + 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, + 0, 1, 1, 1, 0, 0, 1, 1, 0, 1}; + + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/2, /*num_devices_per_task=*/4); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, /*num_replicas=*/4, /*num_cores_per_replica=*/2, topology_attr, + device_assignment_attr); + + TF_ASSERT_OK(status_or.status()); + + const auto& tpu_device_assignment = status_or.value(); + EXPECT_EQ(tpu_device_assignment.compilation_device, + "/job:worker/replica:0/task:0/device:CPU:0"); + const auto& tpu_devices = tpu_device_assignment.tpu_devices; + ASSERT_EQ(tpu_devices.size(), 4); + for (const auto& replica_tpu_devices : tpu_devices) + ASSERT_EQ(replica_tpu_devices.size(), 2); + + EXPECT_EQ(tpu_devices[0][0].device, + "/job:worker/replica:0/task:0/device:TPU:0"); + EXPECT_EQ(tpu_devices[0][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[0][1].device, + "/job:worker/replica:0/task:1/device:TPU:3"); + EXPECT_EQ(tpu_devices[0][1].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][0].device, + "/job:worker/replica:0/task:0/device:TPU:1"); + EXPECT_EQ(tpu_devices[1][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][1].device, + "/job:worker/replica:0/task:1/device:TPU:2"); + EXPECT_EQ(tpu_devices[1][1].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[2][0].device, + "/job:worker/replica:0/task:0/device:TPU:3"); + EXPECT_EQ(tpu_devices[2][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[2][1].device, + "/job:worker/replica:0/task:1/device:TPU:0"); + EXPECT_EQ(tpu_devices[2][1].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[3][0].device, + "/job:worker/replica:0/task:0/device:TPU:2"); + EXPECT_EQ(tpu_devices[3][0].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[3][1].device, + "/job:worker/replica:0/task:1/device:TPU:1"); + EXPECT_EQ(tpu_devices[3][1].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + + auto& xla_device_assignment = tpu_device_assignment.xla_device_assignment; + ASSERT_TRUE(xla_device_assignment.has_value()); + EXPECT_EQ(xla_device_assignment->replica_count(), 4); + EXPECT_EQ(xla_device_assignment->computation_count(), 2); + ASSERT_EQ(xla_device_assignment->computation_devices_size(), 2); + const auto& computation_device_0 = + xla_device_assignment->computation_devices(0); + ASSERT_EQ(computation_device_0.replica_device_ids_size(), 4); + const auto& computation_device_1 = + xla_device_assignment->computation_devices(1); + ASSERT_EQ(computation_device_1.replica_device_ids_size(), 4); + + EXPECT_EQ(computation_device_0.replica_device_ids(0), 0); + EXPECT_EQ(computation_device_0.replica_device_ids(1), 4); + EXPECT_EQ(computation_device_0.replica_device_ids(2), 2); + EXPECT_EQ(computation_device_0.replica_device_ids(3), 6); + EXPECT_EQ(computation_device_1.replica_device_ids(0), 1); + EXPECT_EQ(computation_device_1.replica_device_ids(1), 5); + EXPECT_EQ(computation_device_1.replica_device_ids(2), 3); + EXPECT_EQ(computation_device_1.replica_device_ids(3), 7); +} +TEST(TPURewriteDeviceUtilTest, ValidXLADeviceAssignmentMesh1x2x1x3) { + tpu::TopologyProto topology_proto; + { + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(3); + topology_proto.set_num_tasks(3); + topology_proto.set_num_tpu_devices_per_task(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + } + + std::string topology_attr = topology_proto.SerializeAsString(); + std::vector device_assignment_attr{ + 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0}; + + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/3, /*num_devices_per_task=*/2); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + + auto xla_device_assignment = GetXlaDeviceAssignmentProto( + topology_attr, /*num_replicas=*/2, /*num_cores_per_replica=*/3, + device_assignment_attr); + + TF_ASSERT_OK(xla_device_assignment.status()); + EXPECT_EQ(xla_device_assignment->replica_count(), 2); + EXPECT_EQ(xla_device_assignment->computation_count(), 3); + ASSERT_EQ(xla_device_assignment->computation_devices_size(), 3); + const auto& computation_device_0 = + xla_device_assignment->computation_devices(0); + ASSERT_EQ(computation_device_0.replica_device_ids_size(), 2); + const auto& computation_device_1 = + xla_device_assignment->computation_devices(1); + ASSERT_EQ(computation_device_1.replica_device_ids_size(), 2); + const auto& computation_device_2 = + xla_device_assignment->computation_devices(2); + ASSERT_EQ(computation_device_2.replica_device_ids_size(), 2); + + EXPECT_EQ(computation_device_0.replica_device_ids(0), 1); + EXPECT_EQ(computation_device_0.replica_device_ids(1), 5); + EXPECT_EQ(computation_device_1.replica_device_ids(0), 4); + EXPECT_EQ(computation_device_1.replica_device_ids(1), 0); + EXPECT_EQ(computation_device_2.replica_device_ids(0), 2); + EXPECT_EQ(computation_device_2.replica_device_ids(1), 3); +} + +TEST(TPURewriteDeviceUtilTest, InvalidXLADeviceAssignmentMesh1x2x1x3) { + tpu::TopologyProto topology_proto; + { + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(3); + topology_proto.set_num_tasks(3); + topology_proto.set_num_tpu_devices_per_task(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + } + + std::string topology_attr = topology_proto.SerializeAsString(); + std::vector device_assignment_attr{ + 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0}; + + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/3, /*num_devices_per_task=*/2); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + + auto xla_device_assignment = GetXlaDeviceAssignmentProto( + topology_attr, /*num_replicas=*/2, /*num_cores_per_replica=*/2, + device_assignment_attr); + + EXPECT_THAT(xla_device_assignment, + absl_testing::StatusIs( + absl::StatusCode::kInvalidArgument, + ::testing::HasSubstr( + "must be 'num_replicas' * 'num_cores_per_replica' * "))); +} + +TEST(TPURewriteDeviceUtilTest, ValidGeneralDeviceAssignmentMesh1x2x1x3) { + tpu::TopologyProto topology_proto; + { + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(2); + topology_proto.add_mesh_shape(1); + topology_proto.add_mesh_shape(3); + topology_proto.set_num_tasks(3); + topology_proto.set_num_tpu_devices_per_task(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(1); + topology_proto.add_device_coordinates(0); + topology_proto.add_device_coordinates(2); + } + + std::string topology_attr = topology_proto.SerializeAsString(); + std::vector device_assignment_attr{ + 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0}; + + llvm::SmallVector devices; + std::vector device_names = + MakeDeviceSet(/*num_tasks=*/3, /*num_devices_per_task=*/2); + ASSERT_TRUE(DeviceNamesToParsedNames(device_names, &devices)); + + auto status_or = GetTPUCompilationAndExecutionDevices( + devices, /*num_replicas=*/2, /*num_cores_per_replica=*/3, topology_attr, + device_assignment_attr); + + TF_ASSERT_OK(status_or.status()); + + auto& tpu_device_assignment = status_or.value(); + EXPECT_EQ(tpu_device_assignment.compilation_device, + "/job:worker/replica:0/task:0/device:CPU:0"); + + auto& tpu_devices = tpu_device_assignment.tpu_devices; + ASSERT_EQ(tpu_devices.size(), 2); + for (const auto& replica_tpu_devices : tpu_devices) + ASSERT_EQ(replica_tpu_devices.size(), 3); + + EXPECT_EQ(tpu_devices[0][0].device, + "/job:worker/replica:0/task:1/device:TPU:1"); + EXPECT_EQ(tpu_devices[0][0].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[0][1].device, + "/job:worker/replica:0/task:1/device:TPU:0"); + EXPECT_EQ(tpu_devices[0][1].host, + "/job:worker/replica:0/task:1/device:CPU:0"); + EXPECT_EQ(tpu_devices[0][2].device, + "/job:worker/replica:0/task:2/device:TPU:0"); + EXPECT_EQ(tpu_devices[0][2].host, + "/job:worker/replica:0/task:2/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][0].device, + "/job:worker/replica:0/task:2/device:TPU:1"); + EXPECT_EQ(tpu_devices[1][0].host, + "/job:worker/replica:0/task:2/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][1].device, + "/job:worker/replica:0/task:0/device:TPU:0"); + EXPECT_EQ(tpu_devices[1][1].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(tpu_devices[1][2].device, + "/job:worker/replica:0/task:0/device:TPU:1"); + EXPECT_EQ(tpu_devices[1][2].host, + "/job:worker/replica:0/task:0/device:CPU:0"); + + auto& xla_device_assignment = tpu_device_assignment.xla_device_assignment; + ASSERT_TRUE(xla_device_assignment.has_value()); + EXPECT_EQ(xla_device_assignment->replica_count(), 2); + EXPECT_EQ(xla_device_assignment->computation_count(), 3); + ASSERT_EQ(xla_device_assignment->computation_devices_size(), 3); + const auto& computation_device_0 = + xla_device_assignment->computation_devices(0); + ASSERT_EQ(computation_device_0.replica_device_ids_size(), 2); + const auto& computation_device_1 = + xla_device_assignment->computation_devices(1); + ASSERT_EQ(computation_device_1.replica_device_ids_size(), 2); + const auto& computation_device_2 = + xla_device_assignment->computation_devices(2); + ASSERT_EQ(computation_device_2.replica_device_ids_size(), 2); + + EXPECT_EQ(computation_device_0.replica_device_ids(0), 1); + EXPECT_EQ(computation_device_0.replica_device_ids(1), 5); + EXPECT_EQ(computation_device_1.replica_device_ids(0), 4); + EXPECT_EQ(computation_device_1.replica_device_ids(1), 0); + EXPECT_EQ(computation_device_2.replica_device_ids(0), 2); + EXPECT_EQ(computation_device_2.replica_device_ids(1), 3); +} + +TEST(TPURewriteDeviceUtilTest, TestGetDeviceCoordinates) { + mlir::MLIRContext context; + mlir::Builder builder(&context); + auto device_assignment_attr = builder.getI64ArrayAttr({1, 2, 3}); + auto status_or_device_coodinates = + GetDeviceCoordinates(device_assignment_attr); + ASSERT_TRUE(status_or_device_coodinates.ok()); + auto device_coordinates = status_or_device_coodinates.value(); + EXPECT_EQ(device_coordinates[0], 1); + EXPECT_EQ(device_coordinates[1], 2); + EXPECT_EQ(device_coordinates[2], 3); +} + +TEST(TPURewriteDeviceUtilTest, TestInvalidAttrForDeviceAssignmentDisallowed) { + mlir::MLIRContext context; + mlir::Builder builder(&context); + auto device_assignment_attr = builder.getF32ArrayAttr({1.0, 2.0, 3.0}); + auto status_or_device_coodinates = + GetDeviceCoordinates(device_assignment_attr); + ASSERT_TRUE(!status_or_device_coodinates.ok()); + EXPECT_EQ(status_or_device_coodinates.status().message(), + "bad 'device_assignment' attribute at index 0, not an int"); +} + +TEST(TPURewriteDeviceUtilTest, TestHasModelParallelismFalse) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + EXPECT_FALSE(HasModelParallelism(cluster)); +} + +TEST(TPURewriteDeviceUtilTest, TestHasModelParallelismTrue) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 5)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + EXPECT_TRUE(HasModelParallelism(cluster)); +} + +TEST(TPURewriteDeviceUtilTest, + TestHasModelParallelismFalseMissingCoresPerReplicaAttr) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + EXPECT_FALSE(HasModelParallelism(cluster)); +} + +TEST(TPURewriteDeviceUtilTest, + TestGetHostFailNumCoresPerReplicaMissingAttributes) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + mlir::TF::RuntimeDevices devices; + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostFailDeviceMissingAttributes) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + + mlir::TF::RuntimeDevices devices; + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceFailMissingTopology) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + mlir::TF::RuntimeDevices runtime_devices; + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceFailMissingDeviceAssignment) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + + mlir::TF::RuntimeDevices runtime_devices; + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceFailBadDeviceAssignment) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, + builder.getStrArrayAttr(llvm::ArrayRef( + {"bad_device_assigment"}))); + + mlir::TF::RuntimeDevices runtime_devices; + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceFailBadDeviceName) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + (*module_ref) + ->setAttr("tf.devices", + builder.getStrArrayAttr( + llvm::ArrayRef({"bad_device_name"}))); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + mlir::TF::RuntimeDevices runtime_devices; + (void)GetDevicesFromOp(*module_ref, &runtime_devices); + std::string host_device; + EXPECT_TRUE(mlir::failed( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceTPUReplicate) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallDenseMap> + devices; + auto replicate = mlir::tf_device::ReplicateOp::create( + builder, mlir::UnknownLoc::get(&context), /*num_replicas=*/2, devices, + llvm::ArrayRef>{}, + mlir::ValueRange{}, mlir::TypeRange{}); + builder.setInsertionPoint(&replicate.getBody().front(), + replicate.getBody().front().begin()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + + mlir::TF::RuntimeDevices runtime_devices; + std::string host_device; + EXPECT_TRUE(mlir::succeeded( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); + EXPECT_EQ(host_device, GetDeviceAliasForHostOfLogicalCore(0)); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceNotReplicated) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + (*module_ref) + ->setAttr("tf.devices", + builder.getStrArrayAttr(llvm::ArrayRef( + {"/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", + "/job:localhost/replica:0/task:0/device:TPU:0", + "/job:localhost/replica:0/task:0/device:CPU:0", + "/job:worker/replica:0/task:0/device:CPU:0"}))); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->setAttr(kNumCoresPerReplicaAttr, + builder.getIntegerAttr(builder.getIntegerType(64), 1)); + cluster->setAttr(kTopologyAttr, builder.getStringAttr("")); + cluster->setAttr(kDeviceAssignmentAttr, builder.getArrayAttr({})); + + mlir::TF::RuntimeDevices runtime_devices; + (void)GetDevicesFromOp(*module_ref, &runtime_devices); + std::string host_device; + EXPECT_TRUE(mlir::succeeded( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); + EXPECT_EQ(host_device, "/job:localhost/replica:0/task:0/device:CPU:0"); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceInGenericPipeline) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + (*module_ref) + ->setAttr("tf.devices", + builder.getStrArrayAttr(llvm::ArrayRef( + {"/job:localhost/replica:0/task:0/device:CPU:0"}))); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + + mlir::TF::RuntimeDevices runtime_devices; + (void)GetDevicesFromOp(*module_ref, &runtime_devices); + std::string host_device; + EXPECT_TRUE(mlir::succeeded( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); + EXPECT_EQ(host_device, "/job:localhost/replica:0/task:0/device:CPU:0"); +} + +TEST(TPURewriteDeviceUtilTest, TestGetHostDeviceInGenericPipelineMultiCPUs) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + (*module_ref) + ->setAttr("tf.devices", + builder.getStrArrayAttr(llvm::ArrayRef( + {"/job:chief/replica:0/task:0/device:CPU:0", + "/job:ps/replica:0/task:0/device:CPU:0", + "/job:ps/replica:0/task:1/device:CPU:0", + "/job:worker/replica:0/task:2/device:CPU:0"}))); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + + mlir::TF::RuntimeDevices runtime_devices; + (void)GetDevicesFromOp(*module_ref, &runtime_devices); + std::string host_device; + EXPECT_TRUE(mlir::succeeded( + GetHostDeviceOutsideComputation(runtime_devices, cluster, &host_device))); + EXPECT_EQ(host_device, "/job:chief/replica:0/task:0/device:CPU:0"); +} + +TEST(TPURewriteDeviceUtilTest, TestIsTPUDevice) { + EXPECT_TRUE(IsTPUDevice("/job:localhost/replica:0/task:0/device:TPU:0")); + EXPECT_FALSE(IsTPUDevice("/job:localhost/replica:0/task:0/device:CPU:0")); + EXPECT_FALSE(IsTPUDevice("INVALID_DEVICE")); +} + +TEST(TPURewriteDeviceUtilTest, TestDeviceToHostMapBadTopology) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}} { + func.func @main() -> () { + "tf_device.cluster"() ({ + tf_device.return + }) {device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], num_cores_per_replica = 2 : i64} : () -> () + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + llvm::SmallVector core_to_host; + EXPECT_TRUE(mlir::failed(GetDeviceToHostMap(cluster, core_to_host))); +} + +TEST(TPURewriteDeviceUtilTest, TestDeviceToHostMapBadDeviceAssignment) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}} { + func.func @main() -> () { + "tf_device.cluster"() ({ + tf_device.return + }) {num_cores_per_replica = 2 : i64, topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01*\02\08\01"} : () -> () + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + llvm::SmallVector core_to_host; + EXPECT_TRUE(mlir::failed(GetDeviceToHostMap(cluster, core_to_host))); +} + +// Tests `GetDeviceToHostMap` on a non-replicated TPU cluster. +TEST(TPURewriteDeviceUtilTest, TestDeviceToHostMapNotReplicated) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}} { + func.func @main() -> () { + "tf_device.cluster"() ({ + tf_device.return + }) {device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], num_cores_per_replica = 2 : i64, topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01*\02\08\01"} : () -> () + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + llvm::SmallVector core_to_host; + EXPECT_TRUE(mlir::succeeded(GetDeviceToHostMap(cluster, core_to_host))); + EXPECT_EQ(core_to_host.size(), 2); + EXPECT_EQ(core_to_host[0], "/job:localhost/replica:0/task:0/device:CPU:0"); + EXPECT_EQ(core_to_host[1], "/job:localhost/replica:0/task:0/device:CPU:0"); +} + +// Tests `GetDeviceToHostMap` on a replicated TPU cluster. +TEST(TPURewriteDeviceUtilTest, TestDeviceToHostMapReplicated) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU:2", "/job:localhost/replica:0/task:0/device:TPU:3", "/job:localhost/replica:0/task:0/device:TPU:4", "/job:localhost/replica:0/task:0/device:TPU:5", "/job:localhost/replica:0/task:0/device:TPU:6", "/job:localhost/replica:0/task:0/device:TPU:7", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}} { + func.func @main() -> () { + tf_device.replicate() {n = 4 : i32} { + "tf_device.cluster"() ({ + tf_device.return + }) {device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1], num_cores_per_replica = 2 : i64, topology = "\0A\04\02\02\01\02\10\01\18\08\22 \00\00\00\00\00\00\00\01\01\00\00\00\01\00\00\01\00\01\00\00\00\01\00\01\01\01\00\00\01\01\00\01*\02\08\01"} : () -> () + tf_device.return + } + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + llvm::SmallVector core_to_host; + EXPECT_TRUE(mlir::succeeded(GetDeviceToHostMap(cluster, core_to_host))); + EXPECT_EQ(core_to_host.size(), 2); + EXPECT_EQ(core_to_host[0], "TPU_REPLICATED_HOST_0"); + EXPECT_EQ(core_to_host[1], "TPU_REPLICATED_HOST_1"); +} + +// Tests `GetDeviceToHostMap` on a CPU cluster. +TEST(TPURewriteDeviceUtilTest, TestDeviceToHostMapCPU) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0"}} { + func.func @main() -> () { + "tf_device.cluster"() ({ + tf_device.return + }) {} : () -> () + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + llvm::SmallVector core_to_host; + EXPECT_TRUE(mlir::succeeded(GetDeviceToHostMap(cluster, core_to_host))); + EXPECT_EQ(core_to_host.size(), 1); + EXPECT_EQ(core_to_host[0], "/job:localhost/replica:0/task:0/device:CPU:0"); +} + +} // anonymous namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..5f6cafbb8e1f5f503d557b09d429099dd2ec6b3d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.cc @@ -0,0 +1,357 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Interfaces/DerivedAttributeOpInterface.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/export_utils.h" +#include "tensorflow/compiler/mlir/utils/string_container_utils.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_def_builder.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +void PopulateTfVersions(mlir::ModuleOp module, const VersionDef& versions) { + mlir::Builder b(module.getContext()); + auto producer = + b.getNamedAttr("producer", b.getI32IntegerAttr(versions.producer())); + auto min_consumer = b.getNamedAttr( + "min_consumer", b.getI32IntegerAttr(versions.min_consumer())); + auto bad_consumers = b.getNamedAttr( + "bad_consumers", + b.getI32ArrayAttr(llvm::ArrayRef( + versions.bad_consumers().data(), + versions.bad_consumers().data() + versions.bad_consumers().size()))); + module->setAttr("tf.versions", + b.getDictionaryAttr(llvm::ArrayRef( + {producer, min_consumer, bad_consumers}))); +} + +mlir::LogicalResult ExtractTfVersions(mlir::ModuleOp module, + VersionDef* versions) { + versions->Clear(); + auto version_attr = + module->getAttrOfType("tf.versions"); + if (!version_attr) return mlir::failure(); + + auto producer = + mlir::dyn_cast_or_null(version_attr.get("producer")); + if (!producer) return mlir::failure(); + versions->set_producer(producer.getInt()); + + auto min_consumer = mlir::dyn_cast_or_null( + version_attr.get("min_consumer")); + if (min_consumer) versions->set_min_consumer(min_consumer.getInt()); + + auto bad_consumers = mlir::dyn_cast_or_null( + version_attr.get("bad_consumers")); + if (!bad_consumers) return mlir::success(); + + for (auto bad_consumer : bad_consumers) { + auto bad_consumer_int_attr = + mlir::dyn_cast_or_null(bad_consumer); + if (!bad_consumer_int_attr) return mlir::failure(); + + versions->mutable_bad_consumers()->Add(bad_consumer_int_attr.getInt()); + } + return mlir::success(); +} + +absl::StatusOr GetTfGraphProducerVersion(mlir::ModuleOp module) { + auto versions = module->getAttrOfType<::mlir::DictionaryAttr>("tf.versions"); + if (!versions) { + return errors::Internal( + "Missing 'tf.versions' attribute on the module, abort.\n"); + } + auto producer = mlir::dyn_cast(versions.get("producer")); + if (!producer) { + return errors::Internal( + "Missing 'producer' attribute on the module, abort.\n"); + } + return producer.getInt(); +} + +namespace { + +// Sets type list attribute with the given `name` to the given `types`. If the +// attribute already exists with a different value, returns an error. +template () + .begin())>::value>::type> +absl::Status SetTypeAttribute(absl::string_view name, ContainerT types, + AttrValueMap* values) { + AttrValue value; + auto& type_list = *value.mutable_list(); + for (auto type : types) { + DataType dtype; + TF_RETURN_IF_ERROR(ConvertScalarTypeToDataType(type, &dtype)); + type_list.add_type(dtype); + } + + auto result = values->insert({std::string(name), value}); + assert(result.second && "cannot have multiple attributes with the same name"); + (void)result; + + return absl::OkStatus(); +} + +// Sets shape list attribute with the given `name` to the given `shapes`. If the +// attribute already exists then this will just retain the set value. +template >, + decltype(*std::declval().begin())>::value>::type> +void SetShapeAttribute(absl::string_view name, ContainerT shapes, + AttrValueMap* values) { + AttrValue value; + auto& shape_list = *value.mutable_list(); + for (const std::optional>& shape : shapes) { + TensorShapeProto& tshape = *shape_list.add_shape(); + if (shape.has_value()) { + for (int64_t dim : *shape) { + tshape.add_dim()->set_size(mlir::ShapedType::isDynamic(dim) ? -1 : dim); + } + } else { + tshape.set_unknown_rank(true); + } + } + + // If shape is already set, override it. This can happen if we import + // without shape inference enabled and so couldn't be removed on import and + // are not explicitly dropped later. + (*values)[std::string(name)] = value; +} + +// Collects all the unregistered attributes for an TF dialect operation. +// Attributes "name" and "device" are not included because they are not part +// of an TF op attributes. +absl::Status GetUnregisteredAttrs( + mlir::Operation* inst, const tensorflow::OpRegistrationData* op_reg_data, + absl::flat_hash_set* attrs_to_ignore) { + if (!op_reg_data) { + // This is likely a function call node, so we should continue. + return absl::OkStatus(); + } + + // Collect all the registered attributes. + llvm::DenseSet registered_attrs; + registered_attrs.insert("name"); + registered_attrs.insert("device"); + for (const auto& attr_def : op_reg_data->op_def.attr()) { + registered_attrs.insert(attr_def.name()); + } + // Attributes are not in the registered attributes set will be ignored. + for (auto& attr : inst->getAttrs()) { + if (registered_attrs.find(attr.getName()) == registered_attrs.end()) { + attrs_to_ignore->insert( + absl::string_view(attr.getName().data(), attr.getName().size())); + } + } + return absl::OkStatus(); +} + +// Collects all attribute names to ignore in an MLIR operation when exporting to +// a TensorFlow NodeDef. +absl::StatusOr> GetAttributesToIgnore( + mlir::Operation* inst, mlir::DictionaryAttr derived_attrs, + const tensorflow::OpRegistrationData* op_reg_data, + bool ignore_unregistered_attrs) { + // The elements are owned by the MLIRContext. + absl::flat_hash_set attrs_to_ignore; + + // We ignore attributes attached to the operation when there is already a + // derived attribute defined in ODS. + if (derived_attrs) { + for (auto derived_attr : derived_attrs) { + attrs_to_ignore.insert( + mlir::StringRefToView(derived_attr.getName().strref())); + } + } + + if (ignore_unregistered_attrs) { + TF_RETURN_IF_ERROR( + GetUnregisteredAttrs(inst, op_reg_data, &attrs_to_ignore)); + } + + if (inst->hasTrait()) { + // TODO(b/146937733): Don't use here. + llvm::StringRef attr_name = mlir::OpTrait::AttrSizedOperandSegments< + void>::getOperandSegmentSizeAttr(); + attrs_to_ignore.insert(attr_name.data()); + } + + if (inst->hasTrait()) { + // TODO(b/146937733): Don't use here. + llvm::StringRef attr_name = mlir::OpTrait::AttrSizedResultSegments< + void>::getResultSegmentSizeAttr(); + attrs_to_ignore.insert(attr_name.data()); + } + + if (llvm::isa(inst)) + attrs_to_ignore.insert("is_stateless"); + + if (llvm::isa(inst)) + attrs_to_ignore.insert("shape_invariant"); + + return attrs_to_ignore; +} + +// Populates all derived attributes of a MLIR operation in a proto +// map. +absl::Status PopulateDerivedAttributes(mlir::Operation* inst, + llvm::StringRef name, + mlir::DictionaryAttr derived_attrs, + bool ignore_unregistered_attrs, + AttrValueMap* attributes) { + if (derived_attrs) { + TF_RETURN_WITH_CONTEXT_IF_ERROR( + ConvertAttributes(derived_attrs.getValue(), /*attrs_to_ignore=*/{}, + /*remove_ref_type=*/true, attributes), + "while converting derived attributes for node: ", + mlir::StringRefToView(name)); + } + + // Here we only add the shapes for the leading values with ShapedType, + // assuming values with non-ShapedType are put at the end of the result. + if (!ignore_unregistered_attrs && inst->getNumResults() > 0) { + auto values = inst->getResults(); + auto begin = values.begin(); + auto end = values.begin(); + while (end != values.end() && mlir::isa((*end).getType())) + end++; + if (begin != end) { + mlir::TF::ResultShapeRange output_shapes = { + mlir::TF::ResultShapeIterator(begin), + mlir::TF::ResultShapeIterator(end)}; + SetShapeAttribute("_output_shapes", output_shapes, attributes); + } + } + + return absl::OkStatus(); +} + +// A `Cast` with DstT == SrcT can be introduced in MLIR as a shape cast. But +// `Cast` only has shapes in the TF dialect's types, not TF graph, so it is +// valid to convert a `Cast` to an `Identity`. The `_output_shapes` attribute of +// the `Cast` will be preserved. This transform is needed for the graph to be +// executed on TPU or GPU devices, which do not have `Cast` registered as a +// runtime OpKernel. +void RemoveIdentityCast(NodeDef* node_def) { + auto attr = node_def->mutable_attr(); + if (node_def->op() == "Cast" && attr->contains("SrcT") && + attr->contains("DstT") && + attr->at("SrcT").type() == attr->at("DstT").type() && + attr->contains("Truncate") && !attr->at("Truncate").b()) { + node_def->set_op("Identity"); + attr->insert({{"T", attr->at("SrcT")}}); + attr->erase("SrcT"); + attr->erase("DstT"); + attr->erase("Truncate"); + } +} + +} // namespace + +absl::Status GetAttrValuesFromOperation( + mlir::Operation* inst, llvm::StringRef name, + const tensorflow::OpRegistrationData* op_reg_data, + bool ignore_unregistered_attrs, AttrValueMap* attributes) { + mlir::DictionaryAttr derived_attrs = nullptr; + if (auto interface = llvm::dyn_cast(inst)) + derived_attrs = interface.materializeDerivedAttributes(); + TF_ASSIGN_OR_RETURN(auto attrs_to_ignore, + GetAttributesToIgnore(inst, derived_attrs, op_reg_data, + ignore_unregistered_attrs)); + TF_RETURN_WITH_CONTEXT_IF_ERROR( + ConvertAttributes(inst->getAttrs(), attrs_to_ignore, + /*remove_ref_type=*/false, attributes), + "while converting attributes for node: ", mlir::StringRefToView(name)); + TF_RETURN_IF_ERROR(PopulateDerivedAttributes( + inst, name, derived_attrs, ignore_unregistered_attrs, attributes)); + + // Explicitly handle XlaHostCompute op which has required function attribute + // in TensorFlow op def but it could have an empty value to represent missing + // functions. This value can't be represented using MLIR SymbolRefAttr and + // instead uses optional symbol ref attribute. + // + // TODO(b/182315488): Remove custom handling by finding a better + // representation in MLIR for empty function names. One option could be to use + // TensorFlow op defs to figure out function attributes that are missing in + // MLIR. This will also require some trait to identify optional attributes in + // MLIR. + constexpr char kShapeInferenceGraph[] = "shape_inference_graph"; + if (mlir::isa(inst) && + !inst->hasAttr(kShapeInferenceGraph) && + !attrs_to_ignore.contains(kShapeInferenceGraph)) { + AttrValue value; + value.mutable_func()->set_name(""); + (*attributes)[kShapeInferenceGraph] = value; + } + return absl::OkStatus(); +} + +absl::StatusOr> ConvertTFDialectOpToNodeDef( + mlir::Operation* inst, llvm::StringRef name, + bool ignore_unregistered_attrs) { + TF_ASSIGN_OR_RETURN(auto node_def, GetOperationNodeDef(inst, name)); + TF_ASSIGN_OR_RETURN(auto op_name, + GetTensorFlowOpName(inst->getName().getStringRef())); + const tensorflow::OpRegistrationData* op_reg_data = + tensorflow::OpRegistry::Global()->LookUp(op_name.str()); + TF_RETURN_IF_ERROR(GetAttrValuesFromOperation(inst, name, op_reg_data, + ignore_unregistered_attrs, + node_def->mutable_attr())); + RemoveIdentityCast(node_def.get()); + if (op_reg_data) { + ::tensorflow::AddDefaultsToNodeDef(op_reg_data->op_def, node_def.get()); + } + + return node_def; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..4d52e9e61f18ae78820b55431c7e2be7f2ffb048 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h @@ -0,0 +1,69 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TRANSLATE_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TRANSLATE_UTILS_H_ + +#include + +#include "absl/status/statusor.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op_def_builder.h" +#include "tensorflow/core/framework/versions.pb.h" + +namespace tensorflow { + +// Populates the tf.versions attribute on a module, given a corresponding +// graph VersionDef proto. +void PopulateTfVersions(mlir::ModuleOp module, const VersionDef& versions); + +// Extracts TensorFlow GraphDef version information from the given module. +// Returns failure if version attribute is missing or any of the sub attributes +// are invalid. +mlir::LogicalResult ExtractTfVersions(mlir::ModuleOp module, + VersionDef* versions); + +// Returns TensorFlow GraphDef producer version for the given module. Returns an +// error if the version information is missing for the module or is not valid. +absl::StatusOr GetTfGraphProducerVersion(mlir::ModuleOp module); + +// Extracts the attributes of a MLIR operation and populates the converted +// attributes in a proto map. +absl::Status GetAttrValuesFromOperation( + mlir::Operation* inst, llvm::StringRef name, + const tensorflow::OpRegistrationData* op_reg_data, + bool ignore_unregistered_attrs, AttrValueMap* attributes); + +// Converts a MLIR operation to TensorFlow NodeDef with given node name. This +// name should be unique to the graph it is being inserted to. If the +// `ignore_unregistered_attrs` argument is set to true, the attributes which are +// not in the op registry will be ignored. If the `ignore_unregistered_attrs` +// argument is not set to true, _output_shapes attribute is added to nodes with +// ShapedType for the leading values with ShapedType in the results of the +// nodes. Set it to true if the returned NodeDef will be executed by the linked +// TF Eager runtime. +absl::StatusOr> ConvertTFDialectOpToNodeDef( + mlir::Operation* inst, llvm::StringRef name, + bool ignore_unregistered_attrs); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_TRANSLATE_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..29ee373429391fe3e7d89772b7c627ec9b33d7c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.cc @@ -0,0 +1,41 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h" + +#include + +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TF { + +LogicalResult VerifyShapeOfReshapeOp(ArrayRef shape) { + bool has_dynamic_dim = false; + for (int64_t dim : shape) { + if (dim != ShapedType::kDynamic) { + if (dim < 0) return failure(); + continue; + } + if (has_dynamic_dim) return failure(); + has_dynamic_dim = true; + } + return success(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..3dfcf0caac8e39aa825dde12bd347b46c59d1a43 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verification_utils.h @@ -0,0 +1,33 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFICATION_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFICATION_UTILS_H_ + +#include + +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Returns success when the given shape argument of the Reshape op is valid. +LogicalResult VerifyShapeOfReshapeOp(ArrayRef shape); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFICATION_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.cc new file mode 100644 index 0000000000000000000000000000000000000000..2feca72bca5e0241217a75ddaa1e04a3dca0ed46 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.cc @@ -0,0 +1,73 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" + +namespace tensorflow { +namespace { + +constexpr char kInvalidExecutorGraphMsg[] = + "functions must be of a single Graph with single op Islands: "; + +} // namespace + +mlir::LogicalResult VerifyExportSuitable(mlir::ModuleOp module) { + mlir::WalkResult result = module.walk([&](mlir::func::FuncOp function) { + if (!llvm::hasSingleElement(function)) { + function.emitError(kInvalidExecutorGraphMsg) + << "only single block functions are supported"; + return mlir::WalkResult::interrupt(); + } + + auto block = function.front().without_terminator(); + auto graph = llvm::dyn_cast(block.begin()); + if (!graph) { + block.begin()->emitError(kInvalidExecutorGraphMsg) + << "first op in function is not a tf_executor.graph"; + return mlir::WalkResult::interrupt(); + } + + if (!hasSingleElement(block)) { + function.emitError(kInvalidExecutorGraphMsg) + << "function does not only contain a single tf_executor.graph"; + return mlir::WalkResult::interrupt(); + } + + for (mlir::Operation& op : graph.GetBody()) { + auto island = llvm::dyn_cast(op); + if (!island) continue; + + if (!island.WrapsSingleOp()) { + island.emitError(kInvalidExecutorGraphMsg) + << "tf_executor.island must perfectly wrap a single op"; + return mlir::WalkResult::interrupt(); + } + } + + return mlir::WalkResult::advance(); + }); + + return mlir::failure(result.wasInterrupted()); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h new file mode 100644 index 0000000000000000000000000000000000000000..eb6600b8b1eccc4b0db139a2b18d0c28769b65b4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h @@ -0,0 +1,31 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFY_SUITABLE_FOR_GRAPH_EXPORT_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFY_SUITABLE_FOR_GRAPH_EXPORT_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace tensorflow { + +// Returns whether all functions in module are of single tf_executor.graph and +// each tf_executor.island in tf_executor.graph only has a single op. +mlir::LogicalResult VerifyExportSuitable(mlir::ModuleOp module); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VERIFY_SUITABLE_FOR_GRAPH_EXPORT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.cc new file mode 100644 index 0000000000000000000000000000000000000000..9c020bf630cf044873be62be3b3ff14de4f0d81a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.cc @@ -0,0 +1,133 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/visitor.h" + +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TF { + +WalkResult WalkReachableFunctions( + func::FuncOp func, + llvm::function_ref callback, + SymbolTableCollection* symbol_table) { + llvm::SmallDenseSet visited; + + llvm::SmallVector stack; + stack.push_back(func); + + while (!stack.empty()) { + func::FuncOp f = stack.back(); + stack.pop_back(); + + if (!visited.insert(f).second) { + continue; + } + + WalkResult result = callback(f); + if (result.wasInterrupted()) { + return result; + } else if (result.wasSkipped()) { + continue; + } + + result = f.walk([&](Operation* op) { + const auto uses = SymbolTable::getSymbolUses(op); + if (!uses.has_value()) { + op->emitOpError() << "contains a potentially unknown symbol table"; + return WalkResult::interrupt(); + } + + for (const SymbolTable::SymbolUse& use : *uses) { + func::FuncOp called_func = + symbol_table != nullptr + ? symbol_table->lookupNearestSymbolFrom( + use.getUser(), use.getSymbolRef()) + : SymbolTable::lookupNearestSymbolFrom< + func::FuncOp>(use.getUser(), use.getSymbolRef()); + if (called_func == nullptr) { + op->emitOpError() + << "refers to an unknown symbol (expects a function)"; + return WalkResult::interrupt(); + } + stack.push_back(called_func); + } + + return WalkResult::advance(); + }); + if (result.wasInterrupted()) { + return result; + } + } + + return WalkResult::advance(); +} + +FailureOr> CreatePrunedModule( + ModuleOp module, llvm::ArrayRef function_names) { + SymbolTableCollection symbol_table; + OpBuilder builder(module.getContext()); + + OwningOpRef pruned = ModuleOp::create( + builder, module->getLoc()); /*ALLOW_MLIR_MODULE_OP_CREATE*/ + (*pruned)->setAttrs(module->getAttrs()); + builder.setInsertionPointToEnd(pruned->getBody()); + + llvm::SmallDenseSet added; + for (const llvm::StringRef function_name : function_names) { + auto func = + llvm::dyn_cast_or_null(symbol_table.lookupSymbolIn( + module, builder.getStringAttr(function_name))); + if (func == nullptr) { + return module.emitError() + << "Cannot find function '" << function_name << "'"; + } + + const WalkResult result = WalkReachableFunctions( + func, + [&](func::FuncOp f) { + if (!added.insert(f).second) { + return WalkResult::skip(); + } + builder.clone(*f); + return WalkResult::advance(); + }, + &symbol_table); + if (result.wasInterrupted()) { + return failure(); + } + } + + return pruned; +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.h new file mode 100644 index 0000000000000000000000000000000000000000..2480403fabe6366bd226c99a79dec4744df0a6bc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/visitor.h @@ -0,0 +1,52 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VISITOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VISITOR_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TF { + +// Walks the function by following function call chains and calling the callback +// for each reachable function (including `func`). Each function is visited only +// once even if it's called from multiple places and/or recursively. +// +// The current implementation follows direct calls to `mlir::func::FuncOp` only +// and returns a `mlir::WalkResult::interrupt()` when it encounters a call whose +// callee cannot be resolved to `mlir::func::FuncOp`. +mlir::WalkResult WalkReachableFunctions( + mlir::func::FuncOp func, + llvm::function_ref callback, + mlir::SymbolTableCollection* symbol_table = nullptr); + +// Creates a new MLIR module that contains only the given functions and all +// reachable functions from them. +mlir::FailureOr> CreatePrunedModule( + mlir::ModuleOp module, llvm::ArrayRef function_names); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_VISITOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_call_module_attrs.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_call_module_attrs.h new file mode 100644 index 0000000000000000000000000000000000000000..f02fb328dbe89089464cb5971ee527fdc0335b89 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_call_module_attrs.h @@ -0,0 +1,48 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_CALL_MODULE_ATTRS_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_CALL_MODULE_ATTRS_H_ + +#include "llvm/ADT/StringRef.h" + +namespace mlir { +namespace TF { + +// The main function's name in the serialized stablehlo module embedded in +// XlaCallModule's `module` attribute. +constexpr llvm::StringRef kStablehloMainFunctionName = "main"; + +// After deserializing the stablehlo functions from XlaCallModule, +// this XlaCallModule attribute refers to the deserialized stablehlo main +// function. +constexpr llvm::StringRef kStablehloEntryFunctionAttrName = "_entry_function"; + +// The StableHLO version of the serialized stablehlo module embedded in +// XlaCallModule's `module` attribute, set on deserialization. +constexpr llvm::StringRef kStablehloVersionAttrName = "_stablehlo_version"; + +// Every stablehlo function deserialized from XlaCallModule has this attribute. +constexpr llvm::StringRef kFromXlaCallModuleAttrName = "_from_xla_call_module"; + +// Name of `tf.XlaCallModule`'s dictionary attribute for keeping the +// deserialized stablehlo module's attributes. +constexpr llvm::StringRef kStablehloModuleAttrsAttrName = + "_stablehlo_module_attrs"; + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_CALL_MODULE_ATTRS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..460640de7b4bef970f09efe5a19c2caff9112963 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.cc @@ -0,0 +1,136 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h" + +#include +#include + +#include "absl/log/log.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +mlir::LogicalResult EraseClusterFuncs( + llvm::MutableArrayRef to_be_erased) { + for (auto cluster : to_be_erased) { + auto old_parallel_execute = + cluster->getParentOfType(); + if (!old_parallel_execute) { + LOG(ERROR) << "Parent op of cluster " << cluster.getOperationName().str() + << " is not ParallelExecuteOp."; + return mlir::failure(); + } + + for (auto result : old_parallel_execute.getExecuteOutputs()) { + for (mlir::Operation* user : + llvm::make_early_inc_range(result.getUsers())) { + if (llvm::isa(user)) { + assert(user->use_empty()); + user->erase(); + } + } + } + + for (auto operand : cluster.getOperands()) { + mlir::Operation* def = operand.getDefiningOp(); + if (operand.hasOneUse() && + llvm::isa_and_nonnull(def)) { + operand.dropAllUses(); + def->erase(); + } + } + if (!old_parallel_execute->use_empty()) { + LOG(ERROR) << "Use of parallel execute op " + << old_parallel_execute.getOperationName().str() + << " is not empty."; + return mlir::failure(); + } + old_parallel_execute->erase(); + } + return mlir::success(); +} + +int MovePreservedParallelExecuteChildren( + int num_cores_per_replica, + llvm::SmallVector& concatenated_output_types, + mlir::OpBuilder* builder, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, + mlir::tf_device::ParallelExecuteOp* new_parallel_execute) { + // `num_moved_children` is the number of children that will be preserved. + const size_t num_moved_children = + old_parallel_execute.getRegions().size() - 1; + *new_parallel_execute = mlir::tf_device::ParallelExecuteOp::create( + *builder, old_parallel_execute->getLoc(), + num_moved_children + num_cores_per_replica, concatenated_output_types); + + // `cluster_idx` is the index of the child with the `ClusterFuncOp`, which + // will be replaced. + int cluster_idx = -1; + for (size_t child_idx = 0; + child_idx < old_parallel_execute.getRegions().size(); ++child_idx) { + auto& block = old_parallel_execute.GetRegionBlockWithIndex(child_idx); + if (cluster_func->getBlock() == &block) { + assert(cluster_idx == -1); + cluster_idx = child_idx; + } + } + assert(cluster_idx != -1); + + // Move children that are preserved. + for (int child_idx = 0; child_idx < num_moved_children; ++child_idx) { + int old_idx = child_idx >= cluster_idx ? child_idx + 1 : child_idx; + int new_idx = child_idx >= cluster_idx ? child_idx + num_cores_per_replica + : child_idx; + new_parallel_execute->getRegions()[new_idx].takeBody( + old_parallel_execute.getRegions()[old_idx]); + } + + return cluster_idx; +} + +mlir::tf_device::LaunchOp WrapOpInLaunch(mlir::OpBuilder* builder, + mlir::Location loc, + mlir::Operation* op, + llvm::StringRef device) { + mlir::OpBuilder::InsertPoint insert_point = builder->saveInsertionPoint(); + + auto launch = mlir::tf_device::LaunchOp::create( + *builder, loc, builder->getStringAttr(device), op->getResultTypes()); + launch.getBody().push_back(new mlir::Block); + + builder->setInsertionPointToEnd(&launch.GetBody()); + mlir::tf_device::ReturnOp::create(*builder, loc, op->getResults()); + + // Move op inside cluster. + op->moveBefore(launch.GetBody().getTerminator()); + + builder->restoreInsertionPoint(insert_point); + + return launch; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h new file mode 100644 index 0000000000000000000000000000000000000000..7bcfd1d70d3858558810a54023b9d78eb88d66a9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h @@ -0,0 +1,66 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_REWRITE_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_REWRITE_UTIL_H_ + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/statusor.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { +// Erase rewritten ClusterFuncOp(s). If TPUPartitionedInputV2Op / +// TPUPartitionedOutputV2Op are present, they must be removed along with the +// ClusterFuncOp(s). +mlir::LogicalResult EraseClusterFuncs( + llvm::MutableArrayRef to_be_erased); + +// Move child processes of the ParallelExecute that do not change. These are all +// children except for the child with the ClusterFunc. +// Returns the index of the child with the ClusterFunc. +int MovePreservedParallelExecuteChildren( + int num_cores_per_replica, + llvm::SmallVector& concatenated_output_types, + mlir::OpBuilder* builder, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, + mlir::tf_device::ParallelExecuteOp* new_parallel_execute); + +// Wraps single op in `tf_device.launch` for explicit device assignment. +mlir::tf_device::LaunchOp WrapOpInLaunch(mlir::OpBuilder* builder, + mlir::Location loc, + mlir::Operation* op, + llvm::StringRef device); + +} // namespace tensorflow +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_REWRITE_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..3ef66fe828c2eb782925699a76dd7ebb828731ee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util_test.cc @@ -0,0 +1,121 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h" + +#include + +#include "absl/status/statusor.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tsl/platform/statusor.h" + +// #include +// #include + +namespace tensorflow { +namespace { +absl::StatusOr> GetMlirModuleFromString( + llvm::StringRef string, mlir::MLIRContext* context) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + mlir::OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +TEST(XlaRewriteUtilTest, TestEraseClusterFuncs) { + static const char* const module_str = + R"( +module attributes {tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:GPU:0"]} { + func.func @convert_cluster_func(%arg0: tensor) -> () { + %2 = "tf_device.parallel_execute"() ({ + + %3 = "tf_device.cluster_func"(%arg0) {device = "/job:localhost/replica:0/task:0/device:GPU:0", func = @func} : (tensor) -> tensor + + tf_device.return %3 : tensor + + }) : () -> tensor + return + } + func.func @func(%arg0: tensor) -> tensor { + return %arg0 : tensor + } +} +)"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + llvm::SmallVector cluster_func_ops; + module->walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + cluster_func_ops.push_back(cluster_func); + }); + EXPECT_EQ(cluster_func_ops.size(), 1); + + EXPECT_TRUE(mlir::succeeded(tensorflow::EraseClusterFuncs(cluster_func_ops))); + + llvm::SmallVector new_cluster_func_ops; + module->walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + new_cluster_func_ops.push_back(cluster_func); + }); + EXPECT_EQ(new_cluster_func_ops.size(), 0); +} + +TEST(XlaRewriteUtilTest, TestWrapOpInLaunch) { + static const char* const module_str = + R"( +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0"}} { + func.func @main() -> () { + "tf_device.cluster"() ({ + tf_device.return + }) {} : () -> () + func.return + } +})"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + mlir::tf_device::ClusterOp cluster; + std::string device = "/job:localhost/replica:0/task:0/device:CPU:0"; + module->walk( + [&](mlir::tf_device::ClusterOp descendant) { cluster = descendant; }); + mlir::OpBuilder builder(&context); + auto loc = cluster->getLoc(); + + // Wrap the cluster op into a Launch op + auto launch_op = tensorflow::WrapOpInLaunch(&builder, loc, cluster, device); + + EXPECT_TRUE(llvm::isa(launch_op)); + launch_op->erase(); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..5313a2f6f7fe4a2f98d752eb4908b4be250fcc5e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.cc @@ -0,0 +1,1355 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/hlo/ir/hlo_sharding.h" +#include "xla/hlo/ir/tile_assignment.h" +#include "xla/hlo/parser/hlo_parser.h" +#include "xla/hlo/utils/hlo_sharding_util.h" +#include "xla/tsl/lib/math/math_util.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace { + +constexpr char kNumSplitAttr[] = "num_split"; + +int64_t GetPadding(const int split_dim, const int num_splits, + const PartialTensorShape& partial_tensor_shape) { + // If dim dimension is not defined, no uneven sharding support. + if (partial_tensor_shape.dim_size(split_dim) <= 0) { + return 0; + } + int64_t per_split_size = tsl::MathUtil::CeilOfRatio( + partial_tensor_shape.dim_size(split_dim), num_splits); + int64_t total_padding = + per_split_size * num_splits - partial_tensor_shape.dim_size(split_dim); + return total_padding; +} + +mlir::TF::SliceOp CreateSliceOp(mlir::OpBuilder* builder, + const mlir::Location& location, + mlir::Value input, + const PartialTensorShape& shape) { + mlir::SmallVector slice_start_position; + for (int i = 0; i < shape.dims(); ++i) { + slice_start_position.push_back(0); + } + mlir::SmallVector slice_size; + for (int i = 0; i < shape.dims(); ++i) { + slice_size.push_back(shape.dim_size(i)); + } + + auto start_position_type = + mlir::RankedTensorType::get(shape.dims(), builder->getIntegerType(64)); + + auto start_position_op = + mlir::TF::ConstOp::create(*builder, input.getLoc(), + mlir::DenseIntElementsAttr::get( + start_position_type, slice_start_position)); + + auto slice_size_op = mlir::TF::ConstOp::create( + *builder, input.getLoc(), + mlir::DenseIntElementsAttr::get( + mlir::RankedTensorType::get(shape.dims(), + builder->getIntegerType(64)), + slice_size)); + + auto slice_result_type = + mlir::RankedTensorType::get(slice_size, getElementTypeOrSelf(input)); + + return mlir::TF::SliceOp::create(*builder, input.getLoc(), slice_result_type, + input, start_position_op, slice_size_op); +} + +mlir::TF::PadOp CreatePadOp(mlir::OpBuilder* builder, + const mlir::Location& location, int64_t num_dims, + int64_t split_dim, mlir::Value src_input, + int64_t padding) { + auto input_type = mlir::cast(src_input.getType()); + llvm::SmallVector padding_values; + std::vector padded_shape; + for (int i = 0; i < num_dims; ++i) { + // 0 padding in the beginning. + padding_values.push_back(0); + if (i == split_dim) { + // pad the split dimension to make the total size of the input equal to + // the total size of the split dimension. + padding_values.push_back(padding); + padded_shape.push_back(input_type.getShape()[i] + padding); + } else { + padding_values.push_back(0); + padded_shape.push_back(input_type.getShape()[i]); + } + } + auto padding_type = + mlir::RankedTensorType::get({num_dims, 2}, builder->getIntegerType(64)); + auto paddings = mlir::DenseIntElementsAttr::get(padding_type, padding_values); + auto paddings_value = mlir::TF::ConstOp::create(*builder, location, paddings); + mlir::SmallVector expand_shape(padded_shape.begin(), + padded_shape.end()); + + auto expand_result_type = + mlir::RankedTensorType::get(expand_shape, input_type.getElementType()); + + return mlir::TF::PadOp::create(*builder, location, expand_result_type, + src_input, paddings_value); +} + +// Creates a tf::SplitOp that splits 'src_input' into 'num_splits' ways +// in 'split_dimension' dimension and returns the split values. +mlir::LogicalResult CreateSplitOp( + const int num_split, const int split_dimension, const int64_t padding, + const mlir::Location& location, mlir::Value src_input, + mlir::OpBuilder* builder, mlir::TF::SplitOp* split_op, + bool is_ici_weight_dist_spmd) { + if (padding > 0) { + int64_t num_dims = + mlir::cast(src_input.getType()).getRank(); + auto pad_op = CreatePadOp(builder, location, num_dims, split_dimension, + src_input, padding); + if (is_ici_weight_dist_spmd) { + pad_op->setAttr(kICIWeightDistributionMlirBridgeMarker, + builder->getBoolAttr(true)); + } + src_input = pad_op.getResult(); + } + + // Creates a const op to hold split dimension value. + auto split_dim_type = + mlir::RankedTensorType::get({}, builder->getIntegerType(32)); + auto split_dimension_attr = + mlir::DenseElementsAttr::get(split_dim_type, split_dimension); + + // Correctly set output shapes of split op output if input shape is statically + // known. + mlir::Type output_type; + auto input_type = mlir::cast(src_input.getType()); + + if (input_type.hasRank()) { + if (input_type.getShape()[split_dimension] == mlir::ShapedType::kDynamic) { + output_type = input_type; + } else { + auto shape = llvm::to_vector<4>(input_type.getShape()); + if (shape[split_dimension] % num_split != 0) { + return mlir::emitError( + location, + llvm::formatv( + "incorrect input sharding configuration received. " + "{0}-th dimension of the input must be evenly divisible by {1}", + split_dimension, num_split)); + } + + shape[split_dimension] = shape[split_dimension] / num_split; + output_type = + mlir::RankedTensorType::get(shape, input_type.getElementType()); + } + } else { + output_type = input_type; + } + + auto split_dimension_op = mlir::TF::ConstOp::create( + *builder, location, split_dim_type, split_dimension_attr); + if (is_ici_weight_dist_spmd) { + split_dimension_op->setAttr(kICIWeightDistributionMlirBridgeMarker, + builder->getBoolAttr(true)); + } + + // Creates a split op that splits |src_input| along |split_dimension|. + llvm::SmallVector output_types(num_split, output_type); + *split_op = + mlir::TF::SplitOp::create(*builder, location, output_types, + split_dimension_op.getOutput(), src_input); + (*split_op)->setAttr( + kNumSplitAttr, + builder->getIntegerAttr(builder->getIntegerType(32), num_split)); + if (is_ici_weight_dist_spmd) { + (*split_op)->setAttr(kICIWeightDistributionMlirBridgeMarker, + builder->getBoolAttr(true)); + } + return mlir::success(); +} + +// Creates a tf::ConcatOp that merges `input` values in `concat_dimension`. +mlir::TF::ConcatOp CreateConcatOp(const int concat_dimension, + const mlir::Location& location, + const int64_t padding, + mlir::ArrayRef inputs, + mlir::OpBuilder* builder) { + // Creates a const op to hold concat dimension value. + auto concat_dim_type = + mlir::RankedTensorType::get({}, builder->getIntegerType(32)); + auto concat_dimension_attr = + mlir::DenseElementsAttr::get(concat_dim_type, concat_dimension); + auto concat_dimension_op = mlir::TF::ConstOp::create( + *builder, location, concat_dim_type, concat_dimension_attr); + + // Correctly set output shapes of concat op output if output shape is + // statically known. Since the shape of TPUExecute op must be the same + // across logical devices, we refer to the shape of 0th logical device + // computation output. + mlir::Type output_type; + auto input_type = mlir::cast(inputs[0].getType()); + + if (input_type.hasRank()) { + if (input_type.getShape()[concat_dimension] == mlir::ShapedType::kDynamic) { + output_type = input_type; + } else { + auto shape = llvm::to_vector<4>(input_type.getShape()); + shape[concat_dimension] = shape[concat_dimension] * inputs.size(); + output_type = + mlir::RankedTensorType::get(shape, input_type.getElementType()); + } + } else { + output_type = input_type; + } + + return mlir::TF::ConcatOp::create(*builder, location, output_type, + concat_dimension_op.getOutput(), inputs); +} + +mlir::TF::XlaConcatNDOp CreateXlaConcatNDOp( + const mlir::Location& location, mlir::ArrayRef inputs, + const std::vector& num_concats, + const std::vector& paddings, mlir::OpBuilder& builder) { + llvm::SmallVector output_shape; + if (inputs.empty()) { + mlir::emitError(location, "inputs list to concat ops is empty"); + return nullptr; + } + if (num_concats.size() != paddings.size()) { + mlir::emitError(location, + "num_concats and paddings must be of the same length."); + return nullptr; + } + // All input slices must have the same shape, otherwise XLAConcatND op will + // raise an error. + auto input_slice_type = mlir::cast(inputs[0].getType()); + auto element_type = input_slice_type.getElementType(); + mlir::Type output_type; + if (input_slice_type.hasRank()) { + const auto& slice_shape = input_slice_type.getShape(); + for (int i = 0; i < num_concats.size(); i++) { + auto num_concat = num_concats[i]; + const int max_dim_size = slice_shape[i] * num_concat; + + output_shape.emplace_back(max_dim_size - paddings[i]); + } + VLOG(2) << "SL: CreateXlaConcatNDOp. output_shape=" + << absl::StrJoin(output_shape, ",") + << ", Padding=" << absl::StrJoin(paddings, ","); + output_type = mlir::RankedTensorType::get(output_shape, element_type); + } else { + output_type = input_slice_type; + } + + auto op = mlir::TF::XlaConcatNDOp::create( + builder, location, output_type, inputs, + builder.getI64ArrayAttr(num_concats), builder.getI64ArrayAttr(paddings)); + return op; +} + +mlir::LogicalResult CreateXlaSplitNDOp(const mlir::Location& location, + mlir::Value src_input, + const std::vector& num_splits, + const std::vector& paddings, + mlir::OpBuilder* builder, + mlir::TF::XlaSplitNDOp* xla_split_op, + bool is_ici_weight_dist_spmd) { + auto input_type = mlir::cast(src_input.getType()); + mlir::Type output_type; + if (!input_type.hasRank()) { + mlir::emitError( + location, + "XLA Split/Concat ops are supported only for Ranked Tensors."); + return mlir::failure(); + } + const int rank = input_type.getRank(); + const auto& input_shape = input_type.getShape(); + + auto output_slice_shape = llvm::to_vector<4>(input_type.getShape()); + int num_tiles = 1; + if (num_splits.size() != rank || num_splits.size() != paddings.size()) { + return mlir::failure(); + } + for (int i = 0; i < rank; ++i) { + if (input_shape[i] == mlir::ShapedType::kDynamic) { + output_slice_shape[i] = input_shape[i]; + } else { + output_slice_shape[i] = ((input_shape[i] + paddings[i]) / num_splits[i]); + } + num_tiles *= num_splits[i]; + } + output_type = mlir::RankedTensorType::get(output_slice_shape, + input_type.getElementType()); + + llvm::SmallVector output_types(num_tiles, output_type); + + VLOG(2) << "SL: CreateXlaSplitNDOp. input_shape=" + << absl::StrJoin(input_shape, ",") + << ", Padding: " << absl::StrJoin(paddings, ","); + + *xla_split_op = mlir::TF::XlaSplitNDOp::create( + *builder, location, output_types, src_input, + builder->getI64ArrayAttr(num_splits), builder->getI64ArrayAttr(paddings)); + if (is_ici_weight_dist_spmd) { + (*xla_split_op) + ->setAttr(kICIWeightDistributionMlirBridgeMarker, + builder->getBoolAttr(true)); + } + return mlir::success(); +} + +bool IsShapeKnown(mlir::TensorType type) { + if (!type.hasRank()) return false; + + bool shape_known = false; + for (int i = 0; i < type.getRank(); ++i) { + if (type.getShape()[i] == mlir::ShapedType::kDynamic) { + shape_known = false; + break; + } else { + shape_known = true; + } + } + + return shape_known; +} + +mlir::LogicalResult HandleTileShardedInputsUsingXlaSplitOps( + const mlir::Location& location, const xla::OpSharding& input_sharding, + const mlir::Value& original_source, mlir::OpBuilder* builder, + llvm::SmallVectorImpl* tiled_inputs, + bool is_ici_weight_dist_spmd) { + std::vector num_splits( + input_sharding.tile_assignment_dimensions().begin(), + input_sharding.replicate_on_last_tile_dim() + ? std::prev(input_sharding.tile_assignment_dimensions().end()) + : input_sharding.tile_assignment_dimensions().end()); + + const int rank = input_sharding.replicate_on_last_tile_dim() + ? input_sharding.tile_assignment_dimensions_size() - 1 + : input_sharding.tile_assignment_dimensions_size(); + std::vector paddings; + paddings.reserve(rank); + auto shape = llvm::to_vector<4>( + mlir::cast(original_source.getType()).getShape()); + for (int dim = 0; dim < rank; ++dim) { + paddings.push_back( + GetPadding(dim, input_sharding.tile_assignment_dimensions(dim), + PartialTensorShape(shape))); + } + + mlir::TF::XlaSplitNDOp xla_split_op; + if (mlir::failed(CreateXlaSplitNDOp(location, original_source, num_splits, + paddings, builder, &xla_split_op, + is_ici_weight_dist_spmd))) { + return mlir::failure(); + } + + tiled_inputs->clear(); + tiled_inputs->reserve(input_sharding.tile_assignment_devices_size()); + int64_t repeat_count = + input_sharding.replicate_on_last_tile_dim() + ? *input_sharding.tile_assignment_dimensions().rbegin() + : 1; + for (int i = 0; i < xla_split_op.getResults().size(); i++) { + auto split_op_output = xla_split_op.getResults()[i]; + for (int64_t j = 0; j < repeat_count; ++j) { + tiled_inputs->push_back(split_op_output); + } + } + + return mlir::success(); +} + +// For tile sharded inputs to TPU computation, inject split op between the +// input values and TPU computation so that tiled input values are passed in +// as inputs to TPU computations. If more than one dimension is sharded, then +// a tree of connected split ops are added before tf_device.parallel_execute op. +mlir::LogicalResult HandleTileShardedInputsUsingTfSplitOps( + const mlir::Location& location, const xla::OpSharding& input_sharding, + const mlir::Value& original_source, mlir::OpBuilder* builder, + llvm::SmallVectorImpl* tiled_inputs, + bool is_ici_weight_dist_spmd) { + llvm::SmallVector split_ops_for_tiled_input; + split_ops_for_tiled_input.reserve( + input_sharding.tile_assignment_devices_size()); + + // Creates a tree of split nodes for sharding tiled inputs. Splits nodes + // are created such that input data is sharded in row major order. + // Split nodes at ith depth from the original input node represent nodes + // that split the input data at i-th dimension. + auto dimension_to_splits_map = + GetDimensionIndicesAndNumSplitsFromSharding(input_sharding); + if (!dimension_to_splits_map.ok()) { + LOG(ERROR) << dimension_to_splits_map.status(); + return mlir::failure(); + } + PartialTensorShape shape; + const auto input_type = + mlir::cast(original_source.getType()); + bool input_shape_known = IsShapeKnown(input_type); + if (input_shape_known) { + shape = PartialTensorShape(input_type.getShape()); + } + for (const auto& dimension_and_num_splits : *dimension_to_splits_map) { + const int dimension = dimension_and_num_splits.first; + const int num_splits = dimension_and_num_splits.second; + + int padding = input_shape_known + ? GetPadding(dimension, num_splits, + PartialTensorShape(input_type.getShape())) + : 0; + // Creates root split op. + if (split_ops_for_tiled_input.empty()) { + mlir::TF::SplitOp root_split_op; + auto result = CreateSplitOp(num_splits, dimension, padding, location, + original_source, builder, &root_split_op, + is_ici_weight_dist_spmd); + if (mlir::failed(result)) return mlir::failure(); + + split_ops_for_tiled_input.emplace_back(root_split_op); + continue; + } + + llvm::SmallVector new_split_ops; + new_split_ops.reserve(split_ops_for_tiled_input.size() * num_splits); + + for (auto split_op : split_ops_for_tiled_input) { + for (auto parent_split_output_value : split_op.getResults()) { + mlir::TF::SplitOp child_split_op; + auto result = CreateSplitOp(num_splits, dimension, padding, location, + parent_split_output_value, builder, + &child_split_op, is_ici_weight_dist_spmd); + if (mlir::failed(result)) return mlir::failure(); + + new_split_ops.emplace_back(child_split_op); + } + } + + std::swap(new_split_ops, split_ops_for_tiled_input); + } + + // `split_ops_for_tiled_input` now includes final split nodes + // from which sharded data will be fed into TPUExecute ops -- sorted by + // row major order. + tiled_inputs->clear(); + tiled_inputs->reserve(input_sharding.tile_assignment_devices_size()); + for (auto split_op : split_ops_for_tiled_input) { + for (auto split_op_output : split_op.getResults()) { + int64_t repeat_count = + input_sharding.replicate_on_last_tile_dim() + ? *input_sharding.tile_assignment_dimensions().rbegin() + : 1; + for (int64_t i = 0; i < repeat_count; ++i) { + tiled_inputs->push_back(split_op_output); + } + } + } + + return mlir::success(); +} + +bool UnsupportedPartitionedShardingType(xla::OpSharding::Type sharding) { + return sharding != xla::OpSharding::REPLICATED && + sharding != xla::OpSharding::OTHER; +} + +} // namespace + +absl::StatusOr> GetDimensionIndicesAndNumSplitsFromSharding( + const xla::OpSharding& sharding) { + int64_t tensor_tile_rank = sharding.tile_assignment_dimensions_size(); + if (sharding.replicate_on_last_tile_dim()) { + tensor_tile_rank--; + } + + std::map dimension_to_splits_map; + for (int dim_index = 0; dim_index < tensor_tile_rank; ++dim_index) { + if (sharding.tile_assignment_dimensions(dim_index) > 1) { + dimension_to_splits_map.emplace( + dim_index, sharding.tile_assignment_dimensions(dim_index)); + } + } + + if (dimension_to_splits_map.empty()) { + return absl::InvalidArgumentError(absl::StrCat( + "Arg has unnecessary tiled sharding: ", sharding.DebugString())); + } + + return dimension_to_splits_map; +} + +int GetDimsFromXLAShardingTiled(const xla::OpSharding& xla_sharding) { + return xla_sharding.tile_assignment_dimensions_size() - + (xla_sharding.replicate_on_last_tile_dim() ? 1 : 0) - + xla_sharding.last_tile_dims_size(); +} + +bool IsOtherReplicatedSharding(const xla::OpSharding& xla_sharding) { + int max_dim = GetDimsFromXLAShardingTiled(xla_sharding); + for (int i = 0; i < max_dim; ++i) { + if (xla_sharding.tile_assignment_dimensions(i) != 1) { + return false; + } + } + return xla_sharding.type() == xla::OpSharding::OTHER && + (xla_sharding.replicate_on_last_tile_dim() || + !xla_sharding.last_tile_dims().empty()); +} + +bool IsSplitSharding(const xla::OpSharding& sharding) { + return sharding.type() == xla::OpSharding::OTHER && + !IsOtherReplicatedSharding(sharding); +} + +bool IsReplicatedSharding(const xla::OpSharding& sharding) { + return sharding.type() == xla::OpSharding::REPLICATED || + IsOtherReplicatedSharding(sharding); +} + +mlir::LogicalResult DecodeShardingAttribute(const std::string& shard_str, + xla::OpSharding& sharding, + bool report_error) { + if (sharding.ParseFromString(shard_str)) return mlir::success(); + // TODO(b/287299845) MLIR should only have human readable representation + // going forward. So, remove parsing binary sharding. + absl::StatusOr sharding_hlo = xla::ParseSharding(shard_str); + if (sharding_hlo.ok()) { + sharding = sharding_hlo->ToProto(); + return mlir::success(); + } + if (report_error) + llvm::errs() << std::string(sharding_hlo.status().message()) << "\n"; + return mlir::failure(); +} + +mlir::LogicalResult DecodeShardingAttribute(mlir::Attribute shard_attr, + xla::OpSharding& sharding, + bool report_error) { + if (!mlir::isa(shard_attr)) return mlir::failure(); + + auto shard_str = mlir::cast(shard_attr).getValue().str(); + return DecodeShardingAttribute(shard_str, sharding, report_error); +} + +void EncodeSharding(mlir::Operation* op, llvm::StringRef shard_str) { + if (!op->hasAttrOfType(shard_str)) return; + + ::xla::OpSharding sharding; + auto sharding_proto_str = + op->getAttrOfType(shard_str).getValue().str(); + if (!sharding.ParseFromString(sharding_proto_str)) return; + + auto hlosharding = xla::HloSharding::FromProto(sharding); + if (!hlosharding.ok()) { + op->emitError("Unable to encode sharding to human readable ") + << hlosharding.status().message(); + return; + } + op->setAttr(shard_str, + mlir::StringAttr::get(op->getContext(), hlosharding->ToString())); +} + +mlir::LogicalResult ExtractInputsForLogicalDevices( + const int num_cores_per_replica, + mlir::tf_device::ClusterFuncOp cluster_func, mlir::OpBuilder* builder, + llvm::SmallVectorImpl>* input_list) { + return ExtractInputsForLogicalDevices(num_cores_per_replica, cluster_func, + builder, /*use_xla_nd_ops=*/false, + input_list); +} +mlir::LogicalResult ExtractInputsForLogicalDevices( + const int num_cores_per_replica, + mlir::tf_device::ClusterFuncOp cluster_func, mlir::OpBuilder* builder, + bool use_xla_nd_ops, + llvm::SmallVectorImpl>* input_list) { + // Initialize the input list for each logical devices. + input_list->reserve(num_cores_per_replica); + for (int i = 0; i < num_cores_per_replica; ++i) + input_list->emplace_back(llvm::SmallVector()); + + llvm::SmallVector cluster_func_inputs( + cluster_func.getOperands()); + auto sharding_attrs = + cluster_func.getOperation()->getAttrOfType( + kInputShardingAttr); + // If sharding attribute does not exist, then all inputs are placed on 0th + // logical core by default. + if (!sharding_attrs) { + (*input_list)[0] = cluster_func_inputs; + return mlir::success(); + } + + // Enumerate sharding configuration for each inputs. If input has replicate + // sharding then all logical devices take the value as input. If input has + // maximal sharding then only the specified logical device take the value as + // the input. + for (const auto& sharding_attr_and_index : llvm::enumerate(sharding_attrs)) { + const auto& sharding_attr = sharding_attr_and_index.value(); + const auto input_index = sharding_attr_and_index.index(); + const auto& input_value = cluster_func_inputs[input_index]; + + xla::OpSharding sharding; + if (DecodeShardingAttribute( + mlir::cast(sharding_attr).getValue().str(), + sharding) + .failed()) { + return cluster_func.emitError("incorrect sharding format for inputs"); + } + xla::hlo_sharding_util::ConvertV2ToV1Sharding(sharding); + + const auto input_sharding_type = sharding.type(); + auto tiled_sharding_mismatched = [&](int tiled_input_size) { + return cluster_func.emitError( + llvm::formatv("incorrect {0}-th tiled input sharding received. " + "Product of tile sharding splits({1}) must be equal to " + "number of logical devices : {2}", + input_index, tiled_input_size, num_cores_per_replica)); + }; + + // If input is already partitioned using the `tf.TPUPartitionedInputV2` op, + // only replicated sharding is supported where i-th operand to + // `tf.TPUPartitionedInputV2` op is input to the i-th logical device. + if (auto partitioned_input = + llvm::dyn_cast_or_null( + input_value.getDefiningOp())) { + if (UnsupportedPartitionedShardingType(input_sharding_type)) + return cluster_func->emitOpError() + << "unsupported input sharding type " + << OpSharding_Type_Name(input_sharding_type) << " for " + << input_index << "-th input"; + + if (input_sharding_type == xla::OpSharding::REPLICATED) { + for (const auto& index_and_inputs : llvm::enumerate(*input_list)) { + index_and_inputs.value().emplace_back( + partitioned_input.getOperand(index_and_inputs.index())); + } + } else { + assert(input_sharding_type == xla::OpSharding::OTHER); + if (partitioned_input.getInputs().size() != num_cores_per_replica) + return tiled_sharding_mismatched( + partitioned_input.getInputs().size()); + + for (int i = 0; i < sharding.tile_assignment_devices_size(); ++i) { + const int assigned_logical_device = + sharding.tile_assignment_devices(i); + (*input_list)[assigned_logical_device].emplace_back( + partitioned_input.getInputs()[i]); + } + } + continue; + } + + if (IsSplitSharding(sharding)) { + bool is_ici_weight_dist_spmd = + cluster_func.getOperand(input_index).getDefiningOp() && + cluster_func.getOperand(input_index) + .getDefiningOp() + ->hasAttr(kICIWeightDistributionMlirBridgeMarker); + llvm::SmallVector tiled_inputs; + + if (use_xla_nd_ops) { + auto result = HandleTileShardedInputsUsingXlaSplitOps( + cluster_func.getLoc(), sharding, input_value, builder, + &tiled_inputs, is_ici_weight_dist_spmd); + if (mlir::failed(result)) return mlir::failure(); + } else { + auto result = HandleTileShardedInputsUsingTfSplitOps( + cluster_func.getLoc(), sharding, input_value, builder, + &tiled_inputs, is_ici_weight_dist_spmd); + if (mlir::failed(result)) return mlir::failure(); + } + + const int64_t tiled_inputs_size = tiled_inputs.size(); + if (tiled_inputs_size != num_cores_per_replica) + return tiled_sharding_mismatched(tiled_inputs.size()); + + for (int i = 0; i < sharding.tile_assignment_devices_size(); ++i) { + const int assigned_logical_device = sharding.tile_assignment_devices(i); + (*input_list)[assigned_logical_device].emplace_back(tiled_inputs[i]); + } + } else if (IsReplicatedSharding(sharding)) { + for (auto& inputs : *input_list) inputs.emplace_back(input_value); + } else { + assert(input_sharding_type == xla::OpSharding::MAXIMAL); + const int logical_device_id = sharding.tile_assignment_devices(0); + (*input_list)[logical_device_id].emplace_back(input_value); + } + } + return mlir::success(); +} + +mlir::LogicalResult ParseAndValidateOutputSharding( + const int num_cores_per_replica, + mlir::tf_device::ClusterFuncOp cluster_func, + mlir::SmallVector* output_sharding_list) { + output_sharding_list->reserve(cluster_func.getNumResults()); + + const auto output_sharding_attrs = + cluster_func.getOperation()->getAttrOfType( + kOutputShardingAttr); + if (!output_sharding_attrs) + return cluster_func.emitError( + "output_sharding_configuration missing from cluster func"); + + if (output_sharding_attrs.size() != cluster_func.getNumResults()) + return cluster_func.emitError("incorrect number of output sharding"); + + for (const auto& output_sharding_and_index : + llvm::enumerate(output_sharding_attrs)) { + const auto& output_sharding = output_sharding_and_index.value(); + const int sharding_index = output_sharding_and_index.index(); + if (!mlir::isa(output_sharding)) + return cluster_func.emitError(llvm::formatv( + "non-string output sharding at index {0}", sharding_index)); + + xla::OpSharding sharding; + if (DecodeShardingAttribute( + mlir::cast(output_sharding).getValue().str(), + sharding) + .failed()) { + return cluster_func.emitError("incorrect sharding format for outputs"); + } + xla::hlo_sharding_util::ConvertV2ToV1Sharding(sharding); + + if (sharding.type() == xla::OpSharding::OTHER && + sharding.tile_assignment_devices_size() != num_cores_per_replica) + return cluster_func.emitError(llvm::formatv( + "incorrect sharding format for outputs. Number of " + "tiled outputs({0}) must match the number of logical " + "devices({1})", + sharding.tile_assignment_devices_size(), num_cores_per_replica)); + + if (sharding.type() == xla::OpSharding::MAXIMAL && + ((sharding.tile_assignment_devices(0) >= num_cores_per_replica) || + (sharding.tile_assignment_devices(0) < 0))) + return cluster_func.emitError(llvm::formatv( + "incorrect sharding format for outputs. Maximal " + "sharding should be assigned to device id in range " + "[0, {0}). Currently assigned to {1}", + num_cores_per_replica, sharding.tile_assignment_devices(0))); + + output_sharding_list->emplace_back(std::move(sharding)); + } + return mlir::success(); +} + +namespace { + +bool IsAssignedToLogicalDevice(const int core_id, + const xla::OpSharding& sharding) { + return sharding.type() == xla::OpSharding::MAXIMAL && + sharding.tile_assignment_devices(0) == core_id; +} + +// Returns the index of the return value of region in +// `tf_device.parallel_execute` that represents cluster func output at +// index |cluster_func_output_index|. Regions of parallel_execute may +// have different return values depending on output sharding configuration. +mlir::LogicalResult LookupClusterToCoreIndex( + const mlir::Location& location, + llvm::SmallVector, 4> cluster_to_core_index, + const int core_id, const int cluster_func_output_index, + int* core_output_index) { + *core_output_index = + cluster_to_core_index[core_id][cluster_func_output_index]; + if (*core_output_index == -1) { + mlir::emitError( + location, + llvm::formatv("Attempted to map cluster_func output index {0} to " + "program assigned to core {1}. The tensor at this output " + "index was not assigned or sharded to this core.", + cluster_func_output_index, core_id)); + return mlir::failure(); + } + return mlir::success(); +} + +// Collects tile sharded outputs from a tf_device.parallel_execute to remap from +// the TPU computation result. +mlir::LogicalResult GetTileShardedOutputsToMerge( + const mlir::Location& location, const int cluster_func_output_index, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + int cluster_idx, mlir::tf_device::ParallelExecuteOp new_parallel_execute, + llvm::SmallVector* outputs_to_merge) { + // Reorders outputs from TPUExecute op as defined by the output sharding + // configuration. + const xla::OpSharding& sharding = + output_sharding_config[cluster_func_output_index]; + outputs_to_merge->reserve(sharding.tile_assignment_devices_size()); + for (const auto& core_id_and_index : + llvm::enumerate(sharding.tile_assignment_devices())) { + auto core_id = core_id_and_index.value(); + auto tile_index = core_id_and_index.index(); + + int last_tile_dim_size = *sharding.tile_assignment_dimensions().rbegin(); + if (sharding.replicate_on_last_tile_dim() && + tile_index % last_tile_dim_size != 0) { + continue; + } + + int region_output_index; + auto status = LookupClusterToCoreIndex(location, cluster_to_core_index, + core_id, cluster_func_output_index, + ®ion_output_index); + if (failed(status)) return mlir::failure(); + const auto output_from_logical_device = + new_parallel_execute.GetRegionOutputs(cluster_idx + + core_id)[region_output_index]; + outputs_to_merge->emplace_back(output_from_logical_device); + } + + return mlir::success(); +} + +mlir::LogicalResult HandleTileShardedOutputsUsingXlaConcatOps( + const int cluster_func_output_index, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + const mlir::Location& location, mlir::Value cluster_func_output, + int cluster_idx, mlir::tf_device::ParallelExecuteOp new_parallel_execute, + mlir::OpBuilder& builder) { + // Inject concat ops after parallel_execute to merge outputs from + // concurrently executed computations. + builder.setInsertionPointAfter(new_parallel_execute); + const xla::OpSharding& sharding = + output_sharding_config[cluster_func_output_index]; + + const std::vector num_concats( + sharding.tile_assignment_dimensions().begin(), + sharding.replicate_on_last_tile_dim() + ? std::prev(sharding.tile_assignment_dimensions().end()) + : sharding.tile_assignment_dimensions().end()); + + const int rank = sharding.replicate_on_last_tile_dim() + ? sharding.tile_assignment_dimensions_size() - 1 + : sharding.tile_assignment_dimensions_size(); + std::vector paddings; + paddings.reserve(rank); + auto output_type = + mlir::cast(cluster_func_output.getType()); + if (output_type.hasRank()) { + auto shape = llvm::to_vector<4>(output_type.getShape()); + for (int dim = 0; dim < rank; ++dim) { + paddings.push_back(GetPadding(dim, + sharding.tile_assignment_dimensions(dim), + PartialTensorShape(shape))); + } + } else { + mlir::emitError( + location, "XLA concat/split ops are supported only for Ranked tensor."); + return mlir::failure(); + } + // Reorders outputs from TPUExecute op as defined by the output sharding + // configuration. + llvm::SmallVector outputs_to_merge; + auto status = GetTileShardedOutputsToMerge( + location, cluster_func_output_index, output_sharding_config, + cluster_to_core_index, cluster_idx, new_parallel_execute, + &outputs_to_merge); + if (failed(status)) return mlir::failure(); + + mlir::TF::XlaConcatNDOp concat_op = CreateXlaConcatNDOp( + location, outputs_to_merge, num_concats, paddings, builder); + cluster_func_output.replaceAllUsesWith(concat_op.getResult()); + return mlir::success(); +} + +// Merges outputs from TPU computation for tile-sharded outputs. +mlir::LogicalResult HandleTileShardedOutputsUsingTfConcatOps( + const int cluster_func_output_index, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + const mlir::Location& location, mlir::Value cluster_func_output, + int cluster_idx, mlir::tf_device::ParallelExecuteOp new_parallel_execute, + mlir::OpBuilder* builder) { + // Inject concat ops after parallel_execute to merge outputs from + // concurrently executed computations. + builder->setInsertionPointAfter(new_parallel_execute); + + // Reorders outputs from TPUExecute op as defined by the output sharding + // configuration. + llvm::SmallVector outputs_to_merge; + auto status = GetTileShardedOutputsToMerge( + location, cluster_func_output_index, output_sharding_config, + cluster_to_core_index, cluster_idx, new_parallel_execute, + &outputs_to_merge); + if (failed(status)) return mlir::failure(); + + // Creates a tree of Concat ops that merges outputs from multiple logical + // devices to a single replica output. + const xla::OpSharding& sharding = + output_sharding_config[cluster_func_output_index]; + + auto dimension_to_splits_map = + GetDimensionIndicesAndNumSplitsFromSharding(sharding); + if (!dimension_to_splits_map.ok()) { + LOG(ERROR) << dimension_to_splits_map.status(); + return mlir::failure(); + } + auto output_type = + mlir::cast(cluster_func_output.getType()); + PartialTensorShape shape; + bool output_shape_known = IsShapeKnown(output_type); + if (output_shape_known) { + shape = PartialTensorShape(output_type.getShape()); + } + bool has_paddings = false; + std::vector paddings; + for (auto it = dimension_to_splits_map->rbegin(); + it != dimension_to_splits_map->rend(); ++it) { + int concat_dimension = it->first; + int num_splits = it->second; + + llvm::SmallVector new_outputs; + new_outputs.reserve(num_splits); + for (int i = 0, end = outputs_to_merge.size(); i < end; + i = i + num_splits) { + int64_t padding; + if (output_shape_known) { + padding = GetPadding(concat_dimension, num_splits, shape); + } else { + padding = 0; + } + mlir::TF::ConcatOp concat_op = + CreateConcatOp(concat_dimension, location, padding, + llvm::ArrayRef{ + outputs_to_merge.begin() + i, + outputs_to_merge.begin() + i + num_splits}, + builder); + + paddings.push_back(padding); + has_paddings |= padding > 0; + new_outputs.emplace_back(concat_op.getResult()); + } + + std::swap(new_outputs, outputs_to_merge); + } + + assert(outputs_to_merge.size() == 1); + if (has_paddings) { + // Add slice op to remove paddings. + mlir::TF::SliceOp slice_op = + CreateSliceOp(builder, location, outputs_to_merge[0], shape); + cluster_func_output.replaceAllUsesWith(slice_op.getResult()); + } + cluster_func_output.replaceAllUsesWith(outputs_to_merge[0]); + return mlir::success(); +} + +mlir::LogicalResult ValidateAndGetTiledExecuteOutputShape( + const mlir::Location& location, + const mlir::TensorType cluster_func_output_type, + const xla::OpSharding& output_sharding, bool use_xla_nd_ops, + mlir::Type* tiled_logical_computation_type) { + const auto output_shape = cluster_func_output_type.getShape(); + auto new_output_shape = llvm::to_vector<4>(output_shape); + auto dimension_to_splits_map = + GetDimensionIndicesAndNumSplitsFromSharding(output_sharding); + if (!dimension_to_splits_map.ok()) { + LOG(ERROR) << dimension_to_splits_map.status(); + return mlir::failure(); + } + + for (const auto& dimension_and_output_splits : *dimension_to_splits_map) { + const auto dimension = dimension_and_output_splits.first; + const auto output_splits = dimension_and_output_splits.second; + + if (output_shape[dimension] == mlir::ShapedType::kDynamic) { + *tiled_logical_computation_type = cluster_func_output_type; + break; + } + if (output_shape[dimension] % output_splits == 0) { + new_output_shape[dimension] = output_shape[dimension] / output_splits; + } else { + // Input will be padded to be divisible by output_splits, thus add 1 to + // the output shape. + new_output_shape[dimension] = + (output_shape[dimension] / output_splits) + 1; + } + } + + *tiled_logical_computation_type = mlir::RankedTensorType::get( + new_output_shape, cluster_func_output_type.getElementType()); + + return mlir::success(); +} +} // namespace + +bool AreInputOutputShapesStaticallyKnownForSplitSharding( + llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func) { + bool sharded_input_output_shape_statically_known = true; + // Check input shapes + llvm::SmallVector cluster_func_inputs( + cluster_func.getOperands()); + auto sharding_attrs = + cluster_func.getOperation()->getAttrOfType( + kInputShardingAttr); + // If sharding attribute does not exist, then move on to check outputs. + // We only need to know the shapes in case of split sharding to use correct + // padding. + if (sharding_attrs) { + // Enumerate sharding configuration for each inputs and check shapes. + for (const auto& sharding_attr_and_index : + llvm::enumerate(sharding_attrs)) { + const auto& sharding_attr = sharding_attr_and_index.value(); + const auto input_index = sharding_attr_and_index.index(); + const auto& input_value = cluster_func_inputs[input_index]; + const auto input_type = + mlir::cast(input_value.getType()); + xla::OpSharding input_sharding; + if (DecodeShardingAttribute( + mlir::cast(sharding_attr).getValue().str(), + input_sharding) + .failed()) { + sharded_input_output_shape_statically_known = false; + } + // We only want to know about the shape for split sharding. + if (IsSplitSharding(input_sharding)) { + sharded_input_output_shape_statically_known &= IsShapeKnown(input_type); + } + } + } + // Check output shapes + for (const auto& result_and_index : + llvm::enumerate(cluster_func.getResults())) { + const auto output_index = result_and_index.index(); + const auto& output_sharding = output_sharding_config[output_index]; + const auto cluster_func_output_type = + mlir::cast(result_and_index.value().getType()); + // We only want to know about the shape for split sharding. + if (IsSplitSharding(output_sharding)) { + sharded_input_output_shape_statically_known &= + IsShapeKnown(cluster_func_output_type); + } + } + return sharded_input_output_shape_statically_known; +} + +mlir::LogicalResult GetOutputTypesForLogicalDeviceComputation( + const int core_id, llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func, + llvm::SmallVectorImpl* output_types, + llvm::SmallVectorImpl* cluster_to_core_index) { + return GetOutputTypesForLogicalDeviceComputation( + core_id, output_sharding_config, cluster_func, output_types, + /*use_xla_nd_ops=*/false, cluster_to_core_index); +} + +mlir::LogicalResult GetOutputTypesForLogicalDeviceComputation( + const int core_id, llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func, + llvm::SmallVectorImpl* output_types, bool use_xla_nd_ops, + llvm::SmallVectorImpl* cluster_to_core_index) { + output_types->reserve(cluster_func.getNumResults()); + + int core_index = 0; + for (const auto& result_and_index : + llvm::enumerate(cluster_func.getResults())) { + const auto output_index = result_and_index.index(); + const auto& output_sharding = output_sharding_config[output_index]; + const auto cluster_func_output_type = + mlir::cast(result_and_index.value().getType()); + + // If output shape of cluster func is statically known and output is tiled + // sharded, then the corresponding output shape of cluster func must be + // evenly divisible number of shardings, unless use of XLA split/concat ops + // is enabled. + if (IsSplitSharding(output_sharding)) { + mlir::Type tiled_logical_computation_type; + if (cluster_func_output_type.hasRank()) { + auto result = ValidateAndGetTiledExecuteOutputShape( + cluster_func.getLoc(), cluster_func_output_type, output_sharding, + use_xla_nd_ops, &tiled_logical_computation_type); + if (mlir::failed(result)) return mlir::failure(); + } else { + tiled_logical_computation_type = cluster_func_output_type; + } + cluster_to_core_index->emplace_back(core_index++); + output_types->emplace_back(tiled_logical_computation_type); + } else if (IsReplicatedSharding(output_sharding) || + IsAssignedToLogicalDevice(core_id, output_sharding)) { + cluster_to_core_index->emplace_back(core_index++); + output_types->emplace_back(cluster_func_output_type); + } else { + cluster_to_core_index->emplace_back(-1); + } + } + + return mlir::success(); +} + +mlir::LogicalResult RemapOutputsFromLogicalDevices( + const mlir::Location& location, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + int num_results_pre_cluster, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, int cluster_idx, + mlir::tf_device::ParallelExecuteOp new_parallel_execute, + mlir::OpBuilder* builder) { + return RemapOutputsFromLogicalDevices( + location, output_sharding_config, cluster_to_core_index, + num_results_pre_cluster, old_parallel_execute, cluster_idx, + new_parallel_execute, /*use_xla_nd_ops=*/false, builder); +} + +mlir::LogicalResult RemapOutputsFromLogicalDevices( + const mlir::Location& location, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + int num_results_pre_cluster, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, int cluster_idx, + mlir::tf_device::ParallelExecuteOp new_parallel_execute, + bool use_xla_nd_ops, mlir::OpBuilder* builder) { + for (auto [output_index, old_parallel_execute_output] : + llvm::enumerate(old_parallel_execute.getResults())) { + if (output_index < num_results_pre_cluster) { + // Replace the use of those results of old parallel_execute op from host + // with corresponding results of new parallel_execute op + for (auto& use : llvm::make_early_inc_range( + old_parallel_execute->getResult(output_index).getUses())) { + use.set(new_parallel_execute->getResult(output_index)); + } + continue; + } + + int tpu_cluster_output_index = output_index - num_results_pre_cluster; + const auto& output_sharding = + output_sharding_config[tpu_cluster_output_index]; + const auto output_sharding_type = output_sharding.type(); + + // If output is demultiplexed using the `tf.TPUPartitionedOutputV2` op, only + // replicated sharding is supported where i-th output of + // `tf.TPUPartitionedOutputV2` op maps to the output of i-th logical device. + // Also `tf.TPUPartitionedOutputV2` op must be a unique user of + // TPU Cluster (`tf_device.old_parallel_execute`) output. + mlir::TF::TPUPartitionedOutputV2Op partitioned_output; + for (auto user : old_parallel_execute_output.getUsers()) { + if (auto partitioned_output_user = + llvm::dyn_cast_or_null( + user)) { + partitioned_output = partitioned_output_user; + break; + } + } + if (partitioned_output) { + if (!old_parallel_execute_output.hasOneUse()) + return partitioned_output.emitOpError() + << "must be a unique user of TPU Cluster " + "(tf_device.old_parallel_execute) output " + << *old_parallel_execute_output.getOwner(); + if (UnsupportedPartitionedShardingType(output_sharding_type)) + return old_parallel_execute.emitOpError() + << "unsupported output sharding type " + << OpSharding_Type_Name(output_sharding_type) << " for " + << output_index << "-th output"; + + if (output_sharding_type == xla::OpSharding::REPLICATED) { + for (const auto& index_and_output : + llvm::enumerate(partitioned_output.getOutput())) { + auto idx = (cluster_idx + index_and_output.index()) % + new_parallel_execute->getNumRegions(); + const auto output_from_logical_device = + new_parallel_execute.GetRegionOutputs( + idx)[tpu_cluster_output_index]; + index_and_output.value().replaceAllUsesWith( + output_from_logical_device); + } + } else { + assert(output_sharding_type == xla::OpSharding::OTHER); + llvm::SmallVector tile_sharded_outputs; + if (failed(GetTileShardedOutputsToMerge( + location, tpu_cluster_output_index, output_sharding_config, + cluster_to_core_index, cluster_idx, new_parallel_execute, + &tile_sharded_outputs))) + return mlir::failure(); + for (auto result : + llvm::zip(partitioned_output.getOutput(), tile_sharded_outputs)) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + } + continue; + } + if (IsSplitSharding(output_sharding)) { + if (use_xla_nd_ops) { + auto result = HandleTileShardedOutputsUsingXlaConcatOps( + tpu_cluster_output_index, output_sharding_config, + cluster_to_core_index, location, old_parallel_execute_output, + cluster_idx, new_parallel_execute, *builder); + if (mlir::failed(result)) return mlir::failure(); + } else { + auto result = HandleTileShardedOutputsUsingTfConcatOps( + tpu_cluster_output_index, output_sharding_config, + cluster_to_core_index, location, old_parallel_execute_output, + cluster_idx, new_parallel_execute, builder); + if (failed(result)) return mlir::failure(); + } + continue; + } + + int logical_device_id = 0; + if (output_sharding_type == xla::OpSharding::MAXIMAL) + logical_device_id = output_sharding.tile_assignment_devices(0); + + // For maximal sharding configuration, correctly remap outputs from + // parallel_execute region to users of the cluster func. + int region_output_index; + if (failed(LookupClusterToCoreIndex( + location, cluster_to_core_index, logical_device_id, + tpu_cluster_output_index, ®ion_output_index))) + return mlir::failure(); + + const auto output_from_logical_device = + new_parallel_execute.GetRegionOutputs( + cluster_idx + logical_device_id)[region_output_index]; + old_parallel_execute_output.replaceAllUsesWith(output_from_logical_device); + } + return mlir::success(); +} + +llvm::SmallVector, 4> GetMetadataArgumentMapping( + const tpu::TPUCompileMetadataProto& metadata) { + llvm::SmallVector, 4> input_mappings( + metadata.num_cores_per_replica(), llvm::SmallVector()); + + if (metadata.num_cores_per_replica() == 1) { + input_mappings.front().resize(metadata.args_size()); + std::iota(input_mappings.front().begin(), input_mappings.front().end(), 0); + return input_mappings; + } + + for (const auto& arg_and_idx : llvm::enumerate(metadata.args())) { + const auto& sharding = arg_and_idx.value().sharding(); + const int64_t idx = arg_and_idx.index(); + + const auto sharding_type = sharding.type(); + if (sharding_type == xla::OpSharding::OTHER) { + for (const auto& device : sharding.tile_assignment_devices()) { + CHECK(device >= 0 && device < input_mappings.size()); + input_mappings[device].push_back(idx); + } + } else if (sharding_type == xla::OpSharding::REPLICATED) { + for (auto& input : input_mappings) input.push_back(idx); + } else { + assert(sharding_type == xla::OpSharding::MAXIMAL); + CHECK(sharding.tile_assignment_devices(0) >= 0 && + sharding.tile_assignment_devices(0) < input_mappings.size()); + input_mappings[sharding.tile_assignment_devices(0)].push_back(idx); + } + } + + return input_mappings; +} + +namespace { +mlir::LogicalResult VerifyShardingEquivalent(mlir::Attribute sharding_attr1, + mlir::Attribute sharding_attr2) { + xla::OpSharding sharding_proto1; + if (tensorflow::DecodeShardingAttribute(sharding_attr1, sharding_proto1) + .failed()) { + return mlir::failure(); + } + xla::OpSharding sharding_proto2; + if (tensorflow::DecodeShardingAttribute(sharding_attr2, sharding_proto2) + .failed()) { + return mlir::failure(); + } + + return tensorflow::VerifyShardingEquivalent(sharding_proto1, sharding_proto2); +} +} // namespace + +mlir::LogicalResult VerifyShardingEquivalent( + const xla::OpSharding& sharding_proto1, + const xla::OpSharding& sharding_proto2) { + absl::StatusOr sharding1 = + xla::HloSharding::FromProto(sharding_proto1); + if (!sharding1.ok()) { + return mlir::failure(); + } + absl::StatusOr sharding2 = + xla::HloSharding::FromProto(sharding_proto2); + if (!sharding2.ok()) { + return mlir::failure(); + } + + if (*sharding1 == *sharding2) { + return mlir::success(); + } + + return mlir::failure(); +} + +absl::StatusOr GetXlaShardingAttrFromShardingOp( + mlir::TF::XlaShardingOp sharding) { + if (!sharding.get_XlaShardingV2Attr()) { + return sharding.get_XlaShardingAttr(); + } + + if (sharding.get_XlaShardingAttr() && + VerifyShardingEquivalent(sharding.get_XlaShardingV2Attr(), + sharding.get_XlaShardingAttr()) + .failed()) { + return absl::InvalidArgumentError( + absl::StrCat("_XlaShardingV2 is not equivalent to _XlaSharding: ", + sharding.get_XlaShardingV2().value().str(), " vs ", + sharding.get_XlaSharding().value().str())); + } + return sharding.get_XlaShardingV2Attr(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h new file mode 100644 index 0000000000000000000000000000000000000000..ae98303302238eb93e3897fa68f36482176fcb15 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h @@ -0,0 +1,187 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_SHARDING_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_SHARDING_UTIL_H_ + +#include + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { + +inline constexpr llvm::StringRef kInputShardingAttr = + "input_sharding_configuration"; +inline constexpr llvm::StringRef kOutputShardingAttr = + "output_sharding_configuration"; + +inline constexpr llvm::StringRef kICIWeightDistributionMlirBridgeMarker = + "_ici_weight_distribution_mlir_bridge_marker"; + +// Parses the sharding string. This sharding string can be binary (serialized) +// or human readable. +mlir::LogicalResult DecodeShardingAttribute(const std::string& shard_str, + xla::OpSharding& sharding, + bool report_error = true); + +// Encodes the sharding in human readable form. +mlir::LogicalResult DecodeShardingAttribute(mlir::Attribute shard_attr, + xla::OpSharding& sharding, + bool report_error = true); + +// Parses the sharding attr. This sharding attr can be binary (serialized) +// or human readable. +void EncodeSharding(mlir::Operation* op, llvm::StringRef shard_str); + +// Parses "input_sharding_configuration" attribute and returns a list where i-th +// element is a list of mlir::Value's which represent inputs for the TPU +// computation corresponding to i-th logical device. If the attribute does not +// exist, the all inputs are placed on logical core 0. +mlir::LogicalResult ExtractInputsForLogicalDevices( + int num_cores_per_replica, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::OpBuilder* builder, + llvm::SmallVectorImpl>* input_list); + +// Same as above, except creates tf.XlaSplitND Op for split sharding if +// use_xla_nd_ops is true, otherwise creates tf.Split op. +mlir::LogicalResult ExtractInputsForLogicalDevices( + int num_cores_per_replica, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::OpBuilder* builder, bool use_xla_nd_ops, + llvm::SmallVectorImpl>* input_list); + +// Extracts a list of OpSharding that represent output sharding configuration of +// `tf_device.cluster`. +mlir::LogicalResult ParseAndValidateOutputSharding( + int num_cores_per_replica, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::SmallVector* output_sharding_list); + +// Retrieves output types for TPUExecute op representing execution for provided +// logical device id. TPUExecute op for different logical device may have +// different outputs depending on the output sharding configuration. +mlir::LogicalResult GetOutputTypesForLogicalDeviceComputation( + int core_id, llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func, + llvm::SmallVectorImpl* output_types, + llvm::SmallVectorImpl* cluster_to_core_index); + +// Same as above, except creates tf.XlaSplitND Op for split sharding if +// use_xla_nd_ops is true, otherwise creates tf.Split op. +mlir::LogicalResult GetOutputTypesForLogicalDeviceComputation( + int core_id, llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func, + llvm::SmallVectorImpl* output_types, bool use_xla_nd_ops, + llvm::SmallVectorImpl* cluster_to_core_index); + +// Remaps outputs of `new_parallel_execute` op that represent concurrent +// execution of the `tf_device.cluster_func` at index `cluster_idx` of +// `old_parallel_execute` with its users. +// `num_results_pre_cluster` represent the # of outputs of +// `new_parallel_execute` which are from ops before `tf_device.cluster_func` op. +mlir::LogicalResult RemapOutputsFromLogicalDevices( + const mlir::Location& location, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + int num_results_pre_cluster, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, int cluster_idx, + mlir::tf_device::ParallelExecuteOp new_parallel_execute, + mlir::OpBuilder* builder); + +// Same as above, except creates tf.XlaConcatNd Op for split sharding if +// use_xla_nd_ops is true, otherwise creates tf.Concat op. +mlir::LogicalResult RemapOutputsFromLogicalDevices( + const mlir::Location& location, + llvm::ArrayRef output_sharding_config, + llvm::SmallVector, 4> cluster_to_core_index, + int num_results_pre_cluster, + mlir::tf_device::ParallelExecuteOp old_parallel_execute, int cluster_idx, + mlir::tf_device::ParallelExecuteOp new_parallel_execute, + bool use_xla_nd_ops, mlir::OpBuilder* builder); + +// Determines each logical core argument to metadata argument index mapping, +// based on sharding. The return value is indexed first by logical core then by +// argument index. +llvm::SmallVector, 4> GetMetadataArgumentMapping( + const tpu::TPUCompileMetadataProto& metadata); + +// Gets the proper tensor dimension from XLA OpSharding. +// "replicate_on_last_tile_dim" and "last_tile_dims" should be deducted from the +// real Tensor dimensions when tiled. +// For example: +// f32[8,512](sharding={devices=[1,1,2]0,1 last_tile_dims={REPLICATED}) +// also means a replicated tensor over all devices. +// +// See xla_data.proto for detailed explanations on the fields. +int GetDimsFromXLAShardingTiled(const xla::OpSharding& xla_sharding); + +// A sharding with OTHER type may be REPLICATED if: +// 'replicate_on_last_tile_dim' is true OR +// 'last_tile_dims' is not empty +// AND +// other than replicated last tile dims, all other dims are not sharded. +bool IsOtherReplicatedSharding(const xla::OpSharding& xla_sharding); + +// Returns whether the sharding is split sharding. i.e. A sharding with OTHER +// type but not replicated. +bool IsSplitSharding(const xla::OpSharding& sharding); + +// Returns whether the sharding is replicated. It includes sharding with +// REPLICATED type and replicated OTHER type. +bool IsReplicatedSharding(const xla::OpSharding& sharding); + +// Returns whether the shape of inputs and outputs is statically known when +// split sharding is done on inputs or outputs. +bool AreInputOutputShapesStaticallyKnownForSplitSharding( + llvm::ArrayRef output_sharding_config, + mlir::tf_device::ClusterFuncOp cluster_func); + +// Returns a map of dimension indices and number of splits for tiled sharding. +absl::StatusOr> GetDimensionIndicesAndNumSplitsFromSharding( + const xla::OpSharding& sharding); + +// Verifies that the two sharding attributes are equivalent, by converting them +// to OpSharding and then HloSharding and performing a comparison. Returns +// failure if the attributes are not equivalent or if there is any problem +// in converting the attributes to OpSharding or HloSharding. +mlir::LogicalResult VerifyShardingEquivalent( + const xla::OpSharding& sharding_proto1, + const xla::OpSharding& sharding_proto2); + +// Returns _XlaShardingV2 if it exists or _XlaSharding otherwise. When both +// _XlaSharding and _XlaShardingV2 exist, verifies that they are equivalent +// and returns an error status if they aren't equivalent. +absl::StatusOr GetXlaShardingAttrFromShardingOp( + mlir::TF::XlaShardingOp sharding); +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_UTILS_XLA_SHARDING_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..115859a3c9bbe14a9ff8e9b80672bddd6c9f5f15 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util_test.cc @@ -0,0 +1,224 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" + +#include +#include "absl/status/statusor.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tsl/platform/statusor.h" + +namespace { + +absl::StatusOr> GetMlirModuleFromString( + llvm::StringRef string, mlir::MLIRContext* context) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + mlir::OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +TEST(XLAShardingUtilTest, TestShapesCheckForSplitSharding) { + // Module with sharding with static shapes. + static const char* const module_str = + R"( + module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + func.func @parallel_execute_with_tiled_input(%arg0: tensor<128x9xf32>, %arg1: tensor<128x9xf32>, %arg2: tensor<128x10xi32>, %arg3: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x9xf32>, [%arg2, %arg3] as %ri_2: tensor<128x10xi32>) {n = 2 : i32} { + %1 = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %identity = "tf.Identity"(%ri_1) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor<128x9xf32>) -> tensor<128x9xf32> + tf_device.return %identity : tensor<128x9xf32> + }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor<128x9xf32> + %2, %3 = "tf_device.cluster_func"(%1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02J\01\02R\01\00", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x9xf32>, tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + tf_device.return %2, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.return %0#0, %1#0 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.func @tpu0_func(%arg0: tensor<128x9xf32>, %arg1: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x9xf32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<128x10xi32>, tensor<128x10xi32>) -> (tensor<128x10xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<10x5xi1>) -> tensor<10x5xi1> + func.return %4, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } +} +)"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + llvm::SmallVector cluster_func_ops; + module->walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + cluster_func_ops.push_back(cluster_func); + }); + EXPECT_EQ(cluster_func_ops.size(), 1); + auto cluster_func_op = cluster_func_ops[0]; + + llvm::SmallVector output_shardings; + auto result = tensorflow::ParseAndValidateOutputSharding(2, cluster_func_op, + &output_shardings); + ASSERT_TRUE(succeeded(result)); + ASSERT_TRUE(tensorflow::AreInputOutputShapesStaticallyKnownForSplitSharding( + output_shardings, cluster_func_op)); +} + +TEST(XLAShardingUtilTest, TestShapesCheckForSplitShardingWithUnknownShapes) { + // Module with sharding with unknown shapes + static const char* const module_str = + R"( + module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + func.func @parallel_execute_with_tiled_input(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<128x10xi32>, %arg3: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<*xf32>, [%arg2, %arg3] as %ri_2: tensor<128x10xi32>) {n = 2 : i32} { + %1 = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %identity = "tf.Identity"(%ri_1) {_ici_weight_distribution_mlir_bridge_marker = true} : (tensor<*xf32>) -> tensor<*xf32> + tf_device.return %identity : tensor<*xf32> + }) {_ici_weight_distribution_mlir_bridge_marker = true} : () -> tensor<*xf32> + %2, %3 = "tf_device.cluster_func"(%1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\1A\02\01\02J\01\02R\01\00", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<*xf32>, tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + tf_device.return %2, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.return %0#0, %1#0 : tensor<128x10xi32>, tensor<10x5xi1> + } + func.func @tpu0_func(%arg0: tensor<*xf32>, %arg1: tensor<128x10xi32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<*xf32>) -> (tensor<128x10xi32>, tensor<10x5xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<128x10xi32>, tensor<128x10xi32>) -> (tensor<128x10xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<10x5xi1>) -> tensor<10x5xi1> + func.return %4, %3 : tensor<128x10xi32>, tensor<10x5xi1> + } +} +)"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + llvm::SmallVector cluster_func_ops; + module->walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + cluster_func_ops.push_back(cluster_func); + }); + EXPECT_EQ(cluster_func_ops.size(), 1); + auto cluster_func_op = cluster_func_ops[0]; + + llvm::SmallVector output_shardings; + auto result = tensorflow::ParseAndValidateOutputSharding(2, cluster_func_op, + &output_shardings); + ASSERT_TRUE(succeeded(result)); + ASSERT_FALSE(tensorflow::AreInputOutputShapesStaticallyKnownForSplitSharding( + output_shardings, cluster_func_op)); +} + +TEST(XLAShardingUtilTest, NotDivisibleShardingSplitOpTest) { + static const char* const module_str = + R"( + module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:1/device:CPU:0", "/job:localhost/replica:0/task:1/device:TPU:0", "/job:localhost/replica:0/task:1/device:TPU:1", "/job:localhost/replica:0/task:1/device:TPU_SYSTEM:0"]} { + func.func @uneven_input_sharding_disallowed(%arg0: tensor<128x10xf32>, %arg1: tensor<128x10xf32>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %0:2, %1:2 = tf_device.replicate([%arg0, %arg1] as %ri_1: tensor<128x10xf32>, [%arg2, %arg3] as %ri_2: tensor<*xi32>) {n = 2 : i32} { + %1, %2 = "tf_device.cluster_func"(%ri_1, %ri_2) {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @tpu0_func, num_cores_per_replica = 2, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "\0A\04\01\02\01\02\10\02\18\02\22\10\00\00\00\00\00\00\00\01\00\01\00\00\00\01\00\01", device_assignment = [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0], input_sharding_configuration = ["\08\03\12\12\10\0b\1a\02\01\04\2a\06\0a\02\01\00\20\01\32\02\00\00\1a\02\01\04\22\04\00\01\02\03", "\08\01\1A\01\01\22\01\01"], output_sharding_configuration = ["\08\01\1A\01\01\22\01\00", ""], use_spmd_for_xla_partitioning = false} : (tensor<128x10xf32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) + tf_device.return %1, %2 : tensor<*xi32>, tensor<*xi1> + } + func.return %0#0, %1#0 : tensor<*xi32>, tensor<*xi1> + } + func.func @tpu0_func(%arg0: tensor<128x10xf32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi1>) { + %1, %2 = "tf.A"(%arg0) : (tensor<128x10xf32>) -> (tensor<*xi32>, tensor<*xi1>) + %4 = "tf.B"(%1, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>) + %3 = "tf.XlaSharding"(%2) { _XlaSharding = "", sharding = "" } : (tensor<*xi1>) -> tensor<*xi1> + func.return %4, %3 : tensor<*xi32>, tensor<*xi1> + } +} +)"; + mlir::MLIRContext context; + TF_ASSERT_OK_AND_ASSIGN(mlir::OwningOpRef module, + GetMlirModuleFromString(module_str, &context)); + llvm::SmallVector cluster_func_ops; + module->walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + cluster_func_ops.push_back(cluster_func); + }); + EXPECT_EQ(cluster_func_ops.size(), 1); + auto& cluster_func_op = cluster_func_ops[0]; + int num_cores_per_replica = 4; + mlir::OpBuilder builder(&context); + bool use_xla_nd_ops = true; + + llvm::SmallVector, 4> input_list; + auto result = tensorflow::ExtractInputsForLogicalDevices( + num_cores_per_replica, cluster_func_op, &builder, use_xla_nd_ops, + &input_list); + ASSERT_TRUE(succeeded(result)); + + ASSERT_EQ(input_list.size(), num_cores_per_replica); + ASSERT_GT(input_list.front().size(), 0); + // Get the XLASplit op generated. + auto* op = input_list.front().front().getDefiningOp(); + ASSERT_TRUE(mlir::isa(op)); + // Erase newly created Split op to avoid error during block deletion: + // use_empty() && "Cannot destroy a value that still has uses!" + // + // This is needed only for the unit test. + // + // More explanation: + // We create the XLASplit op in ExtractInputsForLogicalDevices function and + // use the getResults for the op are added to input list. + // Somehow the values from XLASplitND op are not getting released correctly in + // the destructor for the block at the end of the test function. + // This results in "use_empty() && 'Cannot destroy a value that still has + // uses!'" error when the builder and block ops arr deleted when they go out + // of scope. Thus, added Op->destroy for the XLA split to remove it from the + // MLIR block so the above assertion does not occur. We need this only in the + // test code, it is not a problem in the actual pass since tpu_rewrite_pass + // will appropriately add the values to the block. + op->destroy(); + + input_list.clear(); + + // Expect error when use_xla_nd_ops is false. + result = tensorflow::ExtractInputsForLogicalDevices( + num_cores_per_replica, cluster_func_op, &builder, false, &input_list); + ASSERT_TRUE(succeeded(result)); + auto* split_op = input_list.front().front().getDefiningOp(); + ASSERT_TRUE(mlir::isa(split_op)); + + llvm::SmallVector split_inputs(split_op->getOperands()); + // Constant op for the split dimension + auto* const_op = split_inputs[0].getDefiningOp(); + ASSERT_TRUE(mlir::isa(const_op)); + // Pad op for the padding value to make it divisible by num_splits. + auto* pad_op = split_inputs[1].getDefiningOp(); + ASSERT_TRUE(mlir::isa(pad_op)); + llvm::SmallVector pad_inputs(pad_op->getOperands()); + auto* const_pad_value = pad_inputs[1].getDefiningOp(); + ASSERT_TRUE(mlir::isa(const_pad_value)); + // Destroy the ops to avoid error during block deletion (Same as above): + // use_empty() && "Cannot destroy a value that still has uses!" + split_op->destroy(); + const_op->destroy(); + pad_op->destroy(); + const_pad_value->destroy(); +} +} // namespace diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..e1553e817d139a58a9225ac78561db3a61b5d908 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/BUILD @@ -0,0 +1,105 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") +load("//tensorflow:tensorflow.default.bzl", "filegroup", "get_compatible_with_portable") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") +load("//tensorflow/compiler/mlir/quantization/stablehlo:internal_visibility_allowlist.bzl", "internal_visibility_allowlist") + +package_group( + name = "internal_visibility_allowlist_package", + packages = [ + "//tensorflow/compiler/mlir/lite/...", + "//tensorflow/compiler/mlir/quantization/...", + "//tensorflow/compiler/mlir/tensorflow_to_stablehlo/...", + "//tensorflow/compiler/mlir/tf2xla/transforms/...", + "//tensorflow/lite/...", + ] + internal_visibility_allowlist(), +) + +package( + # copybara:uncomment default_applicable_licenses = ["@stablehlo//:license"], + default_visibility = [ + ":internal_visibility_allowlist_package", + "//tensorflow:__pkg__", + ], + licenses = ["notice"], +) + +cc_library( + name = "tf_to_stablehlo", + srcs = [ + "tf_to_stablehlo.cc", + ], + hdrs = [ + "tf_to_stablehlo.h", + ], + compatible_with = get_compatible_with_portable(), + deps = [ + "//tensorflow/compiler/mlir/quantization/stablehlo/cc:saved_model_import", + "//tensorflow/compiler/mlir/quantization/tensorflow:quantize_preprocess", + "//tensorflow/compiler/mlir/tensorflow/transforms:shape_inference_pass", + "//tensorflow/core:core_cpu_base", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], + alwayslink = True, +) + +tf_cc_binary( + name = "tf-to-stablehlo-translate", + srcs = [ + "tf_to_stablehlo_translate.cc", + ], + visibility = [":internal_visibility_allowlist_package"], + deps = [ + ":tf_to_stablehlo", + "//tensorflow/compiler/mlir:init_mlir", + "//tensorflow/compiler/mlir/tensorflow", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + default_tags = [ + "no_oss", + "no_pip", + ], + driver = "//tensorflow/compiler/mlir/tensorflow_to_stablehlo:run_lit.sh", + size_override = { + }, + tags_override = { + }, + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + ":tf-to-stablehlo-translate", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/README.md b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..696044cccdda0701eb2f823c3c9c80f013194556 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/README.md @@ -0,0 +1,123 @@ +# Tensorflow SavedModel to StableHLO (tf-to-stablehlo-translate) + +Converts TensorFlow models (SavedModel or MLIR module) to StableHLO MLIR +modules, preserving model structure and signatures. It enables seamless +integration of TensorFlow models into MLIR-based compiler frameworks for further +optimization and deployment. + +## C++ APIs + +```bash +tf-to-stablehlo-translate \ + --input-path=/path/to/model \ + [--exported-model-signatures=signature1,signature2] \ + [--tag-names=tag1,tag2] \ + [--input-arg-shapes-str=arg-name:shape,...] \ + [--e] \ + [--output-filename=/path/to/output.mlir] +``` + +* `--input-path`: The path to the input TensorFlow SavedModel or MLIR module + with .mlir extension. +* `--exported-model-signatures`: Comma-separated list of exported model + signatures to convert. Ignored for MLIR input. +* `--tags`: Comma-separated list of tags for loading SavedModel. Ignored for + MLIR input. +* `--input-arg-shapes`: A string representation of input argument shapes for + 'main' entry-point, separating tensors with ':', dimension with ',', and + using '?' for unknown sizes. For example, `input-arg-shapes=1,2::1,?` + expresses argument shapes `[1,2]`, `[]` and `[1,?]`. +* `--e`: Elide large elements attrs while dumping the output StableHLO. +* `--output_filename`: Path to the output file where the textual StableHLO MLIR + module will be written (default: stdout). + + +### Examples + +* To convert [microsoft/resnet-50](https://huggingface.co/microsoft/resnet-50) +model to StableHLO with static input shape `4x3x224x224` for input argument with +type `tensor`. + +```bash +tf-to-stablehlo-translate --input-arg-shapes=4,3,224,224 +``` + +* To convert +[google-bert/bert-large-uncased](https://huggingface.co/google-bert/bert-large-uncased) +to StableHLO with static input shapes `1x12`, `1x12`, and `1x12` for input +arguments with types `tensor, tensor, tensor`. + +```bash +tf-to-stablehlo-translate --input-arg-shapes=1,12:1,12:1,12 +``` + +### Dependencies + +* TensorFlow +* MLIR +* Abseil (absl) + +## Python APIs + + +### `savedmodel_to_stablehlo` + +Converts a TensorFlow SavedModel into StableHLO bytecode. + +```Python +from tensorflow.compiler.mlir.tensorflow_to_stablehlo.python import pywrap_tensorflow_to_stablehlo as tf2shlo + +stablehlo_bytes = tf2shlo.savedmodel_to_stablehlo( + input_path="/path/to/your/savedmodel", + exported_model_signatures=["serving_default"], + tag_names=["serve"], + input_arg_shapes_str="1,28,28,3::32" +) + +``` + +#### Arguments: + +* `input_path` (required): Path to your SavedModel directory. +* `exported_model_signatures` (optional): List of signature names to convert. + Defaults to ["serving_default"]. +* `tag_names` (optional): List of tags associated with the SavedModel. Defaults + to ["serve"]. +* `input_arg_shapes_str` (optional): A string representation of input argument + shapes for 'main' entry-point, separating + tensors with ':', dimension with ',', and + using '?' for unknown sizes. For example, + `input-arg-shapes=1,2::1,?` expresses + argument shapes `[1,2], [] and [1,?]`. + +#### Error Handling + +An exception will be raised with details about the error. + +### `tensorflow_module_to_stablehlo` + +Converts a TensorFlow MLIR module string into StableHLO bytecode. + +```Python +from tensorflow.compiler.mlir.tensorflow_to_stablehlo.python import pywrap_tensorflow_to_stablehlo as tf2shlo + +stablehlo_bytes = tf2shlo.tensorflow_module_to_stablehlo( + module_op_str="your_tensorflow_mlir_module_string", + input_arg_shapes_str="1,28,28,3::32" +) +``` + +#### Arguments: + +* `module_op_str` (required): String containing the TensorFlow MLIR module. +* `input_arg_shapes_str` (optional): A string representation of input argument + shapes for 'main' entry-point, separating + tensors with ':', dimension with ',', and + using '?' for unknown sizes. For example, + `input-arg-shapes=1,2::1,?` expresses + argument shapes `[1,2], [] and [1,?]`. + +#### Error Handling + +Return `py::none()` (equivalent to Python's `None`) if there's an error. An +exception will be raised with details about the error. diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..01cfc34f30d6e7b782c8db84ecb7729f3c2e7aca --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/BUILD @@ -0,0 +1,113 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@xla//xla/tsl/platform:build_config_root.bzl", "if_pywrap") +load( + "//tensorflow:tensorflow.default.bzl", + "get_compatible_with_portable", + "tf_py_strict_test", + "tf_python_pybind_extension", +) +load("//tensorflow/compiler/mlir/quantization/stablehlo:internal_visibility_allowlist.bzl", "internal_visibility_allowlist") + +package_group( + name = "internal_visibility_allowlist_package", + packages = [ + "//tensorflow/compiler/mlir/lite/...", + "//tensorflow/compiler/mlir/quantization/...", + "//tensorflow/compiler/mlir/tf2xla/transforms/...", + "//tensorflow/lite/...", + ] + internal_visibility_allowlist(), +) + +package( + # copybara:uncomment default_applicable_licenses = ["@stablehlo//:license"], + default_visibility = [ + ":internal_visibility_allowlist_package", + "//tensorflow:__pkg__", + "//tensorflow/python:__pkg__", + ], + licenses = ["notice"], +) + +# copybara:uncomment_begin(google-only) +# tf_py_strict_test( +# name = "tensorflow_to_stablehlo_test", +# testonly = 1, +# srcs = ["integration_test/tensorflow_to_stablehlo_test.py"], +# deps = [ +# ":pywrap_tensorflow_to_stablehlo", +# "//testing/pymocks:matchers", +# "//third_party/py/mlir", +# "//third_party/py/mlir:ir", +# "//third_party/py/mlir:stablehlo_dialect", +# "//third_party/py/mlir/_mlir_libs:_mlirRegisterEverything", +# "//tensorflow:tensorflow_py", +# "//tensorflow/compiler/mlir/stablehlo", +# "//tensorflow/python/framework:test_lib", +# "//tensorflow/python/platform:client_testlib", +# "//tensorflow/python/types:core", +# ], +# ) +# copybara:uncomment_end + +# This is a header-only target. The purpose of `pywrap_tensorflow_to_stablehlo_lib_*` targets is to expose only +# the symbols that are required by `pywrap_tensorflow_to_stablehlo` that translates them to python functions. +# The only intended use case of this library is by `pywrap_tensorflow_to_stablehlo`. Not letting +# `pywrap_tensorflow_to_stablehlo` directly depend on sub-libraries like `static_range_srq` and instead haiving +# a consolidated impl library `pywrap_tensorflow_to_stablehlo_lib_impl` allows the maintainers to avoid +# declaring multiple impl libraries to `libtensorflow_cc` and `lib_pywrap_tensorflow_internal`, +# which is required to avoid ODR violations. +cc_library( + name = "pywrap_tensorflow_to_stablehlo_lib_header_only", + srcs = [], + hdrs = ["pywrap_tensorflow_to_stablehlo_lib.h"], + compatible_with = get_compatible_with_portable(), + visibility = ["//visibility:private"], # ONLY for `pywrap_tensorflow_to_stablehlo`. + deps = [ + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + ], +) + +# See the comments for `pywrap_tensorflow_to_stablehlo_lib_header_only`. +cc_library( + name = "pywrap_tensorflow_to_stablehlo_lib_impl", + srcs = ["pywrap_tensorflow_to_stablehlo_lib.cc"], + hdrs = ["pywrap_tensorflow_to_stablehlo_lib.h"], + compatible_with = get_compatible_with_portable(), + visibility = [ + "//tensorflow:__pkg__", # For libtensorflow_cc.so. + "//tensorflow/python:__pkg__", # For lib_pywrap_tensorflow_internal.so. + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow_to_stablehlo:tf_to_stablehlo", + "//tensorflow/core:lib", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeWriter", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Support", + "@xla//third_party/python_runtime:headers", + ], +) + +tf_python_pybind_extension( + name = "pywrap_tensorflow_to_stablehlo", + srcs = ["pywrap_tensorflow_to_stablehlo.cc"], + pytype_srcs = ["pywrap_tensorflow_to_stablehlo.pyi"], + # Each dependency MUST be either header-only or exclusive. + deps = [ + "@com_google_absl//absl/strings:string_view", + "@pybind11", + "@pybind11_abseil//pybind11_abseil:absl_casters", + "@pybind11_abseil//pybind11_abseil:status_casters", + "@xla//third_party/python_runtime:headers", + ] + if_pywrap( + if_false = [":pywrap_tensorflow_to_stablehlo_lib_header_only"], + if_true = [":pywrap_tensorflow_to_stablehlo_lib_impl"], + ), +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/integration_test/tensorflow_to_stablehlo_test.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/integration_test/tensorflow_to_stablehlo_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9d7c11f003cfa470e3b5ceafd7e45ffbf8b707c1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/integration_test/tensorflow_to_stablehlo_test.py @@ -0,0 +1,78 @@ +# Copyright 2024 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import tempfile +from mlir import ir +from mlir.dialects import stablehlo +import tensorflow as tf +from tensorflow.compiler.mlir.tensorflow_to_stablehlo.python import pywrap_tensorflow_to_stablehlo as tensorflow_to_stablehlo +from tensorflow.python.platform import test + + +def build_savedmodel(tempdir) -> str: + + class AddOneModel(tf.keras.Model): + + def call(self, x): + return x + 1 + + model = AddOneModel() + + x_train = tf.constant([1, 2, 3, 4, 5], dtype=tf.float32) + y_train = tf.constant([2, 3, 4, 5, 6], dtype=tf.float32) + + model.compile(optimizer='sgd', loss='mse') + model.fit(x_train, y_train, epochs=1) + + path = tempdir + '/add_one_model' + model.save(path) + return path + + +class TensorflowToStableHLOTest(test.TestCase): + + def test_saved_model_to_stablehlo(self): + with tempfile.TemporaryDirectory() as tempdir: + path = build_savedmodel(tempdir) + module_bytecode = tensorflow_to_stablehlo.savedmodel_to_stablehlo( + input_path=path, input_arg_shapes_str='4' + ) + with ir.Context() as ctx: + stablehlo.register_dialect(ctx) + module = ir.Module.parse(module_bytecode) + self.assertIn('stablehlo.add %arg0, %cst : tensor<4xf32>', str(module)) + + def test_tf_mlir_to_stablehlo(self): + assembly = """ + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0 : tensor) -> tensor { + %cst = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %0 = "tf.Add"(%arg0, %cst): (tensor, tensor) -> tensor + func.return %0 : tensor + } + } + """ + module_bytecode = tensorflow_to_stablehlo.tensorflow_module_to_stablehlo( + module=assembly, + input_arg_shapes_str='4', + ) + with ir.Context() as ctx: + stablehlo.register_dialect(ctx) + module = ir.Module.parse(module_bytecode) + self.assertIn('stablehlo.add %arg0, %cst : tensor<4xf32>', str(module)) + + +if __name__ == '__main__': + test.main() diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.cc new file mode 100644 index 0000000000000000000000000000000000000000..9e8234486eb8884ae8f20d2396d28eedf9edeb4d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.cc @@ -0,0 +1,101 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "pybind11/pybind11.h" // from @pybind11 +#include "pybind11/pytypes.h" // from @pybind11 +#include "pybind11_abseil/absl_casters.h" // from @pybind11_abseil // IWYU pragma: keep +#include "pybind11_abseil/status_casters.h" // from @pybind11_abseil // IWYU pragma: keep +#include "tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.h" + +namespace py = pybind11; + +namespace { + +using mlir::tensorflow_to_stablehlo::pywrap::PywrapSavedModelToStablehlo; +using mlir::tensorflow_to_stablehlo::pywrap::PywrapTfModuleToStablehlo; + +} // namespace + +PYBIND11_MODULE(pywrap_tensorflow_to_stablehlo, m) { + m.doc() = "TensorFlow to StableHLO APIs."; + + // LINT.IfChange(savedmodel_to_stablehlo) + m.def( + "savedmodel_to_stablehlo", + [](absl::string_view input_path, + const std::vector& exported_model_signatures = + {"serving_default"}, + const std::vector& tag_names = {"serve"}, + absl::string_view input_arg_shapes_str = "") -> py::bytes { + auto module_bytecode = + PywrapSavedModelToStablehlo(input_path, exported_model_signatures, + tag_names, input_arg_shapes_str); + if (!module_bytecode.ok()) { + PyErr_SetString(PyExc_ValueError, + module_bytecode.status().ToString().c_str()); + throw py::error_already_set(); + } + return py::bytes(module_bytecode.value()); + }, + R"pbdoc( + Converts a TensorFlow SavedModel into StableHLO bytecode. + + * input-path: The path to the input TensorFlow SavedModel. + * exported-model-signatures: Comma-separated list of exported model + signatures to convert. + * tag_names: Comma-separated list of tags for loading SavedModel. + * input-arg-shapes: A string representation of input argument shapes for + 'main' entry-point, separating tensors with ':', dimension with ',', and + using '?' for unknown sizes. For example, 'input-arg-shapes=1,2::1,?' + expresses argument shapes [1,2], [] and [1,?]. + )pbdoc", + py::arg("input_path"), + py::arg("exported_model_signatures") = + std::vector{"serving_default"}, + py::arg("tag_names") = std::vector{"serve"}, + py::arg("input_arg_shapes_str") = ""); + // LINT.ThenChange(pywrap_tensorflow_to_stablehlo.pyi:savedmodel_to_stablehlo) + // + // LINT.IfChange(tensorflow_module_to_stablehlo) + m.def( + "tensorflow_module_to_stablehlo", + [](absl::string_view module_op_str, + absl::string_view input_arg_shapes_str) -> py::bytes { + auto module_bytecode = + PywrapTfModuleToStablehlo(module_op_str, input_arg_shapes_str); + if (!module_bytecode.ok()) { + PyErr_SetString(PyExc_ValueError, + module_bytecode.status().ToString().c_str()); + throw py::error_already_set(); + } + return py::bytes(module_bytecode.value()); + }, + R"pbdoc( + Converts a TensorFlow MLIR module string into StableHLO bytecode. + + * module: TensorFlow MLIR module string. + * input-arg-shapes: A string representation of input argument shapes for + 'main' entry-point, separating tensors with ':', dimension with ',', and + using '?' for unknown sizes. For example, 'input-arg-shapes=1,2::1,?' + expresses argument shapes [1,2], [] and [1,?]. + )pbdoc", + py::arg("module"), py::arg("input_arg_shapes_str") = ""); + // LINT.ThenChange(pywrap_tensorflow_to_stablehlo.pyi:tensorflow_module_to_stablehlo) +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.pyi b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.pyi new file mode 100644 index 0000000000000000000000000000000000000000..6b0e1806783fffe1c3a52600803d9017fb65ec78 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo.pyi @@ -0,0 +1,30 @@ +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# LINT.IfChange(savedmodel_to_stablehlo) +def savedmodel_to_stablehlo( + input_path: str, + exported_model_signatures: list[str] = ["serving_default"], + tag_names: list[str] = ["serve"], + input_arg_shapes_str: str = "", +) -> bytes: ... +# LINT.ThenChange() + +# LINT.IfChange(tensorflow_module_to_stablehlo) +def tensorflow_module_to_stablehlo( + module: str, + input_arg_shapes_str: str = "", +) -> bytes: ... +# LINT.ThenChange() diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.cc new file mode 100644 index 0000000000000000000000000000000000000000..970ba57c98f36651237f13415ee3076451ae4be7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.cc @@ -0,0 +1,141 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Bytecode/BytecodeWriter.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.h" +#include "tensorflow/core/platform/path.h" + +namespace mlir::tensorflow_to_stablehlo::pywrap { + +absl::StatusOr ModuleToBytecode(ModuleOp module) { + std::string bytecode; + llvm::raw_string_ostream os(bytecode); + mlir::BytecodeWriterConfig config; + if (mlir::failed(mlir::writeBytecodeToFile(module, os, config))) { + return absl::InvalidArgumentError("mlir::writeBytecodeToFile failed"); + } + return bytecode; +} + +absl::StatusOr ExportModule(ModuleOp module) { + const std::string output_filename = tensorflow::io::GetTempFilename(".mlir"); + std::string error_msg; + auto output = openOutputFile(output_filename, &error_msg); + if (output == nullptr) { + return absl::UnknownError( + absl::StrCat("Unable to open output path: ", error_msg)); + } + + std::string result; + llvm::raw_string_ostream os(result); + OpPrintingFlags printing_flags; + module.print(os, printing_flags); + + output->os() << result; + output->keep(); + + return output_filename; +} + +absl::StatusOr PywrapSavedModelToStablehlo( + absl::string_view input_path, + const std::vector& exported_model_signatures, + const std::vector& tag_names, + absl::string_view input_arg_shapes_str) { + mlir::DialectRegistry registry; + RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + context.loadAllAvailableDialects(); + + auto module = + TfToStablehlo(input_path, &context, exported_model_signatures, tag_names, + input_arg_shapes_str, /*is_input_mlir_module=*/false); + + if (!module.ok()) { + return absl::UnknownError( + absl::StrCat("Failed to convert SavedModel to StableHLO: ", + module.status().message())); + } + + auto bytecode = ModuleToBytecode(module.value().get()); + if (!bytecode.ok()) { + return absl::UnknownError( + absl::StrCat("Failed to serialize MLIR module to bytecode: ", + bytecode.status().message())); + } + + return bytecode.value(); +} + +absl::StatusOr PywrapTfModuleToStablehlo( + absl::string_view module_op_str, absl::string_view input_arg_shapes_str) { + mlir::DialectRegistry registry; + RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + context.loadAllAvailableDialects(); + + auto tf_module = mlir::parseSourceString(module_op_str, &context); + if (!tf_module) { + return absl::UnknownError("Failed to parse MLIR module"); + } + + auto mlir_file_path = ExportModule(*tf_module); + if (!mlir_file_path.ok()) { + return absl::UnknownError( + absl::StrCat("Failed to write MLIR module to file.", + mlir_file_path.status().message())); + } + + auto module = TfToStablehlo(*mlir_file_path, &context, + /*exported_model_signatures=*/{}, + /*tag_names=*/{}, input_arg_shapes_str, + /*is_input_mlir_module=*/true); + + if (!module.ok()) { + return absl::UnknownError( + absl::StrCat(" Failed to convert SavedModel to StableHLO: ", + module.status().message())); + } + + auto bytecode = ModuleToBytecode(module.value().get()); + if (!bytecode.ok()) { + return absl::UnknownError( + absl::StrCat("Failed to serialize MLIR module to bytecode: ", + bytecode.status().message())); + } + + return bytecode.value(); +} + +} // namespace mlir::tensorflow_to_stablehlo::pywrap diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.h new file mode 100644 index 0000000000000000000000000000000000000000..82765bc6d941bab1d508f9e058c0f9cbfae996d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/python/pywrap_tensorflow_to_stablehlo_lib.h @@ -0,0 +1,67 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_PYTHON_PYWRAP_TENSORFLOW_TO_STABLEHLO_LIB_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_PYTHON_PYWRAP_TENSORFLOW_TO_STABLEHLO_LIB_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" + +namespace mlir::tensorflow_to_stablehlo::pywrap { + +// Converts a TensorFlow SavedModel to a StableHLO MLIR module and serializes it +// to bytecode. +// +// Args: +// input_path: The path to the SavedModel directory. +// exported_model_signatures: Comma-separated list of exported model +// signatures to convert. tag_names: Comma-separated list of tags for loading +// SavedModel. +// input_arg_shapes_str: A string representation of input argument +// shapes for 'main' entry-point, separating tensors with ':', dimension +// with ',', and using '?' for unknown sizes. For example, +// 'input-arg-shapes=1,2::1,?' expresses argument shapes [1,2], [] and [1,?]. +// +// Returns: +// An absl::StatusOr containing the serialized bytecode of the StableHLO +// module on success, or an error status on failure. +absl::StatusOr PywrapSavedModelToStablehlo( + absl::string_view input_path, + const std::vector& exported_model_signatures, + const std::vector& tag_names, + absl::string_view input_arg_shapes_str); + +// Converts a TensorFlow MLIR module string to a StableHLO MLIR module and +// serializes it to bytecode. +// +// Args: +// module_op_str: TensorFlow MLIR module string. +// input_arg_shapes_str: A string representation of input argument +// shapes for 'main' entry-point, separating tensors with ':', dimension +// with ',', and using '?' for unknown sizes. For example, +// 'input-arg-shapes=1,2::1,?' expresses argument shapes [1,2], [] and [1,?]. +// +// Returns: +// An absl::StatusOr containing the serialized bytecode of the StableHLO +// module on success, or an error status on failure. +absl::StatusOr PywrapTfModuleToStablehlo( + absl::string_view module_op_str, absl::string_view input_arg_shapes_str); + +} // namespace mlir::tensorflow_to_stablehlo::pywrap + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_PYTHON_PYWRAP_TENSORFLOW_TO_STABLEHLO_LIB_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tests/test_tf_to_stablehlo.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tests/test_tf_to_stablehlo.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f918f249a4e46d9f81585a26b1a153b437babf1c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tests/test_tf_to_stablehlo.mlir @@ -0,0 +1,22 @@ +// RUN: tf-to-stablehlo-translate %s --input-arg-shapes=1 -o - | FileCheck %s + +// CHECK-LABEL: func.func @main +// CHECK: %[[UQ:.*]] = stablehlo.uniform_quantize %arg0 : (tensor<1xf32>) -> tensor<1x!quant.uniform> +// CHECK: %[[BITCAST_CONVERT_0:.*]] = stablehlo.bitcast_convert %[[UQ]] : (tensor<1x!quant.uniform>) -> tensor<1xi8> +// CHECK: %[[BITCAST_CONVERT_1:.*]] = stablehlo.bitcast_convert %[[BITCAST_CONVERT_0]] : (tensor<1xi8>) -> tensor<1x!quant.uniform> +// CHECK: %[[UDQ:.*]] = stablehlo.uniform_dequantize %[[BITCAST_CONVERT_1]] : (tensor<1x!quant.uniform>) -> tensor<1xf32> +// CHECK: return %[[UDQ]] : tensor<1xf32> +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0 : tensor) -> tensor { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + %1 = "tf.UniformDequantize"(%0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + func.return %1 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.cc new file mode 100644 index 0000000000000000000000000000000000000000..becb6f3abc2ab1444a95483e69d2004c4c086cab --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.cc @@ -0,0 +1,138 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/SourceMgr.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/quantization/stablehlo/cc/saved_model_import.h" +#include "tensorflow/compiler/mlir/quantization/tensorflow/quantize_preprocess.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/public/session.h" + +namespace mlir { +namespace { + +// Extract the mlir TF module and optionally a ::tensorflow::SavedModelBundle +// from a saved model or from an mlir file. +absl::StatusOr ImportSavedModelOrTfMlir( + absl::string_view input_path, MLIRContext* context, + const std::vector& exported_model_signatures, + const std::vector& tag_names, bool is_input_mlir_module) { + if (is_input_mlir_module) { + std::string error_message; + std::unique_ptr file = + openInputFile(input_path, &error_message); + if (!file) { + return absl::AbortedError( + absl::StrCat("Failed to parse input MLIR model: ", error_message)); + } + + llvm::SourceMgr source_mgr; + source_mgr.AddNewSourceBuffer(std::move(file), llvm::SMLoc()); + auto module = parseSourceFile(source_mgr, context); + if (module->getOperation() == nullptr) { + return absl::AbortedError("Failed to parse input MLIR model."); + } + + return quant::stablehlo::ImportedMlirModuleOp(std::move(module), nullptr); + } + + std::unordered_set tag_set(tag_names.begin(), tag_names.end()); + return quant::stablehlo::SavedModelToMlirModuleOp( + input_path, tag_set, exported_model_signatures, *context); +} + +// Convert an TF module to a StableHLO module +absl::StatusOr> ConvertTFToStablehlo( + quant::stablehlo::ImportedMlirModuleOp imported_module, + absl::string_view input_path, MLIRContext* context, + const std::vector& tag_names, + absl::string_view input_arg_shapes_str, bool is_input_mlir_module) { + auto [module_op, saved_model_bundle] = std::move(imported_module); + + // Collect the names of the functions that have aliases so that they may not + // be inlined. + absl::flat_hash_set aliased_function_names; + if (!is_input_mlir_module) { + std::unordered_set tag_set(tag_names.begin(), tag_names.end()); + TF_ASSIGN_OR_RETURN( + auto function_aliases, + quant::stablehlo::GetFunctionAliases(input_path, tag_set)); + quant::stablehlo::UpdateFunctionAliases(function_aliases, *module_op); + absl::c_for_each(function_aliases, [&](const auto& aliases) { + return aliased_function_names.insert(aliases.first); + }); + } + + std::optional session; + if (saved_model_bundle) { + session = saved_model_bundle->GetSession(); + } + TF_ASSIGN_OR_RETURN(auto input_arg_shapes_vec, + TF::ParseArgumentShapes(input_arg_shapes_str)); + llvm::SmallVector> input_arg_shapes( + input_arg_shapes_vec.begin(), input_arg_shapes_vec.end()); + TF_RETURN_IF_ERROR(tensorflow::quantization::PreprocessAndFreezeGraph( + /*mlir_dump_file_prefix=*/"", /*is_inliner_run=*/true, + /*noinline_functions=*/aliased_function_names, *module_op, context, + session, + /*run_tf_to_stablehlo=*/true, /*deserialize_xla_call_module=*/false, + input_arg_shapes)); + + return std::move(module_op); +} + +} // namespace + +absl::StatusOr> TfToStablehlo( + absl::string_view input_path, MLIRContext* context, + const std::vector& exported_model_signatures, + const std::vector& tag_names, + absl::string_view input_arg_shapes_str, bool is_input_mlir_module) { + auto import_module_status = + ImportSavedModelOrTfMlir(input_path, context, exported_model_signatures, + tag_names, is_input_mlir_module); + if (!import_module_status.ok()) { + return import_module_status.status(); + } + + return ConvertTFToStablehlo(*std::move(import_module_status), input_path, + context, tag_names, input_arg_shapes_str, + is_input_mlir_module); +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.h new file mode 100644 index 0000000000000000000000000000000000000000..96844f44d20cc095338faec91c559ec2634f00a0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.h @@ -0,0 +1,56 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_TF_TO_STABLEHLO_H_ +#define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_TF_TO_STABLEHLO_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project + +namespace mlir { + +// Converts a TensorFlow model (either from a SavedModel or an MLIR module) to a +// StableHLO MLIR module. +// +// Args: +// input_path: The path to the input TensorFlow SavedModel or MLIR module. +// context: The MLIR context to use for parsing or creating the MLIR module. +// exported_model_signatures: List of exported model signatures (strings) to +// convert. +// tag_names: List of tag names (strings) used for loading SavedModel. +// Ignored for MLIR input. +// input_arg_shapes_str: A string representation of input argument shapes for +// 'main' entry-point, separating tensors with ':', dimension with ',', and +// using '?' for unknown sizes. For example, 'input-arg-shapes=1,2::1,?' +// expresses argument shapes [1,2], [] and [1,?]. +// is_input_mlir_module: If true, `input_path` is treated as an MLIR +// module instead of a SavedModel. +// +// Returns: +// An absl::StatusOr containing the converted StableHLO MLIR module on +// success, or an absl::Status with an error message on failure. +absl::StatusOr> TfToStablehlo( + absl::string_view input_path, MLIRContext* context, + const std::vector& exported_model_signatures, + const std::vector& tag_names, + absl::string_view input_arg_shapes_str, bool is_input_mlir_module); + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TO_STABLEHLO_TF_TO_STABLEHLO_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo_translate.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo_translate.cc new file mode 100644 index 0000000000000000000000000000000000000000..23a2c2b0c6d5f8b87316d2fe53ce525f553e46bc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo_translate.cc @@ -0,0 +1,134 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow_to_stablehlo/tf_to_stablehlo.h" + +namespace { + +using llvm::cl::opt; + +// NOLINTNEXTLINE +opt input_path(llvm::cl::Positional, + llvm::cl::desc(""), llvm::cl::Required); + +// NOLINTNEXTLINE +opt output_filename("o", llvm::cl::desc(""), + llvm::cl::Optional, llvm::cl::init("-")); + +// NOLINTNEXTLINE +opt input_arg_shapes_str( + "input-arg-shapes", + llvm::cl::desc( + "A string representation of input argument shapes for 'main' " + "entry-point, separating tensors with ':', dimension with ',', and " + "using '?' for unknown sizes. For example, 'input-arg-shapes=1,2::1,?' " + "expresses argument shapes [1,2], [] and [1,?]"), + llvm::cl::Optional, llvm::cl::init("")); + +// NOLINTNEXTLINE +opt exported_model_signatures( + "exported-model-signatures", + llvm::cl::desc( + "Comma-separated list of exported model signatures to convert"), + llvm::cl::Optional, llvm::cl::init("serving_default")); + +// NOLINTNEXTLINE +opt tag_names( + "tags", + llvm::cl::desc("Comma-separated list of tags for loading SavedModel. " + "Ignored for MLIR input"), + llvm::cl::Optional, llvm::cl::init("serve")); + +// NOLINTNEXTLINE +opt elide_large_elements_attrs( + "e", + llvm::cl::desc( + "Elide large elements attrs while dumping the output StableHLO."), + llvm::cl::Optional, llvm::cl::init(false)); + +} // namespace + +namespace mlir { + +namespace { +// Dump the ModuleOp 'module' to the file specified using 'outputFileName' +absl::Status ExportModule(ModuleOp module) { + std::string error_msg; + auto output = openOutputFile(output_filename, &error_msg); + if (output == nullptr) { + return absl::AbortedError( + absl::StrCat("Unable to write to output path: ", error_msg)); + } + + // Export StableHLO MLIR as output + std::string result; + llvm::raw_string_ostream os(result); + OpPrintingFlags printing_flags; + if (elide_large_elements_attrs) { + printing_flags.elideLargeElementsAttrs(); + } + module.print(os, printing_flags); + os.flush(); + + output->os() << result; + output->keep(); + + return absl::OkStatus(); +} + +} // namespace +} // namespace mlir + +int main(int argc, char** argv) { + tensorflow::InitMlir y(&argc, &argv); + llvm::cl::ParseCommandLineOptions(argc, argv, + "TF Saved Model to Stablehlo converter\n"); + + mlir::DialectRegistry registry; + RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + context.loadAllAvailableDialects(); + + bool is_input_mlir_module = absl::EndsWith(input_path, ".mlir"); + std::vector exported_model_signatures_in_vector = + absl::StrSplit(exported_model_signatures, ','); + std::vector tag_names_in_vector = absl::StrSplit(tag_names, ','); + auto module = TfToStablehlo( + input_path, &context, exported_model_signatures_in_vector, + tag_names_in_vector, input_arg_shapes_str, is_input_mlir_module); + if (!module.ok()) { + llvm::errs() << module.status().ToString() << "\n"; + return module.status().raw_code(); + } + + return mlir::ExportModule(module->get()).raw_code(); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..2abedc5490138f6273ecbf12189bd6e9db6845eb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/BUILD @@ -0,0 +1,37 @@ +# Description: +# TF2XLA Bridge and related components. + +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package_group( + name = "tensorflow_mlir_tf2xla", + packages = [ + "//tensorflow/compiler/mlir/tf2xla/...", + ], +) + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +cc_library( + name = "mlir_bridge_rollout_policy", + srcs = ["mlir_bridge_rollout_policy.cc"], + hdrs = ["mlir_bridge_rollout_policy.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:flags", + "//tensorflow/core:framework", + "//tensorflow/core:graph", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/types:optional", + "@llvm-project//mlir:IR", + ], +) + +alias( + name = "compile_mlir_util", + actual = "//tensorflow/compiler/mlir/tf2xla/api/v1:compile_mlir_util_no_tf_dialect_passes", +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..4eec04bcf37a6033b8abbba6a617769013e66479 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/BUILD @@ -0,0 +1,336 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow/core/platform:build_config.bzl", "tf_additional_all_protos", "tf_proto_library") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tf2xla/api:__subpackages__", + "//tensorflow/compiler/mlir/tf2xla/internal:__subpackages__", + ], +) + +cc_library( + name = "compile_mlir_util_no_tf_dialect_passes", + srcs = ["compile_mlir_util.cc"], + hdrs = ["compile_mlir_util.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/mlir/quantization/stablehlo:bridge_passes", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:import_model", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:shape_inference_pass", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tf2xla:mlir_bridge_rollout_policy", + "//tensorflow/compiler/mlir/tf2xla/api/v2:graph_to_tf_executor", + "//tensorflow/compiler/mlir/tf2xla/internal:mlir_pass_instrumentation", + "//tensorflow/compiler/mlir/tf2xla/internal/passes:lowering_passes", + "//tensorflow/compiler/mlir/tf2xla/transforms:xla_legalize_tf_with_tf2xla", + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_argument", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/common_runtime:core_cpu_internal", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:logging", + "//tensorflow/core/platform:status", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:Transforms", + "@stablehlo//:base", + "@xla//xla:shape_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:xla_computation", + "@xla//xla/hlo/ir:hlo", + "@xla//xla/hlo/translate:register", + "@xla//xla/hlo/translate/mhlo_to_hlo:layout_util", + "@xla//xla/hlo/translate/mhlo_to_hlo:mlir_hlo_to_hlo", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo", + "@xla//xla/mlir_hlo:mhlo_passes", + "@xla//xla/mlir_hlo:stablehlo_extension_passes", + "@xla//xla/service:hlo_proto_cc", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "compile_mlir_util_test", + srcs = ["compile_mlir_util_test.cc"], + deps = [ + ":compile_mlir_util_no_tf_dialect_passes", + "//tensorflow/compiler/jit:xla_compile_util", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tf2xla/internal:test_matchers", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:fake_input", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@xla//xla:shape_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:xla_builder", + "@xla//xla/hlo/builder:xla_computation", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "compile_tf_graph", + srcs = ["compile_tf_graph.cc"], + hdrs = ["compile_tf_graph.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:set_tpu_infeed_layout", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu:tpu_compile", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:variant", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla:shape_util", + "@xla//xla:status_macros", + "@xla//xla/client:compile_only_client", + "@xla//xla/hlo/ir:hlo", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/pjrt/proto:compile_options_proto_cc", + "@xla//xla/service:hlo_proto_cc", + ], +) + +tf_cc_test( + name = "compile_tf_graph_test", + testonly = 1, + srcs = ["compile_tf_graph_test.cc"], + data = [ + "testdata/prepare_to_library.mlir", + ], + linkstatic = 1, + deps = [ + ":compile_tf_graph", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:xla_tpu_device", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tf2xla/internal:test_matchers", + "//tensorflow/compiler/mlir/tf2xla/internal/utils:test_metadata_config", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_tpu_backend_registration", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "//tensorflow/core/tpu/kernels/xla:host_compute_ops", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_googletest//:gtest", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@xla//xla:shape_util", + "@xla//xla/client:client_library", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/stream_executor:platform", + "@xla//xla/stream_executor:platform_manager", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/lib/monitoring:test_utils", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "cluster_tf", + srcs = ["cluster_tf.cc"], + hdrs = ["cluster_tf.h"], + visibility = [ + "//tensorflow/compiler/tf2xla:__pkg__", + ], + deps = [ + ":tf_dialect_to_executor", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:lower_cluster_to_runtime_ops", + "//tensorflow/compiler/mlir/tf2xla/internal:clustering_bridge_passes", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/compiler/mlir/tf2xla/internal/inference:inference_metrics_pass", + "//tensorflow/compiler/mlir/tf2xla/internal/passes:clustering_passes", + "//tensorflow/core:framework", + "//tensorflow/core:lib_proto_parsing", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:stacktrace", + "//tensorflow/core/platform:status", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tsl//tsl/platform:error_logging", + "@xla//xla/tsl/platform:errors", + ], +) + +tf_cc_test( + name = "cluster_tf_test", + srcs = ["cluster_tf_test.cc"], + data = [ + "testdata/empty_func.mlir", + "testdata/invalid_executor.mlir", + "testdata/multiple_submodules.mlir", + ], + deps = [ + ":cluster_tf", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:tf_dialect_lib", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:status", + ], +) + +cc_library( + name = "tf_dialect_to_executor", + srcs = ["tf_dialect_to_executor.cc"], + hdrs = ["tf_dialect_to_executor.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/core:framework", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:status", + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tsl//tsl/platform:error_logging", + "@xla//xla/tsl/lib/monitoring:counter", + "@xla//xla/tsl/platform:status", + ], +) + +tf_cc_test( + name = "tf_dialect_to_executor_test", + srcs = ["tf_dialect_to_executor_test.cc"], + data = [ + "testdata/empty_func.mlir", + "testdata/invalid_executor.mlir", + ], + deps = [ + ":tf_dialect_to_executor", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:status", + ], +) + +tf_proto_library( + name = "mlir_bridge_config_v1_proto", + srcs = ["mlir_bridge_config_v1.proto"], + protodeps = tf_additional_all_protos(), + visibility = ["//visibility:public"], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..013eca63bf0264348d8418c8e790cc5170fe7a0a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.cc @@ -0,0 +1,245 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/metrics.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/stacktrace.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::OpPassManager; +using mlir::PassManager; +using mlir::func::FuncOp; + +namespace { + +void CreateReplicatedBridgePipelineV1(OpPassManager &pm) { + pm.addPass( + tensorflow::tf2xla::internal::CreateTPUValidateSessionInputsPass()); + pm.addPass(mlir::tf2xla::internal::CreateInferenceMetricsPass()); + + // Convert to unified compilation and replication attributes. + pm.addNestedPass( + mlir::TF::CreateCanonicalizeCompileAndReplicateAttributesPass()); + // Guarantee all functions have one use, which enables more exact shape + // inference. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + // For V1 compatibility, we process a module where the graph does not have + // feeds and fetched. We extract first the TPU computation in a submodule, + // where it'll be in a function with args and returned values, much more + // like a TF v2 module. We can then run the usual pipeline on this nested + // module. Afterward we inline back in the parent module and delete the + // nested one. + pm.addPass(mlir::tf_executor::CreateTFExecutorTPUV1IslandCoarseningPass()); + pm.addPass(mlir::tf_executor::CreateTFExecutorTPUV1IslandOutliningPass()); +} + +// Run the TF XLA Bridge based on the input pipeline, which can be either TPU +// bridge pipeline or non TPU bridge pipeline. +absl::Status RunTFXLABridge( + ModuleOp module, + llvm::function_ref pipeline_builder, + llvm::StringRef module_name = llvm::StringRef(), + llvm::StringRef dump_prefix = "tf_xla_bridge_v1") { + // Explicitly check that the TensorFlow dialect can constant fold ops. + // Constant folding is essential for the bridge. Without this check, the + // bridge may fail with an error that is difficult to understand and not + // actionable. + if (!mlir::TF::TensorFlowDialect::HasConstantFoldHook()) { + return absl::InternalError( + "TensorFlow dialect missing constant fold hook in TFXLA bridge phase " + "1; this could happen if the binary doesn't link the constant fold " + "hook registration library."); + } + + PassManager bridge(module.getContext()); + bridge.enableVerifier(); + ::tensorflow::applyTensorflowAndCLOptions(bridge); + + // Populate a passmanager with the list of passes that implement the bridge. + pipeline_builder(bridge); + + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile(DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "_" + dump_prefix.str() + "_before"), + module, llvm::StringRef(), &bridge); + } + + if (VLOG_IS_ON(2) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), + kDebugGroupBridgePhase1Clustering)) { + internal::EnablePassIRPrinting(bridge, kDebugGroupBridgePhase1Clustering, + module_name); + } + + LogicalResult result = bridge.run(module); + (void)result; + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile(DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "_" + dump_prefix.str() + "_after"), + module, llvm::StringRef(), &bridge); + } + + return diag_handler.ConsumeStatus(); +} + +} // namespace + +absl::Status RecordStatusIfError(const std::string error_prefix, + bool is_in_fallback_enabled_mode, + absl::Status status) { + if (status.ok()) { + return status; + } + + tensorflow::metrics::UpdateTfMlirBridgeFirstPhaseCounter( + /*bridge_type=*/mlir::TF::kMlirPh1BridgeCounterReplicated, + /*bridge_version=*/mlir::TF::kMlirPh1BridgeCounterV1, + /*device_type*/ mlir::TF::kMlirPh1BridgeCounterTpu, + /*fallback_enabled=*/is_in_fallback_enabled_mode, + /*result=*/"failure"); + tsl::error_logging::Log(mlir::TF::kBridgeComponent, + "TFXLA_PHASE_ONE_MLIR_TPU_V1_COMPAT_BRIDGE", + status.ToString()) + .IgnoreError(); + + return status; +} + +// V1 Compat Bridge takes a TF Executor dialect and extracts the TF2 portion +// and inserts it into a submodule. We just want to run the clustering +// portion of the pipeline on just the single submodule. +absl::Status RunClusteringPipelineOnSubmodule( + ModuleOp parent_module, bool is_in_fallback_enabled_mode) { + int num_submodules = 0; + absl::Status clustering_pipeline_status; + parent_module.walk([&](ModuleOp submodule) { + if (submodule == parent_module) return mlir::WalkResult::advance(); + num_submodules++; + clustering_pipeline_status = RunTFXLABridge( + submodule, + [](OpPassManager &pm) { + internal::AddReplicatedBridgeClusteringPipelinePasses(pm); + }, + /*module_name=*/"", /*dump_prefix=*/"tf_xla_clustering_bridge_v1"); + if (num_submodules > 1) { + return mlir::WalkResult::interrupt(); + } + + return mlir::WalkResult::advance(); + }); + + if (num_submodules > 1) { + auto num_submodules_error = absl::InternalError( + "V1 Compat Bridge has more than one submodule. Erroring out."); + TF_RETURN_IF_ERROR(RecordStatusIfError( + /*error_prefix=*/"Bridge has more than one submodule:", + is_in_fallback_enabled_mode, num_submodules_error)); + } + + if (!clustering_pipeline_status.ok()) { + TF_RETURN_IF_ERROR(RecordStatusIfError( + /*error_prefix=*/"Bridge Errored running clustering pipeline:", + is_in_fallback_enabled_mode, clustering_pipeline_status)); + } + + return absl::OkStatus(); +} + +absl::Status RunSessionTf2xlaClusteringBridge( + ModuleOp module, bool is_in_fallback_enabled_mode) { + VLOG(2) << "TPU Sessions Bridge called stack trace is " + << "(NOTE: this is not an error; rather the stack trace for " + "debugging) : " + << tensorflow::CurrentStackTrace(); + + absl::Status functional_import_status = RunTFXLABridge( + module, [](OpPassManager &pm) { CreateReplicatedBridgePipelineV1(pm); }, + /*module_name=*/"", /*dump_prefix=*/"tf_xla_functional_import_bridge_v1"); + TF_RETURN_IF_ERROR(RecordStatusIfError( + /*error_prefix=*/"Bridge Function Import V1", is_in_fallback_enabled_mode, + functional_import_status)); + + TF_RETURN_IF_ERROR( + RunClusteringPipelineOnSubmodule(module, is_in_fallback_enabled_mode)); + + tensorflow::metrics::UpdateTfMlirBridgeFirstPhaseCounter( + /*bridge_type=*/mlir::TF::kMlirPh1BridgeCounterReplicated, + /*bridge_version=*/mlir::TF::kMlirPh1BridgeCounterV1, + /*device_type*/ mlir::TF::kMlirPh1BridgeCounterTpu, + /*n_fallback_enabled*/ is_in_fallback_enabled_mode, + /*result=*/"success"); + + return absl::OkStatus(); +} + +// Registers a pipeline builder function for TF TPU V1 bridge. +mlir::PassPipelineRegistration<> replicated_clustering_bridge_v1( + "tf-replicated-clustering-bridge-v1", + "Run all the passes involved in transforming a TensorFlow V1 graph before " + "execution so that it is suitable for targeting devices.", + CreateReplicatedBridgePipelineV1); + +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.h new file mode 100644 index 0000000000000000000000000000000000000000..0fd295cb045091b4eb431c4a9f2c67dc3927836f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.h @@ -0,0 +1,45 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_CLUSTER_TF_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_CLUSTER_TF_H_ + +#include "absl/status/status.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +// Run all the passes involved in transforming the graph before execution so +// that it is suitable for targeting devices when called via the TF1 Session +// API. +// These transformations take as input a Tensorflow Graph as an MLIR Module +// and transforms the module in place to cluster the given ops for compilation +// that is compatible with the given device_type. The MLIR should be in the TF +// Executor Dialect for graph nodes and edges or TF Functional. It will convert +// to TF Functional internally. Individual Op inside a node should be the +// Tensorflow Dialect. The output MLIR is in the TF Functional Dialect. The +// input MLIR should not have infeed and outfeed ops, which are unsupported via +// this API. Returns OkStatus if passed, otherwise an error. +absl::Status RunSessionTf2xlaClusteringBridge(mlir::ModuleOp module, + bool is_in_fallback_enabled_mode); + +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_CLUSTER_TF_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..2e0c795518b74a6d4d5cbcfde893366a5dddea05 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf_test.cc @@ -0,0 +1,115 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/cluster_tf.h" + +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/resource_loader.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { +namespace { + +using ::mlir::DialectRegistry; +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::tensorflow::monitoring::testing::CellReader; + +static constexpr char kCompilationStreamz[] = + "/tensorflow/core/tf_mlir_bridge_first_phase_v2_count"; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/api/v1/testdata/"); +} + +class SessionClusterTensorflowDialectTest : public ::testing::Test { + public: + SessionClusterTensorflowDialectTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(SessionClusterTensorflowDialectTest, ClustersTf) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK( + RunSessionTf2xlaClusteringBridge(*mlir_module_, + /*is_in_fallback_enabled_mode=*/false)); + EXPECT_EQ(compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV1, + mlir::TF::kMlirPh1BridgeCounterTpu, + "fallback_disabled", "success"), + 1); +} + +TEST_F(SessionClusterTensorflowDialectTest, FailsWithMultipleSubmodules) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("multiple_submodules.mlir")); + + EXPECT_FALSE( + RunSessionTf2xlaClusteringBridge(*mlir_module_, + /*is_in_fallback_enabled_mode=*/false) + .ok()); + + EXPECT_EQ(compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV1, + mlir::TF::kMlirPh1BridgeCounterTpu, + "fallback_disabled", "failure"), + 1); +} + +} // namespace +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..b56e7a0fc4cee6d555c870b588cce6ace30c60e7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.cc @@ -0,0 +1,1139 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.h" +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "stablehlo/dialect/Base.h" // from @stablehlo +#include "tensorflow/compiler/mlir/quantization/stablehlo/passes/bridge/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/import_model.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/hlo/ir/hlo_sharding.h" +#include "xla/hlo/translate/mhlo_to_hlo/layout_util.h" +#include "xla/hlo/translate/mhlo_to_hlo/mlir_hlo_to_hlo.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/hlo/translate/register.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/mlir_hlo/mhlo/transforms/passes.h" +#include "xla/mlir_hlo/stablehlo_ext/transforms/passes.h" +#include "xla/service/hlo.pb.h" +#include "xla/shape.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/error_payloads.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/core_platform_payloads.pb.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace { +constexpr absl::string_view kGroupSizeAttrName = + "tf2xla.collective_info.group_size"; +constexpr absl::string_view kGroupKeyAttrName = + "tf2xla.collective_info.group_key"; + +// Extracts shape from XlaArgument as TensorShape. If shape is a xla::Shape, +// that is converted to a TensorShape. +absl::StatusOr GetTensorShapeFromXlaArgument( + const XlaArgument& arg) { + if (std::holds_alternative(arg.shape)) { + TensorShape arg_shape; + TF_RETURN_IF_ERROR( + XLAShapeToTensorShape(std::get(arg.shape), &arg_shape)); + return arg_shape; + } else { + return std::get(arg.shape); + } +} + +absl::Status MaybeRewriteLayoutWithShardedShape( + mlir::StringAttr sharding, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + xla::Shape* shape) { + if (!sharding) return absl::OkStatus(); + + xla::OpSharding op_sharding; + if (tensorflow::DecodeShardingAttribute(sharding, op_sharding).failed()) { + return absl::InvalidArgumentError(absl::StrCat( + "failed to parse sharding '", sharding.getValue().str(), "'")); + } + std::optional hlo_sharding; + TF_ASSIGN_OR_RETURN(hlo_sharding, xla::HloSharding::FromProto(op_sharding)); + TF_RETURN_IF_ERROR(RewriteLayoutWithShardedShape( + hlo_sharding, /*use_fast_memory=*/false, shape_determination_fns, shape)); + return absl::OkStatus(); +} + +// Converts arg_shapes to xla::Shape's and store into xla_input_shapes. +absl::Status GetXlaInputShapes( + mlir::ModuleOp module, llvm::ArrayRef arg_shapes, + bool use_tuple_args, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + std::vector* xla_input_shapes) { + xla_input_shapes->clear(); + + mlir::func::FuncOp main_func = + module.lookupSymbol("main"); + TF_RET_CHECK(main_func != nullptr) << "No main function found"; + mlir::FunctionType func_type = main_func.getFunctionType(); + + int num_args = func_type.getNumInputs(); + xla_input_shapes->reserve(num_args); + + std::vector individual_arg_shapes; + individual_arg_shapes.reserve(num_args); + for (int i = 0; i < num_args; ++i) { + individual_arg_shapes.emplace_back(); + xla::Shape& xla_shape = individual_arg_shapes.back(); + + DataType arg_dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(func_type.getInput(i), &arg_dtype)); + + auto layout_preference = shape_determination_fns.layout_preference_fn( + arg_shapes[i].shape, arg_dtype, std::nullopt); + TF_ASSIGN_OR_RETURN(xla_shape, + shape_determination_fns.shape_representation_fn( + arg_shapes[i].shape, arg_dtype, + /*use_fast_memory=*/false, layout_preference)); + + // Rewrite layout with sharding, if sharding is set. + auto sharding = + main_func.getArgAttrOfType(i, "mhlo.sharding"); + TF_RETURN_IF_ERROR(MaybeRewriteLayoutWithShardedShape( + sharding, shape_determination_fns, &xla_shape)); + } + if (use_tuple_args) { + xla_input_shapes->push_back( + xla::ShapeUtil::MakeTupleShape(individual_arg_shapes)); + } else { + *xla_input_shapes = individual_arg_shapes; + } + return absl::OkStatus(); +} + +// Returns a static ranked tensor type corresponding to the given static or +// bounded type by using the bounds as dimension sizes. Returns null if is +// neither. +mlir::RankedTensorType GetBufferType(mlir::Type ty) { + auto ranked_ty = mlir::dyn_cast_or_null(ty); + if (!ranked_ty) return {}; + + int64_t rank = ranked_ty.getRank(); + llvm::SmallVector dims = llvm::to_vector<4>(ranked_ty.getShape()); + llvm::ArrayRef bounds = + mlir::hlo::encodingToBounds(ranked_ty.getEncoding()); + if (!bounds.empty()) { + for (int64_t dim = 0; dim < rank; ++dim) { + if (dims[dim] == mlir::ShapedType::kDynamic) { + dims[dim] = bounds[dim]; + } + } + } + return GetTypeFromTFTensorShape(dims, ranked_ty.getElementType()); +} + +// Calculates computation output shape and build OutputDescription for each +// output based on static shapes in MLIR module. If an output is a resource +// write, `resource_updates` is populated instead of `outputs` for that output. +absl::Status GetOutputInfo( + mlir::ModuleOp module, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + xla::Shape* xla_output_shape, std::vector* outputs, + std::vector* resource_updates) { + auto shape_representation_fn_no_fast_memory = + [shape_determination_fns]( + const xla::Shape& xla_shape) -> absl::StatusOr { + TensorShape shape; + TF_RETURN_IF_ERROR(XLAShapeToTensorShape(xla_shape, &shape)); + TF_ASSIGN_OR_RETURN(DataType dtype, EncodePrimitiveTypeAsDataType( + xla_shape.element_type())); + auto layout_preference = shape_determination_fns.layout_preference_fn( + shape, dtype, std::nullopt); + return shape_determination_fns.shape_representation_fn( + shape, dtype, /*use_fast_memory=*/false, layout_preference); + }; + + mlir::func::FuncOp main_func = + module.lookupSymbol("main"); + mlir::FunctionType func_type = main_func.getFunctionType(); + + outputs->clear(); + outputs->reserve(func_type.getNumResults()); + resource_updates->clear(); + resource_updates->reserve(func_type.getNumResults()); + + std::vector shapes; + shapes.reserve(func_type.getNumResults()); + + llvm::SmallDenseMap output_to_input_alias; + for (unsigned i = 0; i < main_func.getNumArguments(); ++i) + if (auto aliasing_output = main_func.getArgAttrOfType( + i, "tf.aliasing_output")) + output_to_input_alias[aliasing_output.getInt()] = i; + + auto return_op = main_func.begin()->getTerminator(); + for (const auto& type_and_idx : llvm::enumerate(func_type.getResults())) { + size_t idx = type_and_idx.index(); + auto result_ty = mlir::cast(type_and_idx.value()); + + // If the result type isn't static, then the owner of the result may be a + // cast op from a more specific bounded type to an unbounded dynamic type. + // Use the bounded type to get the buffer size. + mlir::RankedTensorType buffer_ty = result_ty; + if (!buffer_ty.hasStaticShape()) { + mlir::Value return_val = return_op->getOperand(idx); + if (auto owner = mlir::dyn_cast_or_null( + return_val.getDefiningOp())) { + // For bounded dynamic type, get a static size by taking bounds as the + // dimensions. These dimensions are marked as dynamic in xla::Shape + // below. + buffer_ty = GetBufferType(owner.getOperand().getType()); + if (!buffer_ty || !buffer_ty.hasStaticShape()) { + return absl::InvalidArgumentError( + "results needs to be static or bounded"); + } + } + } + + xla::Shape shape = xla::TypeToShape(buffer_ty); + if (shape.element_type() == xla::PRIMITIVE_TYPE_INVALID) { + return absl::InvalidArgumentError("XLA conversion failed for MLIR type."); + } + TF_ASSIGN_OR_RETURN(shape, shape_representation_fn_no_fast_memory(shape)); + + if (!result_ty.hasStaticShape()) { + int64_t rank = result_ty.getRank(); + for (int64_t dim = 0; dim < rank; ++dim) { + if (result_ty.isDynamicDim(dim)) { + shape.set_dynamic_dimension(dim, true); + } + } + } + + auto sharding = main_func.getResultAttrOfType( + type_and_idx.index(), "mhlo.sharding"); + TF_RETURN_IF_ERROR(MaybeRewriteLayoutWithShardedShape( + sharding, shape_determination_fns, &shape)); + + auto tensor_type = + mlir::dyn_cast(type_and_idx.value()); + shapes.push_back(shape); + + auto it = output_to_input_alias.find(type_and_idx.index()); + if (it != output_to_input_alias.end() && use_resource_updates_for_aliases) { + // Add resource write. + resource_updates->emplace_back(); + XlaResourceUpdate& resource_update = resource_updates->back(); + resource_update.input_index = it->getSecond(); + resource_update.modified = true; + TF_RETURN_IF_ERROR(ConvertToDataType(tensor_type, &resource_update.type)); + TF_RETURN_IF_ERROR(XLAShapeToTensorShape(shape, &resource_update.shape)); + continue; + } + // Construct OutputDescription for result. + outputs->emplace_back(); + XlaOutputDescription& out_desc = outputs->back(); + TF_RETURN_IF_ERROR(ConvertToDataType(tensor_type, &out_desc.type)); + // TODO(ycao): Support constant output. + out_desc.is_constant = false; + TF_RETURN_IF_ERROR(XLAShapeToTensorShape(shape, &out_desc.shape)); + // Input_index is only meaningful for resource output. Setting it to + // meaningless value -1 for non resource outputs. + out_desc.input_index = + it != output_to_input_alias.end() ? it->getSecond() : -1; + // MLIR-based TF-Compiler bridge doesn't support tensorlist output yet. + // TODO(ycao): Support tensorlist-type output. + out_desc.is_tensor_list = false; + } + + // XLA computation always uses Tuple shape. + *xla_output_shape = xla::ShapeUtil::MakeTupleShape(shapes); + return absl::OkStatus(); +} + +// Creates a vector that maps from the parameters of the XLA computation to +// their original argument positions. +// MLIR-based TF-Compiler bridge doesn't have constant analysis yet, thus no +// inputs are known constants. Therefore, the input mapping between input to +// computation arguments is a trivial in-order 1-1 mapping. +// TODO(ycao): Support computation with compile-time constant, which requires +// non-trivial input mapping as implemented now. +void GetInputMappingForMlir(int num_inputs, std::vector* input_mapping) { + input_mapping->resize(num_inputs, 0); + std::iota(input_mapping->begin(), input_mapping->end(), 0); +} + +static void RegisterDialects(mlir::DialectRegistry& registry) { + mlir::RegisterAllTensorFlowDialects(registry); + xla::RegisterMlirToHloDependentDialects(registry); +} + +// Checks if functions can be inlined after TF -> HLO legalization. Currently +// TPU's are supported, to follow the behavior of inlining functions via the +// Graph based bridge in the TPUCompile op kernel. +bool CanInlineFunctionsPostLegalization(llvm::StringRef device_type) { + return device_type == DEVICE_TPU_XLA_JIT; +} + +// These passes are grouped together and must run in this specific order. +void AddLegalizationPasses(mlir::OpPassManager& pm, bool legalize_chlo, + llvm::StringRef device_type, bool enable_op_fallback, + bool lower_to_xla_hlo) { + if (lower_to_xla_hlo) { + // Lower TF quant ops and types to MHLO int. + mlir::quant::stablehlo::AddQuantizationLoweringPasses(pm); + + pm.addPass(mlir::mhlo::createLegalizeTFPass( + legalize_chlo, + /*tf2xla_fallback_device_type=*/device_type, enable_op_fallback)); + } + + // This has to run after legalization. + pm.addNestedPass( + mlir::mhlo::CreateInfeedsOpsXlaAdjustLayoutPass()); + + if (lower_to_xla_hlo) { + // This has to run after legalization to delete non legal but dead ops. + // This must run before Shape Inference. + pm.addNestedPass(mlir::createCanonicalizerPass()); + + // Run shape inference pass to propagate shapes through tensor_cast + // operations from static to dynamic shapes. This could be generated if the + // shape inference was originally missing in a TF op but the corresponding + // HLO op had static shape after lowering. This has to run after + // canonicalization. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + } +} + +// Builds a pipeline that runs only those passes required to lower TF to HLO. +void CreateMlirLoweringPassesPipeline(mlir::PassManager& pm) { + applyTensorflowAndCLOptions(pm); + + pm.addNestedPass(mlir::TF::CreateLowerQuantizedPass()); + pm.addNestedPass( + mlir::quant::stablehlo::CreateConvertTFQuantTypesPass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::mhlo::createStablehloLegalizeToHloPass()); + pm.addPass(mlir::mhlo::CreateLegalizeTFCollectivePass()); + mlir::quant::stablehlo::AddQuantizationLoweringPasses(pm); + // Legalize to HLO but force MLIR native only. + pm.addPass(mlir::mhlo::createLegalizeTFPass( + /*legalize_chlo=*/true, + /*tf2xla_fallback_device_type=*/std::nullopt, /*prefer_tf2xla=*/false)); + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addPass(mlir::mhlo::CreateLegalizeTFCommunicationPass()); + + auto pass_instrumentors = mlir::GetPassInstrumentors(); + for (const auto& creator : pass_instrumentors) { + pm.addInstrumentation(creator()); + } +} + +// If the module should be dumped then dumps the file and turns on the before +// and after IR printing for the passes. +void MaybeDumpMlirModuleAndPasses(mlir::PassManager& pm, + mlir::ModuleOp module_op, + std::string module_name, std::string tag) { + if (DEBUG_DATA_DUMPER()->ShouldDump(module_name, kDebugGroupMain) || + VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name, kDebugGroupMain, tag), + module_op, /*dirname=*/"", &pm); + } + + if (VLOG_IS_ON(2) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name, kDebugGroupBridgePhase2)) { + // Print the whole module after each pass which requires disabling + // multi-threading as well. + module_op.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique<::tensorflow::DataDumperLoggerConfig>( + [module_name](const std::string& pass_tag_name, mlir::Operation* op) { + return DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name, kDebugGroupBridgePhase2, pass_tag_name); + }, + /*pass_prefix=*/"", + /*print_module_scope=*/true)); + } +} + +// Runs the MLIR pipeline and if successful dumps the results if enabled. +absl::Status RunMlirPipelineAndMaybeDumpResults(mlir::PassManager& pm, + mlir::ModuleOp module_op, + std::string module_name = "") { + // Make sure we catch any error reported by MLIR and forward it to the TF + // error reporting system. Report a generic error if pass manager failed + // without emitting a diagnostic. + mlir::StatusScopedDiagnosticHandler error_handler(module_op.getContext()); + + if (failed(pm.run(module_op))) { + auto status = absl::InvalidArgumentError("TF to XLA legalization failed: "); + tensorflow::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_2, + status); + return error_handler.Combine(status); + } + + if (DEBUG_DATA_DUMPER()->ShouldDump(module_name, kDebugGroupMain) || + VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name, kDebugGroupMain, + "legalize_hlo_after"), + module_op, /*dirname=*/"", &pm); + } + + absl::Status status = error_handler.ConsumeStatus(); + tensorflow::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_2, + status); + return status; +} +} // namespace + +// Creates the MLIR Pipeline. +// If the temporary parameter lower_to_xla_hlo is +// true then the pipeline will include all the legalization passes. +void CreateConvertMlirToXlaHloPipeline( + mlir::OpPassManager& pm, llvm::StringRef device_type, + bool enable_op_fallback, + llvm::MutableArrayRef> + custom_legalization_passes, + bool lower_to_xla_hlo, bool allow_partial_conversion) { + bool legalize_chlo = true; + + pm.addNestedPass( + tf2xla::internal::CreateInputLoweringMetricsPass()); + + pm.addNestedPass( + mlir::mhlo::CreateTFXLADeviceSpecificTransformsPass(device_type)); + + // Note that the region-based control-flow produced here still contains + // function call ops which get inlined by the subsequent inliner pass. + pm.addPass(mlir::TF::CreateTFFunctionalControlFlowToRegions()); + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass( + mlir::TF::CreateDropWhileShapeInvariantPass()); + // Create a replicated TensorList initialization ops for all of its uses. This + // pass undo some CSE because shape_inference is not correctly able to + // identify the shapes of TensorList initialization ops. + // This pass requires CanonicalizerPass before + // CreateTensorListOpsDecompositionPass for clean-ups. + if (lower_to_xla_hlo) { + pm.addNestedPass( + mlir::TF::CreateReplicateTensorListInitOpsPass()); + } + pm.addNestedPass(mlir::createCanonicalizerPass()); + // The SCCP pass performs constant propagation across the IR, which, for + // example, propagates constant arguments into callee functions. + // TODO(hinsu): Investigate if we really need SCCP pass before shape inference + // and can do with just one pass after the shape inference. + pm.addPass(mlir::createSCCPPass()); + // Guarantee all functions have one use, which enables shape inference. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + // Run shape inference pass before tensorlist decomposition to get buffer + // shape of uninitialized TensorLists. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + + // Run SCCP pass again as the availability of shapes may open up new + // opportunities for constant propagation. Note that the shape inference pass + // doesn't materialize new constants even if those are computed internally for + // the purpose of shape inference. These constants might be required by the + // legalization passes. + pm.addPass(mlir::createSCCPPass()); + + if (lower_to_xla_hlo) { + pm.addPass(mlir::TF::CreateTensorListOpsDecompositionPass()); + } + pm.addPass(mlir::TF::CreateStackOpsDecompositionPass()); + if (lower_to_xla_hlo) { + pm.addPass(mlir::TF::CreateTensorArrayOpsDecompositionPass()); + } + pm.addNestedPass( + mlir::TFDevice::CreateDecomposeResourceOpsPass()); + pm.addPass(mlir::TF::CreatePromoteResourcesToArgsPass()); + pm.addPass(mlir::createSymbolDCEPass()); + + // Sink constants to regions so that ops requiring constant operands can + // access the constant and there is no indirection through control flow region + // arguments. Also, note that this pass is in MHLO but it is generic and sinks + // constants for all ops with regions. + pm.addNestedPass( + mlir::mhlo::createSinkConstantsToControlFlowPass()); + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + + if (lower_to_xla_hlo) { + // Legalize any StableHLO ops to MHLO. Bridge still doesn't use StableHLO + // but such ops might be present in the input from upstream like TFRT + // compilation. Later on, this could be merged in the legalization pass when + // we migrate bridge to StableHLO. + + // TODO(b/259459405): Avoid this peculiar use through some refactoring in + // the caller. + // This needs to happen before legalization. + pm.addPass(mlir::mhlo::createStablehloLegalizeToHloPass()); + } + + pm.addNestedPass(mlir::TF::CreateLowerQuantizedPass()); + pm.addNestedPass( + mlir::quant::stablehlo::CreateConvertTFQuantTypesPass()); + + if (lower_to_xla_hlo) { + for (auto& target_pass : custom_legalization_passes) { + pm.addNestedPass(std::move(target_pass)); + } + pm.addPass(mlir::mhlo::CreateLegalizeTFCollectivePass()); + } + + // These passes are grouped together as they have to run in specific order. + // Passes before this can run relatively in any order, as long as they happen + // before legalization. + AddLegalizationPasses(pm, legalize_chlo, device_type, enable_op_fallback, + lower_to_xla_hlo); + + if (lower_to_xla_hlo) { + // This pass operates on MHLO control flow ops so it should be legalized + // after the control flow ops are legalized. + pm.addPass(mlir::mhlo::CreateLegalizeTFCommunicationPass()); + + // Everything should be MHLO after this. + if (!allow_partial_conversion) { + pm.addNestedPass( + mlir::hlo::CreateVerifyTFXLALegalizationPass(legalize_chlo)); + } + } + + if (CanInlineFunctionsPostLegalization(device_type)) { + pm.addPass(mlir::createInlinerPass()); + } + + // In order to export to XLA, we must sink constants to control flow regions, + // since XLA uses functional control flow. + pm.addNestedPass( + mlir::stablehlo_ext::createSinkConstantsToControlFlowPass()); +} + +absl::Status RefineShapes(llvm::ArrayRef arg_shapes, + mlir::ModuleOp module) { + auto producer_or = GetTfGraphProducerVersion(module); + if (!producer_or.ok()) return producer_or.status(); + int64_t producer_version = producer_or.value(); + + llvm::SmallVector shape_backing; + llvm::SmallVector, 4> arg_shapes_copy; + { + // Convert arg_shapes to a mlir friendly format. + size_t count = 0; + for (const TensorOrResourceShape& tensor_resource_shape : arg_shapes) { + if (tensor_resource_shape.is_resource) continue; + count += tensor_resource_shape.shape.dims(); + } + shape_backing.resize(count); + arg_shapes_copy.reserve(arg_shapes.size()); + size_t offset = 0; + for (const TensorOrResourceShape& tensor_resource_shape : arg_shapes) { + if (tensor_resource_shape.is_resource) { + arg_shapes_copy.push_back(llvm::ArrayRef()); + continue; + } + size_t start = offset; + for (tensorflow::TensorShapeDim dim : tensor_resource_shape.shape) { + shape_backing[offset] = dim.size; + ++offset; + } + if (offset == start) { + arg_shapes_copy.push_back(llvm::ArrayRef()); + } else { + arg_shapes_copy.push_back( + llvm::ArrayRef(&shape_backing[start], offset - start)); + } + } + } + + auto main_func = module.lookupSymbol("main"); + + mlir::StatusScopedDiagnosticHandler error_handler(module.getContext()); + mlir::LogicalResult result = mlir::TF::InferShapeForFunction( + main_func, arg_shapes_copy, producer_version); + + if (failed(result)) { + return error_handler.Combine( + absl::InternalError("MLIR Shape refinement failed")); + } + return error_handler.ConsumeStatus(); +} + +absl::Status CreateAndRunMlirBridge( + mlir::ModuleOp module_op, llvm::StringRef device_type, + bool enable_op_fallback, + llvm::MutableArrayRef> + custom_legalization_passes, + bool lower_to_xla_hlo, llvm::StringRef module_name = llvm::StringRef()) { + mlir::PassManager tf2xla(module_op.getContext()); + applyTensorflowAndCLOptions(tf2xla); + CreateConvertMlirToXlaHloPipeline(tf2xla, device_type, enable_op_fallback, + custom_legalization_passes, + lower_to_xla_hlo, + /*allow_partial_conversion=*/false); + + auto pass_instrumentors = mlir::GetPassInstrumentors(); + for (const auto& creator : pass_instrumentors) { + tf2xla.addInstrumentation(creator()); + } + MaybeDumpMlirModuleAndPasses(tf2xla, module_op, module_name.str(), + /*tag=*/"legalize_hlo_before"); + + return RunMlirPipelineAndMaybeDumpResults(tf2xla, module_op, + module_name.str()); +} + +absl::Status BuildHloFromTfInner( + mlir::ModuleOp module_op, xla::XlaBuilder& builder, + llvm::ArrayRef xla_params, std::vector& returns, + llvm::StringRef device_type, + llvm::MutableArrayRef> + custom_legalization_passes) { + TF_RETURN_IF_ERROR(CreateAndRunMlirBridge(module_op, device_type, + /*enable_op_fallback=*/false, + custom_legalization_passes, + /*lower_to_xla_hlo=*/true)); + + return mlir::BuildHloFromMlirHlo(module_op, builder, xla_params, returns); +} + +absl::Status ConvertMLIRWithOptionalXlaComputation( + mlir::ModuleOp module_op, llvm::StringRef device_type, + xla::XlaComputation* xla_computation, bool use_tuple_args, + bool enable_op_fallback, bool return_tuple, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name, bool lower_to_xla_hlo) { + TF_RETURN_IF_ERROR(CreateAndRunMlirBridge( + module_op, device_type, enable_op_fallback, custom_legalization_passes, + lower_to_xla_hlo, module_name)); + + mlir::MlirToHloConversionOptions options; + options.layout_preference_fn = [&](const xla::Shape& xla_shape) + -> absl::StatusOr { + TensorShape shape; + TF_RETURN_IF_ERROR(XLAShapeToTensorShape(xla_shape, &shape)); + TF_ASSIGN_OR_RETURN(DataType dtype, EncodePrimitiveTypeAsDataType( + xla_shape.element_type())); + return shape_determination_fns.layout_preference_fn(shape, dtype, + std::nullopt); + }; + options.shape_representation_fn = + [&](const xla::Shape& xla_shape, bool fast_mem, + mlir::XlaLayoutPreference layout_preference) + -> absl::StatusOr { + TensorShape shape; + TF_RETURN_IF_ERROR(XLAShapeToTensorShape(xla_shape, &shape)); + TF_ASSIGN_OR_RETURN(DataType dtype, EncodePrimitiveTypeAsDataType( + xla_shape.element_type())); + return shape_determination_fns.shape_representation_fn( + shape, dtype, fast_mem, layout_preference); + }; + xla::HloProto hlo_proto; + + if (lower_to_xla_hlo) { + TF_RETURN_IF_ERROR(mlir::ConvertMlirHloToHlo( + module_op, &hlo_proto, use_tuple_args, return_tuple, options)); + *xla_computation = xla::XlaComputation(hlo_proto.hlo_module()); + } + return absl::OkStatus(); +} + +// Wraps the optional lowering version to keep the api the same for clients. +absl::Status ConvertMLIRToXlaComputation( + mlir::ModuleOp module_op, llvm::StringRef device_type, + xla::XlaComputation* xla_computation, bool use_tuple_args, + bool enable_op_fallback, bool return_tuple, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name) { + return ConvertMLIRWithOptionalXlaComputation( + module_op, device_type, xla_computation, use_tuple_args, + enable_op_fallback, return_tuple, shape_determination_fns, + custom_legalization_passes, module_name, /*lower_to_xla_hlo=*/true); +} + +absl::Status CompileMlirSetup( + mlir::ModuleOp module_op, + llvm::ArrayRef arg_shapes) { + // Use arg_shapes to improve the mlir type information of `main` in module_op. + TF_RETURN_IF_ERROR(RefineShapes(arg_shapes, module_op)); + + if (VLOG_IS_ON(2)) + tensorflow::DumpMlirOpToFile("compile_mlir_shape_refiner", module_op); + + return absl::OkStatus(); +} + +absl::Status BuildHloFromTf(mlir::ModuleOp module_op, xla::XlaBuilder& builder, + llvm::ArrayRef xla_params, + std::vector& returns, + llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, + llvm::MutableArrayRef> + custom_legalization_passes) { + if (VLOG_IS_ON(2)) + tensorflow::DumpMlirOpToFile("build_hlo_tf_before", module_op); + + TF_RETURN_IF_ERROR(CompileMlirSetup(module_op, arg_shapes)); + + // Convert MLIR module to XLA HLO proto contained in XlaComputation. + TF_RETURN_IF_ERROR(BuildHloFromTfInner(module_op, builder, xla_params, + returns, device_type, + custom_legalization_passes)); + + if (VLOG_IS_ON(2)) + tensorflow::DumpMlirOpToFile("build_hlo_tf_after", module_op); + + return absl::OkStatus(); +} + +absl::Status PopulateCollectiveInfo(mlir::ModuleOp module_op, + XlaCompilationResult* compilation_result) { + // The StringRef cast is necessary before cxx14. + mlir::IntegerAttr group_key_attr = + module_op->getAttrOfType( + mlir::StringRef(kGroupKeyAttrName.data(), kGroupKeyAttrName.size())); + mlir::IntegerAttr group_size_attr = + module_op->getAttrOfType(mlir::StringRef( + kGroupSizeAttrName.data(), kGroupSizeAttrName.size())); + if (group_key_attr == nullptr && group_size_attr == nullptr) { + // No CollectiveInfo is present. + return absl::OkStatus(); + } + DCHECK(group_key_attr != nullptr) + << "module attribute " << kGroupKeyAttrName + << " is required for CollectiveInfo but not found."; + DCHECK(group_size_attr != nullptr) + << "module attribute " << kGroupSizeAttrName + << " is required for CollectiveInfo but not found."; + int32_t group_key = group_key_attr.getInt(); + int32_t group_size = group_size_attr.getInt(); + VLOG(2) << "Populating CollectiveInfo: group_key=" << group_key + << " group_size=" << group_size; + compilation_result->collective_info = {group_key, group_size, 0}; + return absl::OkStatus(); +} + +absl::Status PopulateResultIOInfo( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + bool use_tuple_args, bool use_resource_updates_for_aliases, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result) { + // Construct mapping from XlaComputation's arg to input edges of execute + // node. + GetInputMappingForMlir(arg_shapes.size(), &compilation_result->input_mapping); + + // Compute all input shapes. + TF_RETURN_IF_ERROR(GetXlaInputShapes(module_op, arg_shapes, use_tuple_args, + shape_determination_fns, + &compilation_result->xla_input_shapes)); + + // Compute all output descriptions and resource writes + return GetOutputInfo( + module_op, use_resource_updates_for_aliases, shape_determination_fns, + &compilation_result->xla_output_shape, &compilation_result->outputs, + &compilation_result->resource_updates); +} + +absl::Status CompileMlirToXlaHlo( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name, bool lower_to_xla_hlo) { + // If we don't want to enable fall back or any form or lowering then we + // don't need to check if the module has unsupported ops. + if (enable_op_fallback && lower_to_xla_hlo && + GetMlirBridge2ndPhaseRolloutPolicy(module_op) == + MlirBridgeRolloutPolicy::kDisabledAfterGraphAnalysis) { + return CompileToHloGraphAnalysisFailedError(); + } + + TF_RETURN_IF_ERROR(CompileMlirSetup(module_op, arg_shapes)); + + // Convert MLIR module to XLA HLO proto contained in XlaComputation. + compilation_result->computation = std::make_shared(); + TF_RETURN_IF_ERROR(ConvertMLIRWithOptionalXlaComputation( + module_op, device_type, compilation_result->computation.get(), + use_tuple_args, enable_op_fallback, use_return_tuple, + shape_determination_fns, custom_legalization_passes, module_name, + lower_to_xla_hlo)); + + if (failed(mlir::verify(module_op))) { + return absl::AbortedError( + "Verification failure. MLIR to Xla conversion aborted."); + } + + // Only attempt to fill in the compilation result's IO info if lowering. + if (lower_to_xla_hlo) { + TF_RETURN_IF_ERROR(PopulateCollectiveInfo(module_op, compilation_result)); + + auto populate_result = PopulateResultIOInfo( + module_op, arg_shapes, use_tuple_args, use_resource_updates_for_aliases, + shape_determination_fns, compilation_result); + if (!populate_result.ok()) { + llvm::errs() << "Failed to populate result io info"; + return populate_result; + } + } + + return absl::OkStatus(); +} + +absl::StatusOr CompileMlirToXlaHloAndSerialize( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name, bool lower_to_xla_hlo) { + TF_RETURN_IF_ERROR(CompileMlirToXlaHlo( + module_op, arg_shapes, device_type, use_tuple_args, enable_op_fallback, + use_return_tuple, use_resource_updates_for_aliases, + shape_determination_fns, compilation_result, custom_legalization_passes, + module_name, lower_to_xla_hlo)); + + return SerializeMlirModule(module_op); +} + +absl::StatusOr CompileSerializedMlirToXlaHlo( + llvm::StringRef mlir_module_string, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + const XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name, bool lower_to_xla_hlo) { + mlir::DialectRegistry mlir_registry; + RegisterDialects(mlir_registry); + mlir::MLIRContext mlir_context(mlir_registry); + mlir::OwningOpRef mlir_module; + + TF_RETURN_IF_ERROR( + DeserializeMlirModule(mlir_module_string, &mlir_context, &mlir_module)); + llvm::SmallVector tensor_or_resource_shapes; + tensor_or_resource_shapes.reserve(arg_shapes.size()); + for (const auto& arg_shape : arg_shapes) + tensor_or_resource_shapes.push_back({arg_shape}); + return CompileMlirToXlaHloAndSerialize( + mlir_module.get(), tensor_or_resource_shapes, device_type, use_tuple_args, + enable_op_fallback, /*use_return_tuple=*/true, + /*use_resource_updates_for_aliases=*/false, shape_determination_fns, + compilation_result, custom_legalization_passes, module_name, + lower_to_xla_hlo); +} + +// Rewrites the given module with specified args. Each of the constant args gets +// inlined in the "main' function and the corresponding argument is removed from +// the signature. For resource args, their subtypes are populated. Returns the +// original indices for the other arguments on success. +static absl::StatusOr> RewriteWithArgs( + mlir::ModuleOp module_op, llvm::ArrayRef args) { + mlir::func::FuncOp main_fn = + module_op.lookupSymbol("main"); + std::vector params; + + bool has_resource_args = false; + auto builder = mlir::OpBuilder(main_fn.getBody()); + std::vector args_to_erase; + for (int idx = 0; idx < args.size(); idx++) { + const XlaArgument& xla_arg = args[idx]; + mlir::BlockArgument mlir_arg = main_fn.getArgument(idx); + if (xla_arg.kind == XlaArgument::kResource) { + mlir::Type element_type; + if (xla_arg.type == DT_INVALID) { + return absl::UnimplementedError(absl::StrCat( + "Argument ", idx, + " is an uninitialized resource variable which is currently" + " unsupported in the MLIR-based TPU bridge")); + } + TF_RETURN_IF_ERROR(ConvertDataType(xla_arg.type, builder, &element_type)); + TF_ASSIGN_OR_RETURN(TensorShape arg_shape, + GetTensorShapeFromXlaArgument(xla_arg)); + auto resource_shape = arg_shape.dim_sizes(); + llvm::SmallVector resource_subtype_shape( + resource_shape.begin(), resource_shape.end()); + auto resource_subtype = + GetTypeFromTFTensorShape(resource_subtype_shape, element_type); + auto resource_type = + mlir::TF::ResourceType::get({resource_subtype}, builder.getContext()); + + auto tensor_type = mlir::cast(mlir_arg.getType()); + if (tensor_type.hasRank()) { + mlir_arg.setType( + GetTypeFromTFTensorShape(tensor_type.getShape(), resource_type)); + } else { + mlir_arg.setType(mlir::UnrankedTensorType::get(resource_type)); + } + has_resource_args = true; + } + if (xla_arg.kind != XlaArgument::kConstant) { + params.push_back(idx); + continue; + } + + TF_ASSIGN_OR_RETURN(auto value_attr, + ConvertTensor(xla_arg.constant_value, &builder)); + // TODO(hinsu): Use the actual location of the constant. + auto constant = mlir::TF::ConstOp::create( + builder, mlir::UnknownLoc::get(module_op.getContext()), value_attr); + mlir_arg.replaceAllUsesWith(constant); + args_to_erase.push_back(idx); + } + + if (has_resource_args) { + llvm::SmallVector updated_argument_types; + updated_argument_types.reserve(main_fn.getNumArguments()); + for (mlir::BlockArgument& arg : main_fn.getArguments()) + updated_argument_types.push_back(arg.getType()); + + main_fn.setType( + mlir::FunctionType::get(main_fn.getContext(), updated_argument_types, + main_fn.getFunctionType().getResults())); + } + + for (int idx : llvm::reverse(args_to_erase)) { + CHECK(llvm::succeeded(main_fn.eraseArgument(idx))); + } + + return params; +} + +absl::Status CompileGraphSetup( + mlir::ModuleOp module_op, llvm::ArrayRef args, + std::vector* remaining_params, + llvm::SmallVector& arg_shapes) { + TF_ASSIGN_OR_RETURN(*remaining_params, RewriteWithArgs(module_op, args)); + arg_shapes.reserve(remaining_params->size()); + for (unsigned idx : *remaining_params) { + const auto& arg = args[idx]; + TF_ASSIGN_OR_RETURN(TensorShape arg_shape, + GetTensorShapeFromXlaArgument(arg)); + arg_shapes.push_back({arg_shape, + /*is_resource=*/arg.kind == XlaArgument::kResource}); + } + + mlir::PassManager pm(module_op.getContext()); + applyTensorflowAndCLOptions(pm); + mlir::TF::StandardPipelineOptions tf_options; + mlir::TF::CreateTFStandardPipeline(pm, tf_options); + + if (VLOG_IS_ON(1)) + tensorflow::DumpMlirOpToFile("compile_graph_setup_before", module_op); + if (VLOG_IS_ON(2)) { + // Print the whole module after each pass which requires disabling + // multi-threading as well. + module_op.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique( + /*print_module_scope=*/true)); + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module_op.getContext()); + if (failed(pm.run(module_op))) return diag_handler.ConsumeStatus(); + if (VLOG_IS_ON(1)) + tensorflow::DumpMlirOpToFile("compile_graph_setup_after", module_op); + + return absl::OkStatus(); +} + +absl::Status BuildHloFromModule(mlir::ModuleOp module_op, + xla::XlaBuilder& builder, + llvm::ArrayRef xla_params, + std::vector& returns, + llvm::ArrayRef args, + llvm::StringRef device_type) { + std::vector remaining_params; + llvm::SmallVector arg_shapes; + TF_RETURN_IF_ERROR( + CompileGraphSetup(module_op, args, &remaining_params, arg_shapes)); + llvm::SmallVector remaining_xla_params; + for (auto i : remaining_params) remaining_xla_params.push_back(xla_params[i]); + TF_RETURN_IF_ERROR(CompileMlirSetup(module_op, arg_shapes)); + + mlir::PassManager tf2xla(module_op.getContext()); + CreateMlirLoweringPassesPipeline(tf2xla); + + MaybeDumpMlirModuleAndPasses(tf2xla, module_op, /*module_name=*/"", + /*tag=*/"legalize_hlo_before"); + + TF_RETURN_IF_ERROR(RunMlirPipelineAndMaybeDumpResults(tf2xla, module_op)); + + TF_RETURN_IF_ERROR(mlir::BuildHloFromMlirHlo(module_op, builder, + remaining_xla_params, returns)); + + if (VLOG_IS_ON(2)) { + tensorflow::DumpMlirOpToFile("build_hlo_tf_after", module_op); + } + + return absl::OkStatus(); +} + +absl::Status CompileGraphToXlaHlo( + mlir::ModuleOp module_op, llvm::ArrayRef args, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes) { + std::vector remaining_params; + llvm::SmallVector arg_shapes; + TF_RETURN_IF_ERROR( + CompileGraphSetup(module_op, args, &remaining_params, arg_shapes)); + + auto compile_mlir_result = CompileMlirToXlaHlo( + module_op, arg_shapes, device_type, use_tuple_args, enable_op_fallback, + use_return_tuple, + /*use_resource_updates_for_aliases=*/true, shape_determination_fns, + compilation_result, custom_legalization_passes); + compilation_result->input_mapping = remaining_params; + return compile_mlir_result; +} + +absl::StatusOr> GraphToModule( + bool unconditionally_use_set_output_shapes, const Graph& graph, + llvm::ArrayRef control_rets, + const FunctionLibraryDefinition& flib_def, mlir::MLIRContext* context) { + mlir::DialectRegistry registry; + RegisterDialects(registry); + context->appendDialectRegistry(registry); + GraphImportConfig config; + config.graph_as_function = true; + config.control_outputs = control_rets; + // Disable shape inference during import as some TensorFlow op fails during + // shape inference with dynamic shaped operands. This in turn causes the + // import to fail. Shape inference during import is going to be removed and + // the shape inference pass is run early in the pass pipeline, shape inference + // during import is not necessary. + config.enable_shape_inference = false; + // Some graphs may require _output_shapes (an unregistered attribute) + // to override shapes. It is unfortunately not always set correctly so only + // do it optionally. + config.unconditionally_use_set_output_shapes = + unconditionally_use_set_output_shapes; + GraphDebugInfo debug_info; + return tensorflow::tf2xla::v2::ConvertGraphToTfExecutor( + graph, debug_info, flib_def, config, context); +} + +absl::Status BuildHloFromGraph( + const Graph& graph, xla::XlaBuilder& builder, + mlir::MLIRContext& mlir_context, llvm::ArrayRef xla_params, + std::vector& returns, bool unconditionally_use_output_shapes, + llvm::ArrayRef args, llvm::ArrayRef control_rets, + llvm::StringRef device_type, const FunctionLibraryDefinition& flib_def) { + TF_ASSIGN_OR_RETURN(mlir::OwningOpRef module, + GraphToModule(unconditionally_use_output_shapes, graph, + control_rets, flib_def, &mlir_context)); + return BuildHloFromModule(module.get(), builder, xla_params, returns, args, + device_type); +} + +void RegisterConvertMlirToXlaHloPipelineWithDefaults() { + static mlir::PassPipelineRegistration<> pipeline( + "tf-to-hlo-pipeline", + "Convert TF dialect to HLO dialect (used for compilation in bridge).", + [](mlir::OpPassManager& pm) { + tensorflow::CreateConvertMlirToXlaHloPipeline( + pm, /*device_type=*/"XLA_CPU_JIT", /*enable_op_fallback=*/false, + /*custom_legalization_passes=*/{}); + }); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h new file mode 100644 index 0000000000000000000000000000000000000000..2345b270276b6cd1df65085f7925f8953139a396 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h @@ -0,0 +1,242 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_MLIR_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_MLIR_UTIL_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_argument.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/hlo/builder/xla_computation.h" +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/framework/tensor_shape.h" + +namespace tensorflow { + +// Lowers MLIR module to XLA HLO inside an XlaComputation. The input module +// should only contain operations in tf dialect. If the input module contains +// operation in the tf_executor dialect, for example, returns an error. +// Exception to this are tf_executor dialect ops that are optimized away through +// canonicalization. +// +// Operations in tf dialect are lowered to XLA HLO through the following steps: +// . Legalizes control flow operations. +// . Decomposes compound resource operations so that the only remaining +// operations on resource variables are resource reads/writes.. +// . Replaces resource reads/writes with function inputs/outputs and +// eliminates the use of resource variables. +// . Legalizes the operations to XLA HLO operations. +// . Canonicalizes the XLA HLO operations. +// +// device_type: XLA JIT device to use for compilation such as "XLA_CPU_JIT", +// "XLA_GPU_JIT" or "XLA_TPU_JIT". +// use_tuple_args: when this is true, always create a tuple argument for the +// entry computation. +// enable_op_fallback: when this is true, prefer tf2xla fallback kernels over +// MLIR +// native kernels for legalization to HLO. +// return_tuple: when this is true, always create a tuple result for the +// entry computation. +// shape_determination_fns: Contains layout preference fn and shape +// representation fn. The two functions are used to determine argument and +// result shapes. +// custom_legalization_passes: passes to run before the default TF legalization +// passes for backend-specific ops. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::Status ConvertMLIRToXlaComputation( + mlir::ModuleOp module_op, llvm::StringRef device_type, + xla::XlaComputation* xla_computation, bool use_tuple_args, + bool enable_op_fallback, bool return_tuple, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns = {}, + llvm::MutableArrayRef> + custom_legalization_passes = {}, + llvm::StringRef module_name = llvm::StringRef()); + +// Creates a MLIR pipeline that lowers MLIR module to MHLO dialect. The input +// module should only contain operations in tf dialect. For example, if the +// input module contains operation in the tf_executor dialect, the pass raises +// an error unless the tf_executor dialect ops are optimized away by +// canonicalization. +// +// The pipeline is used in ConvertMLIRToXlaComputation. And it generally has the +// following pass structure: +// - TensorFlow passes +// - Legalization passes +// - MHLO passes +// +// device_type: XLA JIT device to use for compilation such as "XLA_CPU_JIT", +// "XLA_GPU_JIT" or "XLA_TPU_JIT". +// enable_op_fallback: when this is true, prefer tf2xla fallback kernels over +// MLIR +// native kernels for legalization to HLO. +// custom_legalization_passes: passes to run before the default TF legalization +// passes for backend-specific ops. +// lower_to_xla_hlo: Temporary parameter to be removed in imminent update. If +// true, includes legalization and MHLO lowering passes. +// allow_partial_conversion: when this is true, allow operations that can't be +// legalized. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +void CreateConvertMlirToXlaHloPipeline( + mlir::OpPassManager& pm, llvm::StringRef device_type, + bool enable_op_fallback, + llvm::MutableArrayRef> + custom_legalization_passes, + bool lower_to_xla_hlo = true, bool allow_partial_conversion = false); + +// Helper struct representing argument tensor or resource handle shapes. +struct TensorOrResourceShape { + TensorShape shape; + bool is_resource = false; +}; + +// Refine MLIR types based on new shape information. +ABSL_DEPRECATED("Not meant to be used directly and should be a util.") +absl::Status RefineShapes(llvm::ArrayRef arg_shapes, + mlir::ModuleOp module); + +// Lower TF to MHLO and insert HLO into the XlaBuilder. xla_params are HLO-level +// inputs to module_op that have already been added to the XlaBuilder. returns +// are the returned XlaOps. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::Status BuildHloFromTf(mlir::ModuleOp module_op, xla::XlaBuilder& builder, + llvm::ArrayRef xla_params, + std::vector& returns, + llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, + llvm::MutableArrayRef> + custom_legalization_passes); + +// Apply shape, description, and resource information to inputs and outputs +// in the XlaCompilationResult. This should be called after +// compilation_result->computation was set. +ABSL_DEPRECATED("Not meant to be used directly and should be a util.") +absl::Status PopulateResultIOInfo( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + bool use_tuple_args, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result); + +// Runs MLIR Bridge on an MLIR module. +// +// If lower_to_xla_hlo is true then compiles down into XLA HLO, generates all +// accompanying metadata and stores them in CompilationResult. +// +// If enable_op_fallback is set to false, graph is legalized only if the graph +// analysis for the graph is successful. Otherwise, an error is returned. +// +// Running the MLIR Bridge performs many transformations on the input module +// which is modified in place. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::Status CompileMlirToXlaHlo( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name = llvm::StringRef(), + bool lower_to_xla_hlo = true); + +// Runs MLIR Bridge on a MLIR module. +// +// If lower_to_xla_hlo is true then compiles down into XLA HLO, generates all +// accompanying metadata and stores them in CompilationResult. +// +// If enable_op_fallback is set to false, graph is legalized only if the graph +// analysis for the graph is successful. Otherwise, an error is returned. +// +// On success, returns the serialized MLIR module. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::StatusOr CompileMlirToXlaHloAndSerialize( + mlir::ModuleOp module_op, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, bool use_resource_updates_for_aliases, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes, + llvm::StringRef module_name = llvm::StringRef(), + bool lower_to_xla_hlo = true); + +// Runs MLIR Bridge on a serialized MLIR module. +// +// If lower_to_xla_hlo is true then compiles down into XLA HLO, generates all +// accompanying metadata and stores them in CompilationResult. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::StatusOr CompileSerializedMlirToXlaHlo( + llvm::StringRef mlir_module_string, llvm::ArrayRef arg_shapes, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes = {}, + llvm::StringRef module_name = llvm::StringRef(), + bool lower_to_xla_hlo = true); + +// Compiles a TensorFlow Graph (already converted to MLIR, imported with +// tf_executor dialect still present) into XLA HLO, generates all accompanying +// metadata and stores them in CompilationResult. This will rewrite arguments +// and run the TensorFlow standard pipeline prior to invoking +// `CompileMlirToXlaHlo`. +ABSL_DEPRECATED("Use v2/legalize_tf.h::LegalizeMlirToHlo instead.") +absl::Status CompileGraphToXlaHlo( + mlir::ModuleOp module_op, llvm::ArrayRef args, + llvm::StringRef device_type, bool use_tuple_args, bool enable_op_fallback, + bool use_return_tuple, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + XlaCompilationResult* compilation_result, + llvm::MutableArrayRef> + custom_legalization_passes); + +// Compiles a Graph from TF to HLO and adds the resulting HLO to the +// XlaBuilder. This function adds HLO to a larger HLO computation, so +// HLO-level inputs are supplied, and HLO-level outputs are produced. +// xla_params is the HLO-level inputs and returns is the HLO-level outputs. +// If unconditionally_use_output_shapes is true then the unregistered +// attribute _output_shapes is always used to set the output shapes of the ops. +ABSL_DEPRECATED( + "Use v1/compile_tf_graph.h::CompileTensorflowGraphToHlo instead.") +absl::Status BuildHloFromGraph( + const Graph& graph, xla::XlaBuilder& builder, + mlir::MLIRContext& mlir_context, llvm::ArrayRef xla_params, + std::vector& returns, bool unconditionally_use_output_shapes, + llvm::ArrayRef args, llvm::ArrayRef control_rets, + llvm::StringRef device_type, const FunctionLibraryDefinition& flib_def); + +static inline absl::Status CompileToHloGraphAnalysisFailedError() { + return absl::InternalError("disabled after graph analysis"); +} + +// Register a convenient pipeline for invoking TF/XLA lowering from the command +// line. +void RegisterConvertMlirToXlaHloPipelineWithDefaults(); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_MLIR_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..7257aa91411651b02b5d9e0800b96baa529b38c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util_test.cc @@ -0,0 +1,316 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" + +#include +#include +#include +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/jit/xla_compile_util.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/hlo/builder/xla_builder.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace { + +using ::mlir::OpPassManager; +using ::tensorflow::monitoring::testing::CellReader; +using ::testing::HasSubstr; + +static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + +TEST(LegalizeMlirTest, LegalizesModule) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + + std::vector arg_shapes; + XlaCompilationResult compilation_result; + auto status = CompileSerializedMlirToXlaHlo( + kMlirModuleStr, arg_shapes, /*device_type=*/"XLA_TPU_JIT", + /*use_tuple_args=*/true, /*enable_op_fallback=*/false, + /*shape_determination_fns=*/{}, &compilation_result); + + EXPECT_TRUE(status.ok()); + EXPECT_THAT(status.value(), HasSubstr("stablehlo.constant")); +} + +TEST(LegalizeMlirTest, FailsLegalizesModule) { + constexpr char failed_legalization[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.DoesntExist"() : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + CellReader count( + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_pass_count"); + + std::vector arg_shapes; + XlaCompilationResult compilation_result; + auto status = CompileSerializedMlirToXlaHlo( + failed_legalization, arg_shapes, /*device_type=*/"XLA_TPU_JIT", + /*use_tuple_args=*/true, /*enable_op_fallback=*/false, + /*shape_determination_fns=*/{}, &compilation_result); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(count.Delta("tf.DoesntExist", "Unknown"), 1); +} + +TEST(CompileMlirUtil, CreatesPipeline) { + OpPassManager pass_manager; + llvm::StringRef device_type = "XLA_CPU_JIT"; + + CreateConvertMlirToXlaHloPipeline(pass_manager, device_type, + /*enable_op_fallback=*/false, + /*custom_legalization_passes*/ {}); + + EXPECT_FALSE(pass_manager.getPasses().empty()); +} + +TEST(CompileMlirUtil, HasLegalizationPass) { + OpPassManager pass_manager; + llvm::StringRef device_type = "XLA_CPU_JIT"; + absl::string_view kLegalizeTfPass = "xla-legalize-tf"; + + CreateConvertMlirToXlaHloPipeline(pass_manager, device_type, + /*enable_op_fallback=*/true, + /*custom_legalization_passes*/ {}); + + std::string pass_description; + llvm::raw_string_ostream raw_stream(pass_description); + pass_manager.printAsTextualPipeline(raw_stream); + + EXPECT_THAT(pass_description, HasSubstr(kLegalizeTfPass)); +} + +TEST(CompileMlirUtil, DoesNotHaveLegalizationPass) { + OpPassManager pass_manager; + llvm::StringRef device_type = "XLA_CPU_JIT"; + absl::string_view kLegalizeTfPass = "xla-legalize-tf"; + + CreateConvertMlirToXlaHloPipeline(pass_manager, device_type, + /*enable_op_fallback=*/false, + /*custom_legalization_passes*/ {}, + /*lower_to_xla_hlo=*/false); + + std::string pass_description; + llvm::raw_string_ostream raw_stream(pass_description); + pass_manager.printAsTextualPipeline(raw_stream); + + EXPECT_THAT(pass_description, Not(HasSubstr(kLegalizeTfPass))); +} + +TEST(CompileMlirUtil, DoesNotLowerWhenTold) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + + std::vector arg_shapes; + XlaCompilationResult compilation_result; + auto status = CompileSerializedMlirToXlaHlo( + kMlirModuleStr, arg_shapes, /*device_type=*/"XLA_TPU_JIT", + /*use_tuple_args=*/true, /*enable_op_fallback=*/false, + /*shape_determination_fns=*/{}, &compilation_result, + /*custom_legalization_passes=*/{}, + /*module_name=*/"", + /*lower_to_xla_hlo=*/false); + + EXPECT_TRUE(status.ok()); + EXPECT_THAT(status.value(), HasSubstr("tf.Const")); +} + +TEST(CompileMlirUtil, CanonicalizationIsExplicitDuringInlining) { + OpPassManager pass_manager; + llvm::StringRef device_type = "XLA_CPU_JIT"; + absl::string_view kInlinePass = + "inline{default-pipeline=canonicalize " + "inlining-threshold=4294967295 max-iterations=4 }"; + + CreateConvertMlirToXlaHloPipeline(pass_manager, device_type, + /*enable_op_fallback=*/true, + /*custom_legalization_passes*/ {}); + + std::string pass_description; + llvm::raw_string_ostream raw_stream(pass_description); + pass_manager.printAsTextualPipeline(raw_stream); + + EXPECT_THAT(pass_description, HasSubstr(kInlinePass)); +} + +TEST(LegalizeMlirTest, LegalizesModuleWithDynamicShape) { + constexpr char legalization[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor>) -> tensor> { + %0 = "tf.Identity"(%arg0) : (tensor>) -> tensor> + func.return %0 : tensor> + } + })"; + + std::vector arg_shapes = {{1}}; + XlaCompilationResult compilation_result; + auto status = CompileSerializedMlirToXlaHlo( + legalization, arg_shapes, /*device_type=*/"XLA_TPU_JIT", + /*use_tuple_args=*/true, /*enable_op_fallback=*/false, + /*shape_determination_fns=*/{}, &compilation_result); + + EXPECT_TRUE(status.ok()); +} + +absl::StatusOr> BuildConstOpGraphWithOutputShapes() { + DataType data_type = DT_INT32; + std::initializer_list dims = {2, 3, 4, 5}; + Tensor tensor(data_type, TensorShape(dims)); + for (int i = 0; i < 2 * 3 * 4 * 5; ++i) { + tensor.flat()(i) = i; + } + + NodeDef node; + auto builder = NodeDefBuilder("some_node", "Const") + .Attr("dtype", data_type) + .Attr("value", tensor); + // Create a bad output shape attr. + AttrValue shape_attr; + TensorShapeProto* shape_proto = shape_attr.mutable_list()->add_shape(); + shape_proto->add_dim()->set_size(1); + builder.Attr("_output_shapes", shape_attr); + + TF_RETURN_IF_ERROR(builder.Finalize(&node)); + + return CreateSingleOpGraph(node, {}, {data_type}); +} + +absl::StatusOr> BuildEmptyOpGraph( + std::vector& xla_args) { + DataType data_type = DT_INT32; + XlaCompiler::Argument arg; + arg.type = DT_INT32; + arg.shape = xla::ShapeUtil::MakeShape(xla::S32, {}); + arg.name = "arg0"; + arg.kind = XlaCompiler::Argument::kParameter; + xla_args.push_back(arg); + + NodeDef node; + auto builder = NodeDefBuilder("some_node", "Empty") + .Input(FakeInput(DT_INT32)) + .Attr("dtype", data_type); + + TF_RETURN_IF_ERROR(builder.Finalize(&node)); + + return CreateSingleOpGraph(node, xla_args, {data_type}); +} + +absl::StatusOr BuildHloFromGraph( + Graph& graph, std::vector& xla_args, + bool use_output_shapes) { + xla::XlaBuilder builder( + ::testing::UnitTest::GetInstance()->current_test_info()->name()); + mlir::MLIRContext mlir_context; + llvm::SmallVector xla_params; + for (int i = 0; i < xla_args.size(); ++i) { + xla_params.push_back(Parameter(&builder, i, + std::get(xla_args[i].shape), + "arg" + std::to_string(i))); + } + std::vector returns(1); + TF_RETURN_IF_ERROR( + BuildHloFromGraph(graph, builder, mlir_context, xla_params, returns, + use_output_shapes, xla_args, + /*control_rets=*/{}, DEVICE_TPU, + FunctionLibraryDefinition(OpRegistry::Global()))); + return builder.Build(); +} + +TEST(CompileMlirUtil, UsesCorrectOriginalShapeWithoutOutputShapes) { + std::vector xla_args; + // Build a graph with an op that is supported by the MLIR lowerings. + TF_ASSERT_OK_AND_ASSIGN(auto graph, BuildConstOpGraphWithOutputShapes()); + + auto build_result = + BuildHloFromGraph(*graph, xla_args, /*use_output_shapes=*/false); + + TF_ASSERT_OK(build_result); + EXPECT_THAT(build_result, + XlaComputationProtoContains("opcode: \"constant\"")); +} + +TEST(CompileMlirUtil, UsesIncorrectOutputShapesWhenPresent) { + std::vector xla_args; + TF_ASSERT_OK_AND_ASSIGN(auto graph, BuildConstOpGraphWithOutputShapes()); + + auto build_result = + BuildHloFromGraph(*graph, xla_args, /*use_output_shapes=*/true); + + ASSERT_FALSE(build_result.ok()); + EXPECT_THAT(build_result.status().message(), + HasSubstr("op operand type 'tensor<2x3x4x5xi32>' and result type " + "'tensor<1xi32>' are cast incompatible")); +} + +TEST(CompileMlirUtil, DoesNotLowerFallbackOps) { + std::vector xla_args; + // Build a graph with an op that is not supported by the MLIR lowerings. + TF_ASSERT_OK_AND_ASSIGN(auto graph, BuildEmptyOpGraph(xla_args)); + + auto build_result = + BuildHloFromGraph(*graph, xla_args, /*use_output_shapes=*/true); + + ASSERT_FALSE(build_result.ok()); + EXPECT_THAT(build_result.status().message(), + HasSubstr("'tf.Empty' op unsupported op")); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.cc new file mode 100644 index 0000000000000000000000000000000000000000..870a85ea4ae85e64472f97e5f8a25821bb7c767d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.cc @@ -0,0 +1,381 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "xla/client/compile_only_client.h" +#include "xla/hlo/ir/hlo_input_output_alias_config.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/service/hlo.pb.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/status_macros.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/lib/monitoring/sampler.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/profile_utils/cpu_utils.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tensorflow/core/tpu/tpu_compile.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +using ::tensorflow::tpu::FunctionToHloArgs; +using ::tensorflow::tpu::GuaranteedConsts; +using ::tensorflow::tpu::MlirToHloArgs; +using ::tensorflow::tpu::ShardingAndIndex; + +auto* phase2_bridge_compilation_status = + tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/api/v1/" + "phase2_compilation_status", /*metric_name*/ + "Tracks the compilation status of the non-mlir bridge", + /* metric description */ "status" /* metric label */); + +auto* phase2_bridge_compilation_time = tsl::monitoring::Sampler<1>::New( + {"/tensorflow/core/tf2xla/api/v1/phase2_compilation_time", + "The wall-clock time spent on executing graphs in milliseconds.", + "configuration"}, + // Power of 1.5 with bucket count 45 (> 23 hours) + {tsl::monitoring::Buckets::Exponential(1, 1.5, 45)}); + +// There were no MLIR ops so the old bridge was called successfully. +constexpr char kOldBridgeNoMlirSuccess[] = "kOldBridgeNoMlirSuccess"; +// There were no MLIR ops so the old bridge was called but it failed. +constexpr char kOldBridgeNoMlirFailure[] = "kOldBridgeNoMlirFailure"; + +namespace { + +// Time the execution of kernels (in CPU cycles). Meant to be used as RAII. +struct CompilationTimer { + uint64_t start_cycles = profile_utils::CpuUtils::GetCurrentClockCycle(); + + uint64_t ElapsedCycles() { + return profile_utils::CpuUtils::GetCurrentClockCycle() - start_cycles; + } + + int64_t ElapsedCyclesInMilliseconds() { + std::chrono::duration duration = + profile_utils::CpuUtils::ConvertClockCycleToTime(ElapsedCycles()); + + return std::chrono::duration_cast(duration) + .count(); + } +}; + +// Populates input_output_alias field in the HLO Module proto. +absl::Status PopulateInputOutputAliasing( + mlir::func::FuncOp main_fn, + XlaCompiler::CompilationResult* compilation_result, bool use_tuple_args) { + constexpr char kAliasingAttr[] = "tf.aliasing_output"; + llvm::SmallDenseMap output_to_input_alias; + unsigned num_arguments = main_fn.getNumArguments(); + for (unsigned arg_index = 0; arg_index < num_arguments; ++arg_index) { + if (auto aliasing_output = main_fn.getArgAttrOfType( + arg_index, kAliasingAttr)) + output_to_input_alias[aliasing_output.getInt()] = arg_index; + } + + if (output_to_input_alias.empty()) return absl::OkStatus(); + + xla::HloModuleProto* module_proto = + compilation_result->computation->mutable_proto(); + absl::StatusOr program_shape_or_status = + compilation_result->computation->GetProgramShape(); + TF_RET_CHECK(program_shape_or_status.ok()); + + xla::ProgramShape& program_shape = program_shape_or_status.value(); + if (!program_shape.result().IsTuple()) + return absl::InternalError("Expect result to have tuple shape"); + + xla::HloInputOutputAliasConfig config(program_shape.result()); + for (auto alias : output_to_input_alias) { + if (use_tuple_args) { + TF_RETURN_IF_ERROR(config.SetUpAlias( + xla::ShapeIndex({alias.first}), 0, xla::ShapeIndex({alias.second}), + xla::HloInputOutputAliasConfig::AliasKind::kMayAlias)); + } else { + TF_RETURN_IF_ERROR(config.SetUpAlias( + xla::ShapeIndex({alias.first}), alias.second, xla::ShapeIndex({}), + xla::HloInputOutputAliasConfig::AliasKind::kMayAlias)); + } + } + *module_proto->mutable_input_output_alias() = config.ToProto(); + return absl::OkStatus(); +} + +bool failed(const absl::Status& status) { return !status.ok(); } + +// Transforms the given module to be suitable for export to TensorFlow GraphDef +// and then exports all functions to the given library. +absl::Status PrepareAndExportToLibrary(mlir::ModuleOp module, + FunctionLibraryDefinition* flib_def) { + // Pass pipeline is defined here instead of leveraging the phase one export + // pipeline because only the functional to executor dialect conversion and + // breakup islands passes are common between the export pipeline and here. + // Reconsider this if there is more commonality in the future with more + // passes. + mlir::PassManager manager(module.getContext()); + applyTensorflowAndCLOptions(manager); + manager.addPass(mlir::TF::CreatePrepareTpuComputationForTfExportPass()); + manager.addPass(mlir::TF::CreateTFRegionControlFlowToFunctional()); + manager.addPass(mlir::TF::CreateTFShapeInferencePass()); + manager.addNestedPass( + mlir::CreateFunctionalToExecutorDialectConversionPass()); + manager.addPass(mlir::CreateBreakUpIslandsPass()); + + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + + if (VLOG_IS_ON(2)) { + llvm::StringRef module_name = llvm::StringRef(); + constexpr const char* kDebugGroupBridgePhase2 = + "v1_prepare_and_export_to_library"; + internal::EnablePassIRPrinting(manager, kDebugGroupBridgePhase2, + module_name); + } + + auto prepare_status = manager.run(module); + auto diag_handler_status = diag_handler.ConsumeStatus(); + // There are cases where the scoped diagnostic handler catches a failure that + // the running of the passes does not. That causes the handler to throw if + // it is not consumed. + if (failed(prepare_status) || failed(diag_handler_status)) { + return diag_handler_status; + } + + GraphExportConfig config; + config.export_entry_func_to_flib = true; + absl::flat_hash_set control_ret_nodes; + return tensorflow::tf2xla::v2::ConvertTfExecutorToGraph( + module, config, /*graph=*/nullptr, flib_def, &control_ret_nodes); +} + +absl::Status CompileTFFunctionWithoutMlir( + FunctionToHloArgs function_computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + const XlaShapeLayoutHelpers::ShapeDeterminationFns + shape_determination_funcs, + const std::vector& arg_shapes, + const DeviceType& device_type, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client, + XlaCompiler::CompilationResult* compilation_result) { + absl::Status comp_status = CompileTFFunctionToHlo( + *function_computation.flib_def, function_computation.graph_def_version, + shape_determination_funcs, arg_shapes, device_type, + function_computation.guaranteed_constants, *function_computation.function, + metadata, client, arg_core_mapping, per_core_arg_shapes, use_tuple_args, + compilation_result); + if (comp_status.ok()) { + phase2_bridge_compilation_status->GetCell(kOldBridgeNoMlirSuccess) + ->IncrementBy(1); + } else { + phase2_bridge_compilation_status->GetCell(kOldBridgeNoMlirFailure) + ->IncrementBy(1); + } + + return comp_status; +} + +absl::Status CompileMLIRTFFunction( + mlir::ModuleOp mlir_module, const tpu::TPUCompileMetadataProto& metadata, + bool use_tuple_args, + const XlaShapeLayoutHelpers::ShapeDeterminationFns + shape_determination_funcs, + const std::vector& arg_shapes, + const DeviceType& device_type, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client, + XlaCompiler::CompilationResult* compilation_result) { + if (!mlir::SetTPUInfeedLayout(mlir_module)) + return absl::InternalError("Failed to set layouts attribute"); + + if (VLOG_IS_ON(2)) { + tensorflow::DumpMlirOpToFile("legalize_with_old_bridge", mlir_module); + } + constexpr char kEntryFuncName[] = "main"; + auto main_fn = mlir_module.lookupSymbol(kEntryFuncName); + if (!main_fn) { + return absl::InternalError( + "TPU compile op requires module with a entry function main"); + } + + // Export functions to the library. + auto flib_def = std::make_unique( + OpRegistry::Global(), FunctionDefLibrary()); + TF_RETURN_IF_ERROR(PrepareAndExportToLibrary(mlir_module, flib_def.get())); + + if (VLOG_IS_ON(2)) { + tensorflow::DumpMlirOpToFile("legalize_with_old_bridge_post_transform", + mlir_module); + } + VersionDef versions; + if (mlir::failed(ExtractTfVersions(mlir_module, &versions))) { + return absl::InternalError( + "module attribute in _TPUCompileMlir op is missing tf versions."); + } + + NameAttrList func; + func.set_name(kEntryFuncName); + GuaranteedConsts consts; + + *compilation_result = {}; + + TF_RETURN_IF_ERROR(CompileTFFunctionToHlo( + *flib_def, versions.producer(), shape_determination_funcs, arg_shapes, + device_type, consts, func, metadata, client, arg_core_mapping, + per_core_arg_shapes, use_tuple_args, compilation_result)); + + if (compilation_result != nullptr && + compilation_result->computation != nullptr) { + if (auto module_name = mlir_module.getName()) { + compilation_result->computation->mutable_proto()->set_name( + module_name->str()); + } + } + + return PopulateInputOutputAliasing(main_fn, compilation_result, + use_tuple_args); +} + +absl::Status CompileMLIRTFFunction( + tpu::MlirToHloArgs mlir_computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + const XlaShapeLayoutHelpers::ShapeDeterminationFns + shape_determination_funcs, + const std::vector& arg_shapes, + const DeviceType& device_type, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client, + XlaCompiler::CompilationResult* compilation_result) { + if (mlir_computation.mlir_module_op.has_value()) { + return CompileMLIRTFFunction( + mlir_computation.mlir_module_op.value(), metadata, use_tuple_args, + shape_determination_funcs, arg_shapes, device_type, arg_core_mapping, + per_core_arg_shapes, client, compilation_result); + } + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::mhlo::registerAllMhloDialects(registry); + mlir::MLIRContext context(registry); + + mlir::OwningOpRef mlir_module; + TF_RETURN_IF_ERROR(DeserializeMlirModule(mlir_computation.mlir_module, + &context, &mlir_module)); + + return CompileMLIRTFFunction(*mlir_module, metadata, use_tuple_args, + shape_determination_funcs, arg_shapes, + device_type, arg_core_mapping, + per_core_arg_shapes, client, compilation_result); +} + +} // namespace + +absl::Status CompileTensorflowGraphToHlo( + const std::variant& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + const XlaShapeLayoutHelpers::ShapeDeterminationFns + shape_determination_funcs, + const std::vector& arg_shapes, + tsl::DeviceType device_type, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client, + XlaCompiler::CompilationResult* compilation_result) { + LOG_FIRST_N(INFO, 1) << "Compiling MLIR computation to XLA HLO using the " + "old (non-MLIR) tf2xla bridge"; + + CompilationTimer timer; + *compilation_result = {}; + bool has_mlir = computation.index() == 0; + + std::string mlir_string = has_mlir ? "has_mlir" : "has_function_to_hlo"; + const std::string kBridgePhase2Config = + absl::StrCat("graph_old_bridge_", mlir_string); + + if (has_mlir) { + TF_RETURN_IF_ERROR(CompileMLIRTFFunction( + std::get<0>(computation), metadata, use_tuple_args, + shape_determination_funcs, arg_shapes, device_type, arg_core_mapping, + per_core_arg_shapes, client, compilation_result)); + + } else { + FunctionToHloArgs function_computation = std::get<1>(computation); + TF_RETURN_IF_ERROR(CompileTFFunctionWithoutMlir( + function_computation, metadata, use_tuple_args, + shape_determination_funcs, arg_shapes, device_type, arg_core_mapping, + per_core_arg_shapes, client, compilation_result)); + } + + phase2_bridge_compilation_time->GetCell(kBridgePhase2Config) + ->Add(timer.ElapsedCyclesInMilliseconds()); + + return absl::OkStatus(); +} + +}; // namespace v1 +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h new file mode 100644 index 0000000000000000000000000000000000000000..a4ea196db1c1099a4605aed7e7101e5eeefc165d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h @@ -0,0 +1,55 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_TF_GRAPH_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_TF_GRAPH_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/types/variant.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "xla/client/compile_only_client.h" +#include "xla/pjrt/proto/compile_options.pb.h" +#include "xla/shape.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +// Compiles the given Tensorflow graph into xla::HLO. The result is in +// compilation_result. If the input computation is in MLIR, it will be +// converted to a Tensorflow graph. Otherwise, the graph compiler will be run. +absl::Status CompileTensorflowGraphToHlo( + const std::variant& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_funcs, + const std::vector& arg_shapes, + tsl::DeviceType device_type, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client, + XlaCompiler::CompilationResult* compilation_result); + +} // namespace v1 +} // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_COMPILE_TF_GRAPH_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..e1b6401dc7d496d80f0fae939d06c491f534867e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph_test.cc @@ -0,0 +1,272 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h" + +#include +#include +#include +#include + +#include +#include +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/client_library.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/lib/monitoring/test_utils.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { +namespace { + +using ::tensorflow::monitoring::testing::CellReader; +using ::tensorflow::tpu::FunctionToHloArgs; +using ::tensorflow::tpu::MlirToHloArgs; +using ::tensorflow::tpu::ShardingAndIndex; +using ::tsl::monitoring::testing::Histogram; + +static constexpr char kCompilationTimeStreamzName[] = + "/tensorflow/core/tf2xla/api/v1/phase2_compilation_time"; + +static constexpr char kCompilationStatusStreamzName[] = + "/tensorflow/core/tf2xla/api/v1/phase2_compilation_status"; + +static constexpr char kPlatformName[] = "Host"; +constexpr char kEntryFuncName[] = "main"; + +static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } +})"; + +MlirToHloArgs CreateTestMlirToHloArgs(const char* module_str = kMlirModuleStr) { + MlirToHloArgs mlir_to_hlo_args; + mlir_to_hlo_args.rollout_state = + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_DISABLED; + mlir_to_hlo_args.mlir_module = module_str; + + return mlir_to_hlo_args; +} + +absl::StatusOr CompileWithComputation( + std::variant computation, + bool use_serialized_mlir) { + XlaCompilationResult compilation_result; + + se::Platform* platform = + se::PlatformManager::PlatformWithName(kPlatformName).value(); + auto client = + xla::ClientLibrary::GetOrCreateCompileOnlyClient(platform).value(); + + bool use_tuple_args = true; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + + tpu::TPUCompileMetadataProto metadata_proto; + std::vector arg_shapes; + if (computation.index() == 0) { + TF_RETURN_IF_ERROR(tensorflow::tf2xla::internal::ConfigureMetadata( + std::get<0>(computation).mlir_module, arg_shapes, metadata_proto)); + } + + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns; + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::mhlo::registerAllMhloDialects(registry); + mlir::MLIRContext context(registry); + mlir::OwningOpRef mlir_module; + + if (std::holds_alternative(computation) && + !use_serialized_mlir) { + TF_RETURN_IF_ERROR(DeserializeMlirModule( + std::get<0>(computation).mlir_module, &context, &mlir_module)); + std::get<0>(computation).mlir_module_op = mlir_module.get(); + std::get<0>(computation).mlir_module = ""; + } + + absl::Status compilation_status = + tensorflow::tf2xla::v1::CompileTensorflowGraphToHlo( + computation, metadata_proto, use_tuple_args, shape_determination_fns, + arg_shapes, tsl::DeviceType("XLA_TPU_JIT"), &arg_core_mapping, + &per_core_arg_shapes, client, &compilation_result); + + if (!compilation_status.ok()) return compilation_status; + + return compilation_result; +} + +using CompileTFGraphTest = ::testing::TestWithParam; + +TEST_P(CompileTFGraphTest, RecordsStreamzForMlirFallback) { + bool use_serialized_mlir = GetParam(); + + CellReader compilation_time(kCompilationTimeStreamzName); + + MlirToHloArgs mlir_to_hlo_args = CreateTestMlirToHloArgs(); + + TF_EXPECT_OK( + CompileWithComputation(mlir_to_hlo_args, use_serialized_mlir).status()); + + Histogram histogram = compilation_time.Delta("graph_old_bridge_has_mlir"); + + EXPECT_EQ(histogram.num(), 1); +} + +TEST_P(CompileTFGraphTest, RecordsStreamzForFunctionToHlo) { + bool use_serialized_mlir = GetParam(); + + CellReader compilation_time(kCompilationTimeStreamzName); + CellReader compilation_status(kCompilationStatusStreamzName); + + FunctionDef empty_function = + tensorflow::FunctionDefHelper::Create("empty", {}, {}, {}, {}, {}); + + tensorflow::FunctionDefLibrary fdef; + *(fdef.add_function()) = empty_function; + tensorflow::FunctionLibraryDefinition flib_def( + tensorflow::OpRegistry::Global(), fdef); + + OpInputList guaranteed_constants; + NameAttrList function; + function.set_name("empty"); + + FunctionToHloArgs function_to_hlo_args = {&function, + &flib_def, + /*graph_def_version=*/0, + {&guaranteed_constants}}; + + TF_EXPECT_OK(CompileWithComputation(function_to_hlo_args, use_serialized_mlir) + .status()); + + Histogram histogram = + compilation_time.Delta("graph_old_bridge_has_function_to_hlo"); + + EXPECT_EQ(histogram.num(), 1); + EXPECT_EQ(compilation_status.Delta("kOldBridgeNoMlirSuccess"), 1); +} + +TEST_P(CompileTFGraphTest, SuccessfullyCompilesWithManualSharding) { + bool use_serialized_mlir = GetParam(); + + // MLIR module from failing test. + constexpr char kSupportedManualSharding[] = R"( + module @module___inference_tpu_function_41 attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1617 : i32}} { + func.func @main(%arg0: tensor<2x2xf32>) -> (tensor<2x2xf32> {mhlo.sharding = "\08\03\1A\02\02\01\22\02\00\01"}) { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.XlaSharding"(%arg0) {_XlaSharding = "\08\03\1A\02\02\01\22\02\00\01", sharding = "\08\03\1A\02\02\01\22\02\00\01"} : (tensor<2x2xf32>) -> tensor<2x2xf32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.XlaSharding"(%outputs) {_XlaSharding = "\08\03\1A\02\02\01\22\02\00\01", sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<2x2xf32>) -> tensor<2x2xf32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.XlaSpmdFullToShardShape"(%outputs_0) {dim = -1 : i64, manual_sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<2x2xf32>) -> tensor<1x2xf32> + %control_4 = tf_executor.island wraps "tf._XlaHostComputeMlir"(%outputs_2) {host_mlir_module = "", manual_sharding = true, recv_key = "host_compute_channel_0_retvals", send_key = "host_compute_channel_0_args"} : (tensor<1x2xf32>) -> () + %outputs_5, %control_6 = tf_executor.island(%control_4) wraps "tf._XlaHostComputeMlir"() {host_mlir_module = "module {\0A func.func @host_func() -> tensor<1x2xf32> {\0A %0 = \22tf.Const\22() {value = dense<0.1> : tensor<1x2xf32>} : () -> tensor<1x2xf32> \0A return %0 : tensor<1x2xf32>}}", manual_sharding = true, recv_key = "host_compute_channel_1_retvals", send_key = "host_compute_channel_1_args"} : () -> tensor<1x2xf32> + %outputs_7, %control_8 = tf_executor.island wraps "tf.XlaSpmdShardToFullShape"(%outputs_5) {dim = -1 : i64, full_shape = #tf_type.shape<2x2>, manual_sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<1x2xf32>) -> tensor<2x2xf32> + %outputs_9, %control_10 = tf_executor.island wraps "tf.XlaSharding"(%outputs_7) {_XlaSharding = "\08\03\1A\02\02\01\22\02\00\01", sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<2x2xf32>) -> tensor<2x2xf32> + tf_executor.fetch %outputs_9 : tensor<2x2xf32> + } + return %0 : tensor<2x2xf32> + } + } + )"; + auto mlir_to_hlo_args = CreateTestMlirToHloArgs(kSupportedManualSharding); + + auto result = CompileWithComputation(mlir_to_hlo_args, use_serialized_mlir); + + EXPECT_TRUE(result.ok()); +} + +TEST_P(CompileTFGraphTest, DoesNotInlineStatelessRandomOps) { + bool use_serialized_mlir = GetParam(); + + static constexpr char kHasReturnValues[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> (tensor<32x64xf32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %cst = "tf.Const"() {value = dense<[524170, 523952]> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_0 = "tf.Const"() {value = dense<[32, 64]> : tensor<2xi32>} : () -> tensor<2xi32> + %0 = "tf.StatelessRandomNormal"(%cst_0, %cst) : (tensor<2xi32>, tensor<2xi32>) -> tensor<32x64xf32> + return %0 : tensor<32x64xf32> + } + })"; + + auto compilation_result = CompileWithComputation( + CreateTestMlirToHloArgs(kHasReturnValues), use_serialized_mlir); + EXPECT_TRUE(compilation_result.ok()); + + // The StatelessRandomNormal must not be replaced by a literal tensor. + EXPECT_THAT(compilation_result, + ComputationProtoContains("tf.StatelessRandomNormal")); +} + +TEST_P(CompileTFGraphTest, TestRunsShapeInference) { + bool use_serialized_mlir = GetParam(); + + static constexpr char kShapeInferenceModule[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %0 = "tf.Const"() <{value = dense<-1> : tensor<3360x8xi32>}> : () -> tensor<3360x8xi32> + %cst_33 = "tf.Const"() <{value = dense<[1120, -1]> : tensor<2xi32>}> : () -> tensor<2xi32> + %cst_34 = "tf.Const"() <{value = dense<[3, 1120, -1]> : tensor<3xi32>}> : () -> tensor<3xi32> + %cst_63 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %1965:4 = "tf._XlaHostComputeMlir"(%0, %cst_34, %cst_63, %cst_33) <{host_mlir_module = "#loc1 = loc(\22Reshape:\22)\0A#loc2 = loc(\22Reshape_4\22)\0A#loc3 = loc(\22Reshape\22)\0A#loc9 = loc(fused[#loc1, #loc2, #loc3])\0Amodule {\0A func.func @host_func(%arg0: tensor<3360x?xi32> loc(fused[#loc1, #loc2, #loc3]), %arg1: tensor<3xi32> loc(fused[#loc1, #loc2, #loc3]), %arg2: tensor loc(fused[#loc1, #loc2, #loc3]), %arg3: tensor<2xi32> loc(fused[#loc1, #loc2, #loc3])) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>) {\0A %0 = \22tf.Reshape\22(%arg0, %arg1) {_xla_outside_compilation = \220\22} : (tensor<3360x?xi32>, tensor<3xi32>) -> tensor<3x1120x?xi32> loc(#loc9)\0A %1:3 = \22tf.Split\22(%arg2, %0) {_xla_outside_compilation = \220\22} : (tensor, tensor<3x1120x?xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1x1120x?xi32>) loc(#loc10)\0A %2 = \22tf.Reshape\22(%1#0, %arg3) {_xla_outside_compilation = \220\22} : (tensor<1x1120x?xi32>, tensor<2xi32>) -> tensor<1120x?xi32> loc(#loc11)\0A %3 = \22tf.Shape\22(%2) {_xla_outside_compilation = \220\22} : (tensor<1120x?xi32>) -> tensor<2xi32> loc(#loc12)\0A return %1#1, %1#2, %2, %3 : tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32> loc(#loc9)\0A } loc(#loc9)\0A} loc(#loc)\0A#loc = loc(unknown)\0A#loc4 = loc(\22Split:\22)\0A#loc5 = loc(\22split\22)\0A#loc6 = loc(\22Reshape_5\22)\0A#loc7 = loc(\22Shape:\22)\0A#loc8 = loc(\22Shape_4\22)\0A#loc10 = loc(fused[#loc4, #loc5])\0A#loc11 = loc(fused[#loc1, #loc6])\0A#loc12 = loc(fused[#loc7, #loc8])\0A", recv_key = "host_compute_channel_0_retvals", send_key = "host_compute_channel_0_args"}> : (tensor<3360x8xi32>, tensor<3xi32>, tensor, tensor<2xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>) + return + } + } + )"; + + auto compilation_result = CompileWithComputation( + CreateTestMlirToHloArgs(kShapeInferenceModule), use_serialized_mlir); + EXPECT_TRUE(compilation_result.ok()); +} + +INSTANTIATE_TEST_SUITE_P(CompileTFGraphTest, CompileTFGraphTest, + ::testing::Bool()); + +} // namespace +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/mlir_bridge_config_v1.proto b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/mlir_bridge_config_v1.proto new file mode 100644 index 0000000000000000000000000000000000000000..9a2253d694e0fdb6832fc758b99c28582c4bb66e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/mlir_bridge_config_v1.proto @@ -0,0 +1,11 @@ +// This proto defines messages for TF2XLA MLIR bridge configuration. + +syntax = "proto2"; + +package tensorflow.tf2xla.v1; + +message MlirBridgeConfig { + // Whether to enable tf2xla mlir bridge in compiling SavedModel. By default, + // it is enabled. + optional bool enable_tf2xla_mlir_bridge = 1 [default = true]; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/empty_func.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/empty_func.mlir new file mode 100644 index 0000000000000000000000000000000000000000..826ce6955e709a9285f4c917ab8004923075c4ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/empty_func.mlir @@ -0,0 +1,5 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/invalid_executor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/invalid_executor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..79974124efbe109c4ed5856e2dc062e8e279fe5e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/invalid_executor.mlir @@ -0,0 +1,17 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() { + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/multiple_submodules.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/multiple_submodules.mlir new file mode 100644 index 0000000000000000000000000000000000000000..63773d6126738748a0f4dbb4043065734d9be1d9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/multiple_submodules.mlir @@ -0,0 +1,9 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } + + module @submodule attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1647 : i32}} { + } + +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/prepare_to_library.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/prepare_to_library.mlir new file mode 100644 index 0000000000000000000000000000000000000000..90343e75ea4091f629e30c178ce5a58f3eabadd0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/testdata/prepare_to_library.mlir @@ -0,0 +1,10 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %0 = "tf.Const"() <{value = dense<-1> : tensor<3360x8xi32>}> : () -> tensor<3360x8xi32> + %cst_33 = "tf.Const"() <{value = dense<[1120, -1]> : tensor<2xi32>}> : () -> tensor<2xi32> + %cst_34 = "tf.Const"() <{value = dense<[3, 1120, -1]> : tensor<3xi32>}> : () -> tensor<3xi32> + %cst_63 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %1965:4 = "tf._XlaHostComputeMlir"(%0, %cst_34, %cst_63, %cst_33) <{host_mlir_module = "#loc1 = loc(\22Reshape:\22)\0A#loc2 = loc(\22Reshape_4\22)\0A#loc3 = loc(\22Reshape\22)\0A#loc9 = loc(fused[#loc1, #loc2, #loc3])\0Amodule {\0A func.func @host_func(%arg0: tensor<3360x?xi32> loc(fused[#loc1, #loc2, #loc3]), %arg1: tensor<3xi32> loc(fused[#loc1, #loc2, #loc3]), %arg2: tensor loc(fused[#loc1, #loc2, #loc3]), %arg3: tensor<2xi32> loc(fused[#loc1, #loc2, #loc3])) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>) {\0A %0 = \22tf.Reshape\22(%arg0, %arg1) {_xla_outside_compilation = \220\22} : (tensor<3360x?xi32>, tensor<3xi32>) -> tensor<3x1120x?xi32> loc(#loc9)\0A %1:3 = \22tf.Split\22(%arg2, %0) {_xla_outside_compilation = \220\22} : (tensor, tensor<3x1120x?xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1x1120x?xi32>) loc(#loc10)\0A %2 = \22tf.Reshape\22(%1#0, %arg3) {_xla_outside_compilation = \220\22} : (tensor<1x1120x?xi32>, tensor<2xi32>) -> tensor<1120x?xi32> loc(#loc11)\0A %3 = \22tf.Shape\22(%2) {_xla_outside_compilation = \220\22} : (tensor<1120x?xi32>) -> tensor<2xi32> loc(#loc12)\0A return %1#1, %1#2, %2, %3 : tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32> loc(#loc9)\0A } loc(#loc9)\0A} loc(#loc)\0A#loc = loc(unknown)\0A#loc4 = loc(\22Split:\22)\0A#loc5 = loc(\22split\22)\0A#loc6 = loc(\22Reshape_5\22)\0A#loc7 = loc(\22Shape:\22)\0A#loc8 = loc(\22Shape_4\22)\0A#loc10 = loc(fused[#loc4, #loc5])\0A#loc11 = loc(fused[#loc1, #loc6])\0A#loc12 = loc(fused[#loc7, #loc8])\0A", recv_key = "host_compute_channel_0_retvals", send_key = "host_compute_channel_0_args"}> : (tensor<3360x8xi32>, tensor<3xi32>, tensor, tensor<2xi32>) -> (tensor<1x1120x?xi32>, tensor<1x1120x?xi32>, tensor<1120x?xi32>, tensor<2xi32>) + return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.cc new file mode 100644 index 0000000000000000000000000000000000000000..6cf86841831bf62f93a9c458e18a1ef459cc14ef --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.cc @@ -0,0 +1,193 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h" + +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "xla/tsl/lib/monitoring/counter.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/platform/error_payloads.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/core_platform_payloads.pb.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::OpPassManager; +using mlir::Pass; +using mlir::PassManager; +using mlir::func::FuncOp; + +auto *tf_dialect_to_executor_dialect_status = tsl::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/api/v1/tf_dialect_to_executor_dialect_status", + "Counts how often a successful export from TF Dialect to Executor Dialect " + "is", + "status"); + +constexpr char kExportSuccess[] = "success"; +constexpr char kExportFailed[] = "failed"; + +namespace { + +void AddTfDialectToExecutorPasses(OpPassManager &pm) { + auto add_pass = [&](std::unique_ptr pass) { + pm.addNestedPass(std::move(pass)); + pm.addPass(mlir::CreateBreakUpIslandsPass()); + }; + + pm.addPass(mlir::tf_executor::CreateTFExecutorTPUV1IslandInliningPass()); + // There are cases where we don't consume all compilation and + // replication attributes like we do for the V2 pipeline, so we need to + // convert them from unified to legacy attributes before they get + // exposed to outside of the bridge. + pm.addNestedPass( + mlir::TFTPU::CreateConvertToLegacyCompileAndReplicateAttributesPass()); + + pm.addPass(mlir::TF::CreateTFRegionControlFlowToFunctional()); + add_pass(mlir::CreateFunctionalToExecutorDialectConversionPass()); + add_pass(mlir::TFDevice::CreateReplicateToIslandPass( + /*legacy_graph_export=*/true)); + add_pass(mlir::TFDevice::CreateReplicaIDToDeviceOrdinalPass()); + add_pass(mlir::TFDevice::CreateParallelExecuteToIslandsPass( + /*legacy_graph_export=*/true)); + add_pass(mlir::TFDevice::CreateLaunchToDeviceAttributePass( + /*legacy_graph_export=*/true)); + pm.addNestedPass(mlir::TFTPU::CreateTPUDevicePropagationPass()); + pm.addNestedPass(mlir::TFTPU::CreateTPUColocateSplitsPass()); + pm.addPass(mlir::createSymbolDCEPass()); + if (tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_convert_control_to_data_outputs_pass) { + bool composite_tpuexecute_side_effects = + tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_composite_tpuexecute_side_effects; + pm.addPass( + mlir::tf_executor::CreateTFExecutorConvertControlToDataOutputsPass( + composite_tpuexecute_side_effects)); + } + pm.addPass(mlir::TF::CreateVerifySuitableForExportPass()); +} + +absl::Status RecordStatusIfError(absl::Status status) { + if (status.ok()) { + return absl::OkStatus(); + } + + VLOG(1) << "Failed to export from TF Dialect to TF Executor Dialect. " + << status; + tf_dialect_to_executor_dialect_status->GetCell(kExportFailed)->IncrementBy(1); + + constexpr char bridge_subcomponent[] = + "TFXLA_TF_FUNCTIONAL_TO_EXECUTOR_EXPORT_v1"; + constexpr char kBridgeComponent[] = "TFXLABridge"; + + tsl::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_1, + status); + + tsl::error_logging::Log(kBridgeComponent, bridge_subcomponent, + status.ToString()) + .IgnoreError(); + + return status; +} + +} // namespace + +absl::Status ExportFromTensorflowDialectToExecutor( + ModuleOp module, llvm::StringRef module_name) { + PassManager tf_to_executor(module.getContext()); + ::tensorflow::applyTensorflowAndCLOptions(tf_to_executor); + tf_to_executor.enableVerifier(); + + AddTfDialectToExecutorPasses(tf_to_executor); + + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "tfxla_bridge_v1_tfdialect_to_executor_before"), + module, llvm::StringRef(), &tf_to_executor); + + if (VLOG_IS_ON(2) || + DEBUG_DATA_DUMPER()->ShouldDump( + module_name.str(), kDebugGroupBridgePhase1ExecutorExport)) { + internal::EnablePassIRPrinting( + tf_to_executor, kDebugGroupBridgePhase1ExecutorExport, module_name); + } + } + + LogicalResult result = tf_to_executor.run(module); + (void)result; // Ignore the error since it's captured by the diag_handler. + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "tfxla_bridge_v1_tfdialect_to_executor_after"), + module, llvm::StringRef(), &tf_to_executor); + } + + if (result.failed()) { + return RecordStatusIfError(diag_handler.ConsumeStatus()); + } + + tf_dialect_to_executor_dialect_status->GetCell(kExportSuccess) + ->IncrementBy(1); + + return diag_handler.ConsumeStatus(); +} + +// Registers a pipeline builder function for TF Graph export. Should be +// the same as ExportFromTensorflowDialectToExecutor just in PassRegistration +// form. +mlir::PassPipelineRegistration<> tf_dialect_to_executor_pipeline( + "tf-dialect-to-executor-v1", + "Run passes to convert from TF Dialect to Executor in preparation for " + "exporting module back to TF Graph.", + AddTfDialectToExecutorPasses); + +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h new file mode 100644 index 0000000000000000000000000000000000000000..6773d23937c9f11b65cbbbc614f8f98b6b1ab744 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h @@ -0,0 +1,58 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_TF_DIALECT_TO_EXECUTOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_TF_DIALECT_TO_EXECUTOR_H_ + +#include "absl/base/attributes.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/platform/status.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { + +// Given the input Module op that's in the Tensorflow Dialect, convert the MLIR +// module in place to the Tensorflow Executor Dialect. Returns an OK Status if +// success, otherwise failure with an error message. +// The Tensorflow Executor Dialect is required to export an MLIR module to a +// Tensorflow GraphDef. This API will add control dependencies and verify that +// the conversion was successful. This version adds extra control dependencies +// for replication and parallel execution ops, which may slow performance. +// Prefer to use the v2 of this API. +// +// This also converts the Tensorflow Dialect MLIR into the Tensorflow Executor +// dialect that is suitable to be exported to GraphDef. Graph -> MLIR -> Graph +// is not perfectly round trippable, so this API will attempt to make the module +// exportable and verify some properties of the Tensorflow Executor MLIR that +// are required by Graph Export. It will return an error if it cannot. +// +// Input: A MLIR Module in the Tensorflow Dialect with no +// `tf_device.cluster_func` ops. +// Output: A MLIR module in the Tensorflow Executor Dialect. + +ABSL_DEPRECATED( + "Use v2/tf_dialect_to_executor.h::ExportFromTensorflowDialectToExecutor " + "instead.") +absl::Status ExportFromTensorflowDialectToExecutor( + mlir::ModuleOp module, llvm::StringRef module_name = llvm::StringRef()); + +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V1_TF_DIALECT_TO_EXECUTOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..f2378f8c5575d499255d0fd6a7b332fa409d977b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor_test.cc @@ -0,0 +1,104 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v1/tf_dialect_to_executor.h" + +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/resource_loader.h" + +namespace tensorflow { +namespace tf2xla { +namespace v1 { +namespace { + +using mlir::ModuleOp; + +using mlir::DialectRegistry; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; +using tensorflow::monitoring::testing::CellReader; + +static constexpr char kCompilationStreamz[] = + "/tensorflow/core/tf2xla/api/v1/tf_dialect_to_executor_dialect_status"; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/api/v1/testdata/"); +} + +class TensorflowDialectToExecutorTest : public ::testing::Test { + public: + TensorflowDialectToExecutorTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(TensorflowDialectToExecutorTest, ConvertsToExecutor) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(ExportFromTensorflowDialectToExecutor(*mlir_module_)); + + EXPECT_EQ(compilation_status.Delta("success"), 1); +} + +TEST_F(TensorflowDialectToExecutorTest, ErrorsWhenCannotConvert) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("invalid_executor.mlir")); + + EXPECT_FALSE(ExportFromTensorflowDialectToExecutor(*mlir_module_).ok()); + + EXPECT_EQ(compilation_status.Delta("failed"), 1); +} + +} // namespace +} // namespace v1 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..3aa4a5d914e31ab3d83121ccb4334ec2b80b1531 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/BUILD @@ -0,0 +1,412 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow/core/platform:build_config.bzl", "tf_proto_library") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +# Please reach out to tf-bridge-team@ before using the TF2XLA bridge. +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + ":__subpackages__", + ], +) + +cc_library( + name = "legalize_tf", + srcs = ["legalize_tf.cc"], + hdrs = ["legalize_tf.h"], + visibility = [ + "//learning/brain/google/xla:__pkg__", + "//learning/brain/mlir/bridge:__pkg__", + "//tensorflow/compiler/mlir/quantization/stablehlo:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/api/v2/testing:__pkg__", + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt:__pkg__", + ], + deps = [ + ":device_type_proto_cc", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tf2xla/api/v1:compile_tf_graph", + "//tensorflow/compiler/mlir/tf2xla/internal:compilation_timer", + "//tensorflow/compiler/mlir/tf2xla/internal:legalize_tf_to_hlo", + "//tensorflow/compiler/mlir/tf2xla/internal:reproducer_proto_cc", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:variant", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@tsl//tsl/platform:protobuf", + "@xla//xla:shape_util", + "@xla//xla:xla_proto_cc", + "@xla//xla/client:compile_only_client", + "@xla//xla/hlo/ir:hlo", + "@xla//xla/pjrt/proto:compile_options_proto_cc", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "legalize_tf_test", + srcs = ["legalize_tf_test.cc"], + deps = [ + ":legalize_tf", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir/tf2xla/api/v2/testing:compile_mlir", + "//tensorflow/compiler/mlir/tf2xla/internal:test_matchers", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + "@com_google_googletest//:gtest", + "@llvm-project//mlir:Pass", + "@xla//xla:shape_util", + "@xla//xla/client:client_library", + "@xla//xla/stream_executor:platform", + "@xla//xla/stream_executor:platform_manager", + "@xla//xla/tsl/lib/monitoring:test_utils", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "legalize_tf_test_gpu", + srcs = ["legalize_tf_test_gpu.cc"], + tags = [ + "config-cuda-only", + "no_oss", # This test only runs with GPU. + "requires-gpu-nvidia", + ], + deps = [ + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir/tf2xla/api/v2/testing:compile_mlir", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_proto_library( + name = "device_type_proto", + srcs = ["device_type.proto"], + visibility = [ + "//learning/serving/contrib/tfrt/mlir/saved_model_analysis:__pkg__", + ], +) + +cc_library( + name = "cluster_tf", + srcs = ["cluster_tf.cc"], + hdrs = ["cluster_tf.h"], + visibility = [ + "//learning/serving/contrib/tfrt/mlir/saved_model_analysis:__pkg__", + "//tensorflow/compiler/mlir/tfrt:__pkg__", + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt:__pkg__", + "//tensorflow/compiler/tf2xla:__pkg__", + ], + deps = [ + ":device_type_proto_cc", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/internal:clustering_bridge_passes", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/compiler/mlir/tf2xla/internal/passes:clustering_passes", + "//tensorflow/core:framework", + "//tensorflow/core:lib_proto_parsing", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:stacktrace", + "//tensorflow/core/platform:status", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tsl//tsl/platform:error_logging", + "@xla//xla/tsl/platform:errors", + ], +) + +tf_cc_test( + name = "cluster_tf_test", + srcs = ["cluster_tf_test.cc"], + data = [ + "testdata/empty_func.mlir", + "testdata/invalid_executor.mlir", + "testdata/outside_compilation.mlir", + ], + deps = [ + ":cluster_tf", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:tf_dialect_lib", + "//tensorflow/compiler/mlir/tf2xla/api/v2/testing:utils", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:status", + ], +) + +cc_library( + name = "tf_dialect_to_executor", + srcs = ["tf_dialect_to_executor.cc"], + hdrs = ["tf_dialect_to_executor.h"], + visibility = [ + "//learning/serving/contrib/tfrt/mlir/saved_model_analysis:__pkg__", + "//tensorflow/compiler/mlir/tfrt:__pkg__", + "//tensorflow/compiler/tf2xla:__pkg__", + ], + deps = [ + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/internal:logging_hooks", + "//tensorflow/core:framework", + "//tensorflow/core/platform:error_payloads", + "//tensorflow/core/platform:status", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tsl//tsl/platform:error_logging", + "@xla//xla/tsl/lib/monitoring:counter", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:status", + ], +) + +tf_cc_test( + name = "tf_dialect_to_executor_test", + srcs = ["tf_dialect_to_executor_test.cc"], + data = [ + "testdata/empty_func.mlir", + "testdata/func_with_dead_ops.mlir", + "testdata/invalid_executor.mlir", + ], + deps = [ + ":tf_dialect_to_executor", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tf2xla/api/v2/testing:utils", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@xla//xla/tsl/lib/core:status_test_util", + ], +) + +cc_library( + name = "tf_executor_to_graph", + srcs = [ + "tf_executor_to_graph.cc", + ], + hdrs = [ + "tf_executor_to_graph.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:export_utils", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow:verify_suitable_for_graph_export", + "//tensorflow/compiler/mlir/tensorflow/translate:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/utils:name_utils", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:graph", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/graph/regularization:util", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla:status_macros", + ], +) + +tf_cc_test( + name = "tf_executor_to_graph_test", + srcs = ["tf_executor_to_graph_test.cc"], + data = [ + "testdata/valid_executor.mlir", + "testdata/valid_graph.txt", + ], + deps = [ + ":tf_executor_to_graph", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test_main", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@riegeli//riegeli/bytes:fd_reader", + "@riegeli//riegeli/bytes:read_all", + "@tsl//tsl/platform:protobuf", + "@xla//xla/tsl/lib/core:status_test_util", + ], +) + +cc_library( + name = "graph_to_tf_executor", + srcs = [ + "graph_to_tf_executor.cc", + ], + hdrs = [ + "graph_to_tf_executor.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":mlir_roundtrip_flags", + "//tensorflow/compiler/jit:shape_inference_helpers", + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_attr", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_attributes", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/translate:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tf2xla/internal:graph_to_tf_executor_util", + "//tensorflow/compiler/mlir/tf2xla/internal:node_order", + "//tensorflow/compiler/tf2xla:functionalize_control_flow", + "//tensorflow/compiler/tf2xla:functionalize_control_flow_util", + "//tensorflow/compiler/tf2xla:tf2xla_defs", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/common_runtime:function_body", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:DerivedAttributeOpInterface", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla:status_macros", + ], +) + +tf_cc_test( + name = "graph_to_tf_executor_test", + srcs = ["graph_to_tf_executor_test.cc"], + data = [ + "testdata/graph_with_flib_def.txt", + "testdata/valid_graph.txt", + ], + deps = [ + ":graph_to_tf_executor", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tf2xla/internal:graph_to_tf_executor_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test_main", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@riegeli//riegeli/bytes:fd_reader", + "@riegeli//riegeli/bytes:read_all", + "@tsl//tsl/platform:protobuf", + "@xla//xla/tsl/lib/core:status_test_util", + ], +) + +cc_library( + name = "mlir_roundtrip_flags", + hdrs = ["mlir_roundtrip_flags.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..74f249ea4b0ac8a4a3db7b5b44168a0a04f8d0aa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.cc @@ -0,0 +1,214 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h" + +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/device_type.pb.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/metrics.h" +#include "tensorflow/core/platform/error_payloads.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/stacktrace.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::OpPassManager; +using mlir::PassManager; +using mlir::func::FuncOp; + +// Run the TF XLA Bridge based on the input pipeline, which can be either TPU +// bridge pipeline or non TPU bridge pipeline. +absl::Status RunTFXLABridge( + ModuleOp module, + llvm::function_ref pipeline_builder, + llvm::StringRef module_name = llvm::StringRef(), + llvm::StringRef dump_prefix = "tf_xla_bridge_v2") { + // Explicitly check that the TensorFlow dialect can constant fold ops. + // Constant folding is essential for the bridge. Without this check, the + // bridge may fail with an error that is difficult to understand and not + // actionable. + if (!mlir::TF::TensorFlowDialect::HasConstantFoldHook()) { + return absl::InternalError( + "TensorFlow dialect missing constant fold hook in TFXLA bridge phase " + "1; this could happen if the binary doesn't link the constant fold " + "hook registration library."); + } + + PassManager bridge(module.getContext()); + bridge.enableVerifier(); + ::tensorflow::applyTensorflowAndCLOptions(bridge); + + // Populate a passmanager with the list of passes that implement the bridge. + pipeline_builder(bridge); + + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + dump_prefix.str() + "_before"), + module, llvm::StringRef(), &bridge); + } + + if (VLOG_IS_ON(2) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), + kDebugGroupBridgePhase1Clustering)) { + ::tensorflow::tf2xla::internal::EnablePassIRPrinting( + bridge, kDebugGroupBridgePhase1Clustering, module_name); + } + + LogicalResult result = bridge.run(module); + (void)result; + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + dump_prefix.str() + "_after"), + module, llvm::StringRef(), &bridge); + } + + return diag_handler.ConsumeStatus(); +} + +absl::Status RecordIfErrorStatus(const std::string error_prefix, + bool fallback_enabled, std::string bridge_type, + std::string device_type, absl::Status status) { + if (status.ok()) { + return status; + } + + VLOG(2) << error_prefix << " " << status; + tensorflow::metrics::UpdateTfMlirBridgeFirstPhaseCounter( + /*bridge_type*/ bridge_type, /*bridge_version=*/"v2", device_type, + /*fallback_enabled=*/fallback_enabled, + /*result=*/"failure"); + + tsl::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_1, + status); + + std::string bridge_subcomponent = "TFXLA_PHASE_ONE_MLIR_TPU_BRIDGE"; + if (device_type != "tpu") { + bridge_subcomponent = "TFXLA_PHASE_ONE_MLIR_CPU/GPU_BRIDGE"; + } + + tsl::error_logging::Log(mlir::TF::kBridgeComponent, bridge_subcomponent, + status.ToString()) + .IgnoreError(); + + return status; +} + +void CreateReplicatedClusteringPipeline(OpPassManager &pm, + llvm::StringRef module_name) { + // Since the internal bridge clustering passes are shared among TF1/TF2 + // TF2-only passes should go here. However, this should be very rare and + // new passes generally should go into the internal + // AddReplicatedBridgeClusteringPipelinePasses. + pm.addPass(tensorflow::tf2xla::internal::CreateTPUValidateInputsPass()); + pm.addNestedPass( + mlir::TF::CreateCanonicalizeCompileAndReplicateAttributesPass()); + tensorflow::tf2xla::internal::AddReplicatedBridgeClusteringPipelinePasses( + pm, module_name); +} + +void CreateReplicatedClusteringPipelineV2(OpPassManager &pm) { + CreateReplicatedClusteringPipeline(pm, /*module_name=*/""); +} + +absl::Status RunFunctionTf2xlaClusteringBridge( + ModuleOp module, bool is_supported_by_replicated_brige, + bool is_in_fallback_enabled_mode, llvm::StringRef module_name) { + std::string device_type = is_supported_by_replicated_brige + ? mlir::TF::kMlirPh1BridgeCounterTpu + : mlir::TF::kMlirPh1BridgeCounterNonTpu; + + VLOG(2) + << (is_supported_by_replicated_brige ? "Replicated" : "NonReplicated") + << " Bridge called stack trace is " + << "(NOTE: this is not an error; rather the stack trace for debugging) : " + << tensorflow::CurrentStackTrace(); + absl::Status clustering_status = + is_supported_by_replicated_brige + ? RunTFXLABridge( + module, + [module_name](OpPassManager &pm) { + CreateReplicatedClusteringPipeline(pm, module_name); + }, + module_name, /*dump_prefix=*/"tf_xla_bridge_v2_replicated") + : RunTFXLABridge( + module, + [](OpPassManager &pm) { + tensorflow::tf2xla::internal:: + AddNonReplicatedBridgeClusteringPipelinePasses(pm); + }, + module_name, /*dump_prefix=*/"tf_xla_bridge_v2_nonreplicated"); + + std::string bridge_type = is_supported_by_replicated_brige + ? mlir::TF::kMlirPh1BridgeCounterReplicated + : mlir::TF::kMlirPh1BridgeCounterNonReplicated; + // TODO(b/317798386): add is_supported_by_replicated_brige as a filter. + TF_RETURN_IF_ERROR(RecordIfErrorStatus( + /*error_prefix=*/"clustering_v2", is_in_fallback_enabled_mode, + bridge_type, device_type, clustering_status)); + + // TODO(b/317798386): add is_supported_by_replicated_brige as a filter. + tensorflow::metrics::UpdateTfMlirBridgeFirstPhaseCounter( + bridge_type, /*bridge_version=*/"v2", device_type, + /*fallback_enabled=*/is_in_fallback_enabled_mode, + /*result=*/"success"); + + return absl::OkStatus(); +} + +mlir::PassPipelineRegistration<> replicated_clustering_bridge_v2( + "tf-replicated-clustering-bridge-v2", + "Run all the passes involved in transforming a TensorFlow 2 graph before " + "execution so that it is suitable for targeting devices.", + CreateReplicatedClusteringPipelineV2); + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h new file mode 100644 index 0000000000000000000000000000000000000000..71281bdee69d6736e2fe3f032d04ef4e302214bd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h @@ -0,0 +1,61 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_CLUSTER_TF_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_CLUSTER_TF_H_ + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/api/v2/device_type.pb.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +// Run all the passes involved in transforming the graph before execution so +// that it is suitable for targeting devices when called with the TF 2 Function +// API. Users that need clustering with the Session API should use the v1 Bridge +// API. These transformations take as input a Tensorflow Graph as an MLIR Module +// and transforms the module in place to cluster the given ops for compilation +// that is compatible with the given device_type. The MLIR should be in the TF +// Executor Dialect for graph nodes and edges or be in TF Functional already. +// Individual Op inside a node should be the Tensorflow Functional Dialect. The +// output MLIR is in the TF Functional Dialect. Returns OkStatus if passed, +// otherwise an error. +// +// Inputs: +// module - The MLIR Module that will be clustered. Expected to be in TF +// Executor Dialect or TF Functional Dialect. Will convert to TF Functional. +// is_supported_by_replicated_brige - If the graph targets the replicated +// bridge. Set it to true for replicated/partitioned graphs. e.g. replicated +// and single-core TPU graphs. Set this to false if the graph is not +// replicated, e.g. CPU/GPU graphs. is_in_fallback_enabled_mode - Whether this +// was called with fallback to the non-MLIR Bridge. This is just for logging +// purposes and doesn't affect logic. module_name - What the input module name +// is for debugging help. +// +// Output: Modifies the input module in place with clustered operations. +// status - Whether the transformation to cluster the input MLIR module was +// successful. +absl::Status RunFunctionTf2xlaClusteringBridge( + mlir::ModuleOp module, bool is_supported_by_replicated_brige, + bool is_in_fallback_enabled_mode, + llvm::StringRef module_name = llvm::StringRef()); +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_CLUSTER_TF_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..388e8c2d739cff7ac485c76bb55cfb4358f1ceff --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf_test.cc @@ -0,0 +1,168 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h" + +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/resource_loader.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace { + +using ::mlir::DialectRegistry; +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::mlir::WalkResult; +using ::mlir::func::FuncOp; +using ::tensorflow::monitoring::testing::CellReader; +using ::tensorflow::tf2xla::v2::testing::TestDataPath; + +static constexpr char kCompilationStreamz[] = + "/tensorflow/core/tf_mlir_bridge_first_phase_v2_count"; + +class FunctionClusterTensorflowDialectTest : public ::testing::Test { + public: + FunctionClusterTensorflowDialectTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(FunctionClusterTensorflowDialectTest, ClustersTfReplicatedBridge) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(RunFunctionTf2xlaClusteringBridge( + *mlir_module_, /*is_supported_by_replicated_brige*/ true, + /*is_in_fallback_enabled_mode=*/false)); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + EXPECT_EQ(compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV2, + mlir::TF::kMlirPh1BridgeCounterTpu, + "fallback_disabled", "success"), + 1); +} + +TEST_F(FunctionClusterTensorflowDialectTest, + RunsOutsideCompilationReplicatedBridge) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("outside_compilation.mlir")); + + TF_EXPECT_OK(RunFunctionTf2xlaClusteringBridge( + *mlir_module_, /*is_supported_by_replicated_brige*/ true, + /*is_in_fallback_enabled_mode=*/false)); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + bool has_cluster_op = false; + main.walk([&](mlir::tf_device::ClusterFuncOp cluster_op) { + has_cluster_op = true; + return WalkResult::advance(); + }); + + EXPECT_TRUE(has_cluster_op); + EXPECT_EQ(compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV2, + mlir::TF::kMlirPh1BridgeCounterTpu, + "fallback_disabled", "success"), + 1); +} + +TEST_F(FunctionClusterTensorflowDialectTest, ClustersTFNonReplicatedBridge) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(RunFunctionTf2xlaClusteringBridge( + *mlir_module_, /*is_supported_by_replicated_brige*/ false, + /*is_in_fallback_enabled_mode=*/false)); + + FuncOp main = mlir_module_->lookupSymbol("main"); + ASSERT_TRUE(main); + + EXPECT_EQ( + compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterNonReplicated, + mlir::TF::kMlirPh1BridgeCounterV2, + mlir::TF::kMlirPh1BridgeCounterNonTpu, + "fallback_disabled", "success"), + 1); +} + +TEST_F(FunctionClusterTensorflowDialectTest, LogsFallbackMode) { + CellReader compilation_status(kCompilationStreamz); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(RunFunctionTf2xlaClusteringBridge( + *mlir_module_, /*is_supported_by_replicated_brige*/ true, + /*is_in_fallback_enabled_mode=*/true)); + + EXPECT_EQ(compilation_status.Delta(mlir::TF::kMlirPh1BridgeCounterReplicated, + mlir::TF::kMlirPh1BridgeCounterV2, + mlir::TF::kMlirPh1BridgeCounterTpu, + "fallback_enabled", "success"), + 1); +} + +} // namespace +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/device_type.proto b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/device_type.proto new file mode 100644 index 0000000000000000000000000000000000000000..90acc8ac2f17823455d19fb62b04f77954869a3e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/device_type.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; + +package tensorflow.tf2xla.v2; + +// The requested device type to compile for. +enum DeviceType { + DEVICE_TYPE_UNSPECIFIED = 0; + XLA_TPU_JIT = 1; + XLA_CPU_JIT = 2; + XLA_GPU_JIT = 3; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.cc new file mode 100644 index 0000000000000000000000000000000000000000..cf4fdc158882db6cb68549b7649910fafd358740 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.cc @@ -0,0 +1,2844 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/inlined_vector.h" +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/DerivedAttributeOpInterface.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/jit/shape_inference_helpers.h" +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/node_order.h" +#include "tensorflow/compiler/tf2xla/functionalize_control_flow.h" +#include "tensorflow/compiler/tf2xla/functionalize_control_flow_util.h" +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" +#include "xla/status_macros.h" +#include "xla/tsl/platform/crash_analysis.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/common_runtime/function_body.h" +#include "tensorflow/core/common_runtime/function_def_utils.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/common_runtime/graph_runner.h" +#include "tensorflow/core/common_runtime/shape_refiner.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_def_builder.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/graph_debug_info_builder.h" +#include "tensorflow/core/graph/graph_node_util.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/graph/tensor_id.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/stack_frame.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow/core/protobuf/saved_object_graph.pb.h" +#include "tensorflow/core/protobuf/saver.pb.h" +#include "tensorflow/core/protobuf/struct.pb.h" +#include "tensorflow/core/protobuf/trackable_object_graph.pb.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tensorflow/core/util/dump_graph.h" + +static inline absl::string_view StringRefToView(llvm::StringRef ref) { + return {ref.data(), ref.size()}; +} + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using ::mlir::NamedAttrList; +using ::mlir::TensorType; +using ::tsl::StatusOr; + +constexpr absl::string_view kOutputShapesAttrName = "_output_shapes"; + +void LoadImporterDialects(mlir::MLIRContext& context) { + // Load dialects involved in the conversion + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialectsImpl(registry, false); + context.appendDialectRegistry(registry); + for (llvm::StringRef name : registry.getRegisteredDialectNames()) + context.getOrLoadDialect(name); +} + +bool IsOutputShapesAttribute(const AttrValue& attr_value, + llvm::StringRef attr_name) { + return attr_name.compare(kOutputShapesAttrName) == 0 && + attr_value.value_case() == AttrValue::kList; +} + +bool IsResourceOutputShapesAttribute(const AttrValue& attr_value, + llvm::StringRef attr_name) { + if (attr_name == "_handle_dtypes" || attr_name == "_handle_shapes") + return attr_value.value_case() == AttrValue::kList; + return false; +} + +class NameUniquifier : public OpOrArgNameMapper { + public: + explicit NameUniquifier(const FunctionLibraryDefinition& flib) + : flib_(flib) {} + + private: + bool IsUnique(llvm::StringRef name) override { + return !flib_.Contains(std::string(name)); + } + + std::string GetName(OpOrVal op_or_val) override { + DCHECK(false) << "Unimplemented"; + return ""; + } + + const FunctionLibraryDefinition& flib_; +}; + +std::string LogGraphImportConfig(const GraphImportConfig& config) { + std::ostringstream ss; + + ss << "graph_func_name: " << config.graph_func_name; + GraphImportConfig::InputArrays inputs; + ss << "\ninputs: "; + for (auto& it : inputs) { + ss << "\n\t" << it.first << " -> " + << DataTypeString(it.second.imported_dtype) << " " + << it.second.shape.DebugString(); + } + ss << "\noutputs:"; + for (auto& output : config.outputs) ss << " " << output; + ss << "\ncontrol_outputs:"; + for (auto& output : config.control_outputs) ss << " " << output; + ss << "\nprune_unused_nodes: " << config.prune_unused_nodes; + ss << "\nconvert_legacy_fed_inputs: " << config.convert_legacy_fed_inputs; + ss << "\ngraph_as_function: " << config.graph_as_function; + ss << "\nupgrade_legacy: " << config.upgrade_legacy; + ss << "\nrestrict_functionalization_to_compiled_nodes: " + << config.restrict_functionalization_to_compiled_nodes; + ss << "\nenable_shape_inference: " << config.enable_shape_inference; + ss << "\nunconditionally_use_set_output_shapes: " + << config.unconditionally_use_set_output_shapes; + ss << "\nxla_compile_device_type: " << config.xla_compile_device_type; + + return ss.str(); +} + +// Stateful helper class to import a TensorFlow model into an MLIR Module. +// +// This is the base class that contains common utilities shared between the +// GraphDef importer and SavedModel importer. +// +// A subclass is expected to call `PrepareConvert` first to perform necessary +// preparation over the graph and also certain internal bookkeeping data. +// Afterwards the other protected methods can be called. +class ImporterBase { + protected: + explicit ImporterBase( + const FunctionLibraryDefinition& flib, const GraphDebugInfo& debug_info, + const GraphImportConfig& specs, mlir::ModuleOp module, + std::unordered_map* tf_name_to_mlir_name, + NameUniquifier* function_name_uniquifier, + llvm::StringRef function_name_for_debug_info = "") + : builder_(module.getContext()), + module_(module), + context_(module.getContext()), + tf_name_to_mlir_name_(tf_name_to_mlir_name), + graph_flib_(flib), + specs_(specs), + debug_info_(debug_info), + function_name_for_debug_info_(function_name_for_debug_info), + function_name_uniquifier_(function_name_uniquifier), + error_handler_(module.getContext()) { + // Log import config. + if (VLOG_IS_ON(1)) { + LOG(INFO) << "Importing with: " << LogGraphImportConfig(specs); + for (auto& it : *tf_name_to_mlir_name) { + LOG(INFO) << "\t" << it.first << " -> " << it.second; + } + } + + stack_traces_ = LoadTracesFromDebugInfo(debug_info_); + } + + // Returns the inferred function signature of the given function body. Input + // types are unranked tensor of the respective datatype in the function and + // result types are inferred by the shape_refiner_. Result types need not be + // unranked tensors and could be ranked tensors in cases where result type + // depends on an op with static output shape like tf.Const. + absl::StatusOr InferLibFunctionType( + const FunctionBody& fbody); + + // Extracts arg and ret nodes from FunctionBody. + void GetArgsAndRetsFromFunctionBody( + const FunctionBody& fbody, + absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes, + absl::InlinedVector* control_ret_nodes); + + // Prepares converting the graph to an MLIR module. This step removes the + // backedges of the graph, orders the nodes and infers the shapes. + // PrepareConvert needs to ensure that the original `graph` is cloned prior + // execution. The cloning procedure relies on the roundtrip through the + // GraphDef. Graph to GraphDef def conversion is heavy, in case, `graph_def` + // was obtained previously provide it to the PrepareConvert to reuse. + absl::Status PrepareConvert(const Graph& graph, + std::unique_ptr graph_def = nullptr); + + // Converts the prepared graph to a Function and adds it to the module. A set + // of nodes from the graph are given to converted to the arguments and returns + // of the function. + absl::Status Convert(llvm::StringRef func_name, mlir::FunctionType func_type, + const absl::InlinedVector& arg_nodes, + const absl::InlinedVector& ret_nodes, + const absl::InlinedVector& control_ret_nodes, + llvm::ArrayRef attrs); + + // Finds out the function definition for the given function name from the + // graph and converts it to a function of the module. This method is called + // on demand because the graph flib_def does not provide an iterator + // interface. + absl::Status ConvertLibFunction(llvm::StringRef func_name); + + // Returns the list of nodes in the graph. Nodes are presented in the reverse + // order of a post-order depth-first visit starting from the graph's source + // nodes. + llvm::ArrayRef GetOrderedNodes() const { return ordered_nodes_; } + + // Returns the inferred input type at index `idx` of the `node` in the + // context. + absl::StatusOr InferInputType(const Node& node, int idx, + mlir::Builder builder); + + // Returns the inferred output type at index `idx` of the `node` in the + // context. + absl::StatusOr InferOutputType(const Node& node, int idx, + mlir::Builder builder); + + // Convert deferred TF functions to the MLIR representation. + // Conversion is deferred for efficiency reasons, e.g., to limit depth + // of recursion and reduce stack size pressure. + absl::Status ConvertDeferredFunctions(); + + private: + // Most types with subtypes have only one subtype. + using ElementSubtypes = llvm::SmallVector; + + // Metadata used for deferred function conversion. + struct DeferredConversionMetaData { + DeferredConversionMetaData( + const std::string& function_name, + const std::vector& attributes) + : function_name(function_name), attributes(attributes) {} + + std::string function_name; + std::vector attributes; + }; + + // Adds all the ordered_nodes to the shape refiner shape_refiner_. Then all + // data type and shape information is maintained by the shape_refiner_. + // TODO(jpienaar): Remove once shape inference on import is removed. + absl::Status AddNodesToShapeRefiner( + std::unordered_map* node_name_map); + + // Prune nodes that do not feed into fetch nodes. + absl::Status PruneUnreachableNodes( + std::unordered_map* node_name_map); + + // Converts feeds to Placeholder nodes. + absl::Status ConvertFeedsToPlaceholders( + std::unordered_map* node_name_map); + + // Converts the inferred shape referred to by 'handle' in 'context', with + // given element type, and returns an MLIR tensor type. + absl::StatusOr ConvertDataTypeAndShape( + DataType dtype, const shape_inference::ShapeHandle& handle, + const std::vector* handle_subtypes, + shape_inference::InferenceContext* context, mlir::Builder builder); + + // Converts the inferred shape referred to by 'handle' in 'context', with + // given element type, and returns an MLIR tensor type. + absl::StatusOr ConvertElementTypeAndShape( + mlir::Type element_type, const shape_inference::ShapeHandle& handle, + shape_inference::InferenceContext* context, mlir::Builder builder); + + // Converts the inferred subtypes for an element type to corresponding MLIR + // types in 'context'. + absl::StatusOr ConvertSubtypes( + const std::vector* handle_subtypes, + shape_inference::InferenceContext* context, mlir::Builder builder); + + // Converts the tensor proto into an MLIR elements attribute. + absl::StatusOr ConvertTensorProto( + const TensorProto& value) { + return tensorflow::ConvertTensorProto(value, &builder_); + } + + // Converts func name in graphdef to mlir::SymbolRefAttribute. + absl::StatusOr ConvertFunctionCallName( + const std::string& func_name); + + // Converts the given non-function-call AttrValue to an MLIR Attribute. + absl::StatusOr ConvertAttributeValue(const AttrValue& value); + + // Converts the given function-call AttrValue to MLIR Attributes and pushes + // them to the given attributes list. For example, if there is a kFunc + // AttrValue {name : foo, attrs : {k1 : bar, k2 : rfc}}, it will convert it to + // a list of MLIR Attributes: {{base_name : foo}, {base_name.k1 : bar}, + // {base_name.k2 : rfc}}. + absl::Status ConvertFunctionCallAttribute(const std::string& base_name, + const AttrValue& value, + NamedAttrList* attributes); + + // Helper to create either a tf_executor operation or a TF operation wrapped + // in an island. + mlir::Operation* CreateOperation( + const Node& node, llvm::StringRef node_type_name, + const mlir::OperationState& result, + const llvm::SmallVectorImpl& control_operands); + + // Converts one NodeDef from the input GraphDef into an Operation and + // inserts it into the MLIR module using builder_. + absl::Status ConvertNode(const Node& node); + + // If the input graph represents a while-loop, the edges pointing from a + // "NextIteration" node to a "Merge" node add cyclic dependencies and make the + // topological sorting impossible. We need to remove these edges from the + // input graph to infer shapes and construct a Function. For each + // "NextIteration" node, there are two operations, "NextIteration.source" + // and "NextIteration.sink" are added to the MLIR module. + using BackEdge = BackEdgeHelper::BackEdge; + + // Removes backedges from the input graph. The removed edges are added back to + // to OpBuilder after the remaining graph is converted to the Function. + absl::Status RemoveBackedges(); + + // Restores backedges removed during shape inference to the final Function. + absl::Status AddBackedges(); + + // Restores a single backedge in the Function by adding a replicated + // operation before the dst operation. + absl::Status AddBackedge(mlir::Operation* sink, mlir::Operation* dst, + int dst_input); + + // Adds the input arguments and return operation to the function. The + // arguments are added as basic block argument. Also the argument types and + // the id of the nodes from the input graph needs to be specified. + absl::Status ConvertFunctionArgAndRets( + mlir::func::FuncOp func, mlir::tf_executor::GraphOp graph_op, + llvm::ArrayRef arg_types, + const absl::InlinedVector& arg_nodes, + const absl::InlinedVector& ret_nodes, + const absl::InlinedVector& control_ret_nodes); + + // Gets the location information of the given node. It uses the + // "original_node_name" in the NodeDef to get the corresponding file location + // (FileLineColLoc) from the input DebugInfo and returns an CallSiteLoc. If + // there are multiple "original_node_names", a FusedLoc is returned. If the + // node name couldn't be found in the input DebugInfo, a NameLoc is used as + // the location. + mlir::Location GetLocation(const Node& node); + + // Appends the location string for the node to the error message and returns + // the combined error absl::Status. + absl::Status EmitErrorWithLocationStr(const Node& node, + const absl::Status& error_status); + + // If node is an Identity node carrying XLA metadata in its name, extracts it + // and attaches it as an attribute to the MLIR op corresponding to node's + // input. + absl::Status HandleXlaMetadata(const Node& node); + + // Inserts a placeholder node in the graph to replace a feed output tensor, + // and returns the new placeholder node and a boolean indicating if the + // original input node was removed from the graph. Uses of the feed output + // tensor are replaced with this placeholder node. If the feed output tensor + // is of a single output node, the control dependencies are forwarded to the + // the placeholder node, and the original node will be removed. + // Note: This modifies the graph, and so any list of ordered nodes needs to be + // reconstructed. + absl::StatusOr> CreatePlaceholderNodeForFeed( + const TensorShapeProto& shape, DataType dtype, Node* node, int index, + const std::unordered_map& node_name_map); + + // Gets the input and output nodes corresponding to the specified input and + // output nodes in specs_. If there are no input or output nodes specified, + // nodes will be empty. + absl::Status GetInputOutputNodes( + const std::unordered_map& node_name_map, + std::unordered_set* nodes); + + // The input graph with backedges removed. The removed backedges are stored + // in the back_edge_helper. + BackEdgeHelper back_edge_helper_; + // A map between node and output index, for each backedge. + absl::flat_hash_map back_edge_node_output_; + absl::flat_hash_map back_edge_dst_inputs_; + // A map between sink and source operation of NextIteration + absl::flat_hash_map + next_iteration_sink_source_; + + // All nodes and version information about the (copied) imported graph. + std::unique_ptr graph_; + std::vector ordered_nodes_; + + // Maps from a Node ID to a MLIR value. + using NodeValueMap = absl::flat_hash_map; + + mlir::OpBuilder builder_; + mlir::ModuleOp module_; + mlir::MLIRContext* context_; + std::unordered_map* tf_name_to_mlir_name_; + const FunctionLibraryDefinition& graph_flib_; + const GraphImportConfig& specs_; + const GraphDebugInfo& debug_info_; + StackTracesMap stack_traces_; + llvm::StringRef function_name_for_debug_info_; + NodeValueMap node_values_; + // TODO(jpienaar): Remove once shape inference on import is removed. + // The shape_refinner_ will be nullptr if shape inference on import is + // not enabled. + std::unique_ptr shape_refiner_ = nullptr; + NameUniquifier* function_name_uniquifier_; + mlir::StatusScopedDiagnosticHandler error_handler_; + // All the TF ops encountered that aren't modelled in dialect. + llvm::DenseSet unmodelled_op_names_; + + protected: + // Maps feed as TensorId to new Placeholder node name. + absl::flat_hash_map remapped_feeds_; + // Keep track of functions required deferred conversion. + std::queue deferred_functions_; +}; + +// Mapping from node name to feed (index and ArrayInfo). Node name must outlive +// this map. +using FeedsByNode = absl::flat_hash_map< + absl::string_view, + absl::flat_hash_map*>>; + +// Creates from a `GraphImportConfig::InputArrays` a mapping from a feeds output +// tensor name to index and ArrayInfo. Keys and values are backed by +// `GraphImportConfig::InputArrays`. +absl::StatusOr GetFeedsByNode( + const GraphImportConfig::InputArrays& inputs) { + FeedsByNode feeds_by_node; + feeds_by_node.reserve(inputs.size()); + + for (const auto& input : inputs) { + TensorId tensor = ParseTensorName(input.first); + if (tensor.index() < 0) + return absl::FailedPreconditionError( + absl::StrCat("Feed output tensor must be a data output '", + tensor.ToString(), "'")); + + auto& node = feeds_by_node[tensor.node()]; + if (!node.insert({tensor.index(), &input}).second) + return absl::FailedPreconditionError( + absl::StrCat("Multiple feeds for the same output tensor '", + tensor.ToString(), "'")); + } + + return feeds_by_node; +} + +// Creates a unique name for a node that will be replacing a feed output tensor. +std::string GetUniqueNodeName( + absl::string_view node_name, int index, + const std::unordered_map& node_name_map) { + std::string new_node_name_base = absl::StrCat(node_name, "_", index); + int count = 0; + std::string new_node_name = new_node_name_base; + while (node_name_map.find(new_node_name) != node_name_map.end()) { + new_node_name = absl::StrCat(new_node_name_base, "_", count++); + } + return new_node_name; +} + +absl::Status ImporterBase::ConvertDeferredFunctions() { + while (!deferred_functions_.empty()) { + auto conversion_metadata = deferred_functions_.front(); + deferred_functions_.pop(); + + const FunctionDef* func_def = + graph_flib_.Find(conversion_metadata.function_name); + // Converts the graph to an MLIR function and adds it to the module. + // We populate the NodeSpec so that all the _Arg ops get their shape + // added correctly. + GraphImportConfig specs; + specs.enable_shape_inference = specs_.enable_shape_inference; + specs.unconditionally_use_set_output_shapes = + specs_.unconditionally_use_set_output_shapes; + for (const auto& name_and_value : func_def->attr()) { + if (name_and_value.first == "_input_shapes") { + auto& list = name_and_value.second.list(); + auto& signature = func_def->signature(); + // Some models have "_input_shapes" attribute, but with its value empty + if (list.shape_size() > 0 && + list.shape_size() != signature.input_arg_size()) { + return absl::FailedPreconditionError(absl::StrCat( + "Number of input arguments must be equal to the length of " + "_input_shapes attribute in function '", + StringRefToView(conversion_metadata.function_name), "'.")); + } + for (int i = 0, e = signature.input_arg_size(); i < e; i++) { + auto& input_arg = signature.input_arg(i); + auto& array_info = specs.inputs[input_arg.name()]; + array_info.imported_dtype = input_arg.type(); + // set to unranked for empty "_input_shapes" attribute + if (list.shape_size() > 0) + array_info.shape = list.shape(i); + else + array_info.shape.set_unknown_rank(true); + } + } + } + + ImporterBase importer(graph_flib_, debug_info_, specs, module_, + tf_name_to_mlir_name_, function_name_uniquifier_, + conversion_metadata.function_name); + + std::unique_ptr fbody; + TF_RETURN_IF_ERROR( + FunctionDefToBodyHelper(*func_def, AttrSlice(), &graph_flib_, &fbody)); + TF_RETURN_IF_ERROR(importer.PrepareConvert(*fbody->graph)); + + TF_ASSIGN_OR_RETURN(auto func_type, importer.InferLibFunctionType(*fbody)); + + absl::InlinedVector arg_nodes; + absl::InlinedVector ret_nodes; + absl::InlinedVector control_ret_nodes; + importer.GetArgsAndRetsFromFunctionBody(*fbody, &arg_nodes, &ret_nodes, + &control_ret_nodes); + const std::string& mlir_func_name = + (*tf_name_to_mlir_name_)[conversion_metadata.function_name]; + + TF_RETURN_IF_ERROR(importer.Convert(mlir_func_name, func_type, arg_nodes, + ret_nodes, control_ret_nodes, + conversion_metadata.attributes)); + + // Additional function bodies could be discovered during the deferred + // loading of the current function. Add them to the working queue. + while (!importer.deferred_functions_.empty()) { + deferred_functions_.push(importer.deferred_functions_.front()); + importer.deferred_functions_.pop(); + } + } + + return absl::OkStatus(); +} + +absl::Status ImporterBase::RemoveBackedges() { + // Remove all the backedges. So the nodes can be added to the shape refiner. + TF_RETURN_IF_ERROR(back_edge_helper_.Remove(graph_.get())); + VLOG(1) << "Found " << (back_edge_helper_.RemovedEdges().size()) + << " backedges."; + + // Creates a map for quickly identifying whether a node output is a backedge. + for (const auto& edge : back_edge_helper_.RemovedEdges()) { + if (back_edge_node_output_.find(edge.src) != back_edge_node_output_.end() && + back_edge_node_output_[edge.src] != edge.src_output) { + return absl::FailedPreconditionError( + "More than one of the src node outputs are backedges!"); + } + back_edge_node_output_[edge.src] = edge.src_output; + // We expect a merge to receive a single backedge (multiple NextIteration + // nodes feeding into the same merge is unexpected here). + DCHECK(!back_edge_dst_inputs_.contains(edge.dst)); + back_edge_dst_inputs_[edge.dst] = edge; + } + + // Obtains a RPO ordering, using node names as a tiebreak for stable sorting. + + ordered_nodes_.clear(); + TopologicalOrdering( + *graph_, [&](Node* n) { ordered_nodes_.push_back(n); }, GroupByDevice()); + return absl::OkStatus(); +} + +absl::Status CopyStackTraces(const Graph& from, Graph* to) { + // Copy over the stack traces. + // TODO(jpienaar): This really shouldn't be needed, copying the Graph above + // and then needing these traversals is unfortunate. + std::unordered_map node_map = from.BuildNodeNameIndex(); + for (Node* node : to->nodes()) { + if (const Node* old_node = node_map[node->name()]) { + if (const std::shared_ptr& stack = + old_node->GetStackTrace()) { + DVLOG(2) << "Stack for " << node->name() << " " + << old_node->GetStackTrace()->ToString( + AbstractStackTrace::TracePrintingOptions()); + node->SetStackTrace(stack); + } else { + DVLOG(1) << "No stack for " << node->name() << " (" << node + << ") in Graph " << &from; + } + } else { + DVLOG(1) << "No stack for " << node->name() << " (" << node + << ") in Graph " << &from; + } + } + + return absl::OkStatus(); +} + +absl::StatusOr> +ImporterBase::CreatePlaceholderNodeForFeed( + const TensorShapeProto& shape, DataType dtype, Node* node, int index, + const std::unordered_map& node_name_map) { + DCHECK_LT(index, node->num_outputs()); + const bool update_inplace = node->num_outputs() == 1 && index == 0; + std::string new_node_name = + update_inplace ? node->name() + : GetUniqueNodeName(node->name(), index, node_name_map); + + Node* placeholder_node; + NodeBuilder builder(new_node_name, "Placeholder"); + builder.Attr("shape", shape); + builder.Attr("dtype", dtype); + TF_RETURN_IF_ERROR(builder.Finalize(graph_.get(), &placeholder_node)); + + // Update edges from original feed with Placeholder node. + std::vector data_edges; + std::vector control_edges; + for (const tensorflow::Edge* edge : node->out_edges()) { + if (edge->src_output() == index) { + data_edges.push_back(edge); + } else if (update_inplace && edge->IsControlEdge()) { + control_edges.push_back(edge); + } + } + + for (const auto* edge : data_edges) { + TF_RETURN_IF_ERROR(graph_->UpdateEdge(placeholder_node, 0, edge->dst(), + edge->dst_input())); + } + + // TODO(lyandy): Preserve control dependencies properly by not forwarding + // control dependencies to data outputs and not removing single output nodes. + // When a data output is replaced as a feed, unless there is another non feed + // data output or an explicit control output used by the same node, transitive + // control dependencies are not to be executed. For single output nodes, + // Placeholders can be converted to a NoOp if there are no uses, and + // PlaceholderWithDefault can be converted to an Identity. + for (const auto* edge : control_edges) { + graph_->AddControlEdge(placeholder_node, edge->dst()); + graph_->RemoveControlEdge(edge); + } + + if (update_inplace) { + graph_->RemoveNode(node); + } + + return std::pair(placeholder_node, update_inplace); +} + +absl::Status ImporterBase::GetInputOutputNodes( + const std::unordered_map& node_name_map, + std::unordered_set* nodes) { + auto add_node = [&](absl::string_view name) { + auto it = node_name_map.find(std::string(name)); + if (it == node_name_map.end()) { + return absl::FailedPreconditionError( + absl::StrCat("Graph does not contain node: ", name)); + } + nodes->insert(it->second); + return absl::OkStatus(); + }; + + // Remap feeds and fetches to newly created Placeholder nodes. + for (const auto& input : specs_.inputs) { + TensorId tensor = ParseTensorName(input.first); + auto remapped_it = remapped_feeds_.find(tensor); + if (remapped_it != remapped_feeds_.end()) { + TF_RETURN_IF_ERROR(add_node(remapped_it->second)); + } else { + TF_RETURN_IF_ERROR(add_node(tensor.node())); + } + } + + for (const auto& output : specs_.outputs) { + TensorId tensor = ParseTensorName(output); + auto remapped_it = remapped_feeds_.find(tensor); + if (remapped_it != remapped_feeds_.end()) { + TF_RETURN_IF_ERROR(add_node(remapped_it->second)); + } else { + TF_RETURN_IF_ERROR(add_node(tensor.node())); + } + } + + for (const auto& control_output : specs_.control_outputs) + TF_RETURN_IF_ERROR(add_node(control_output)); + + return absl::OkStatus(); +} + +// TODO(jpienaar): Remove this post shape inference on import flag is removed. +absl::Status ImporterBase::AddNodesToShapeRefiner( + std::unordered_map* node_name_map) { + shape_refiner_ = + std::make_unique(graph_->versions(), graph_->op_registry()); + // Some operations (for example "TPUExecute") don't have shape inference + // function defined, so we should set this to false for adding nodes with + // these types of operations. + shape_refiner_->set_require_shape_inference_fns(false); + shape_refiner_->set_function_library_for_shape_inference(&graph_flib_); + + TF_ASSIGN_OR_RETURN(auto feeds_by_node, GetFeedsByNode(specs_.inputs)); + + // First add all nodes to the refiner. + for (Node* node : ordered_nodes_) { + // We need to use a TensorFlow node to teach the shape refiner that user + // specifies certain data type and shape for the inputs in the `specs_`. + // This node shouldn't have any inputs, only have one output and its + // output type/shape is only determined by its "named" attributes. (The + // attributes should have fixed names so we can use the info from `specs_` + // to set the value of them.) `Placeholder` satisfies these constraints. + // + // Therefore, if the input node isn't a `Placeholder`, we create one and use + // it to replace the original input node, so the shape refiner can + // successfully propagate the user's input type and shape to the rest of the + // graph. + bool node_added_to_shape_refiner = false; + auto it = feeds_by_node.find(node->name()); + if (it != feeds_by_node.end()) { + auto op_name = node->op_def().name(); + if (op_name != "Placeholder" && op_name != "LegacyFedInput" && + op_name != FunctionLibraryDefinition::kArgOp) { + for (const auto& output_tensor : it->second) { + const int index = output_tensor.first; + const ArrayInfo& array_info = output_tensor.second->second; + + DataType dtype = array_info.imported_dtype; + // Uses the existing output type if it isn't specified by the user. + if (dtype == DT_INVALID) { + dtype = node->output_type(index); + } + + TF_ASSIGN_OR_RETURN( + auto placeholder_node_and_removed, + CreatePlaceholderNodeForFeed(array_info.shape, dtype, node, index, + *node_name_map)); + + Node* placeholder_node = placeholder_node_and_removed.first; + if (placeholder_node_and_removed.second) { + // Original node has been removed from the graph. + node = placeholder_node; + node_added_to_shape_refiner = true; + } + remapped_feeds_[{it->first, index}] = placeholder_node->name(); + (*node_name_map)[placeholder_node->name()] = placeholder_node; + // Add the new placeholder node to the shape refiner. + absl::Status status = shape_refiner_->AddNode(placeholder_node); + if (!status.ok()) { + return EmitErrorWithLocationStr(*placeholder_node, status); + } + } + } else { + auto index_it = it->second.find(0); + if (index_it == it->second.end()) { + return absl::FailedPreconditionError( + absl::StrCat("Missing feed output tensor at index 0 for node '", + node->name(), "'")); + } + node->AddAttr("shape", index_it->second->second.shape); + DataType dtype = index_it->second->second.imported_dtype; + // Uses the existing output type if it isn't specified by the user. + if (dtype == DT_INVALID) { + dtype = node->output_type(0); + } + node->AddAttr("dtype", dtype); + } + } + if (!node_added_to_shape_refiner) { + // Add the node to the shape refiner if the node hasn't been removed. + absl::Status status = shape_refiner_->AddNode(node); + if (!status.ok()) { + return EmitErrorWithLocationStr(*node, status); + } + } + + auto set_shape_from_list_attr = [&](const AttrValue* attr) { + auto& list = attr->list(); + // This follows the same approach as in ValidateShape, but only flags + // warning in case where there are mismatch in number of shapes and + // outputs and in which case it just returns without attempting to refine. + if (list.shape_size() != node->num_outputs()) { + LOG(WARNING) << "Node '" << node->name() << "' has " + << node->num_outputs() << " outputs but the " + << kOutputShapesAttrName + << " attribute specifies shapes for " << list.shape_size() + << " outputs"; + return absl::OkStatus(); + } + + for (const auto& shape : llvm::enumerate(list.shape())) { + auto* node_context = shape_refiner_->GetContext(node); + shape_inference::ShapeHandle handle; + absl::Status status = + node_context->MakeShapeFromShapeProto(shape.value(), &handle); + if (!status.ok()) { + return EmitErrorWithLocationStr(*node, status); + } + node_context->set_output(shape.index(), handle); + } + return absl::OkStatus(); + }; + + // If it is the argument node, the shape handle is set explicitly, so it + // can be propagated to the body nodes of the function. + if (absl::string_view(node->type_string()) == + FunctionLibraryDefinition::kArgOp) { + auto* node_context = shape_refiner_->GetContext(node); + DCHECK(node_context != nullptr); + if (const AttrValue* attr = node->attrs().Find("shape")) { + shape_inference::ShapeHandle handle; + absl::Status status = + node_context->MakeShapeFromShapeProto(attr->shape(), &handle); + if (!status.ok()) { + return EmitErrorWithLocationStr(*node, status); + } + node_context->set_output(0, handle); + } else if (const AttrValue* attr = + node->attrs().Find(kOutputShapesAttrName)) { + TF_RETURN_IF_ERROR(set_shape_from_list_attr(attr)); + } else { + node_context->set_output(0, node_context->UnknownShape()); + } + } + + // Following GraphConstructor::ValidateShape called from + // GraphConstructor::Convert, override the shape if _output_shapes is set. + if (specs_.unconditionally_use_set_output_shapes || + node->op_def().name() == "ReadVariableOp") { + if (const AttrValue* attr = node->attrs().Find(kOutputShapesAttrName)) + TF_RETURN_IF_ERROR(set_shape_from_list_attr(attr)); + } + } + + // Since we might have inserted and removed nodes from the graph, fix + // source/sink edges and reconstruct the RPO ordering of nodes + FixupSourceAndSinkEdges(graph_.get()); + + // Prune nodes in the graph that are not reachable from the output. + if (specs_.prune_unused_nodes) { + std::unordered_set prune_start; + TF_RETURN_IF_ERROR(GetInputOutputNodes(*node_name_map, &prune_start)); + if (!prune_start.empty()) { + if (PruneForReverseReachability(graph_.get(), prune_start)) { + VLOG(1) << "Pruned unused nodes in graphdef"; + } else { + VLOG(1) << "No unused nodes in graphdef to prune"; + } + } else { + VLOG(1) << "No output nodes specified, skipping pruning"; + } + } else { + VLOG(1) << "Pruning unused nodes in graphdef is disabled"; + } + + // Re-initialize ordered_nodes_ since we might have modified the graph. + ordered_nodes_.clear(); + TopologicalOrdering( + *graph_, [&](Node* n) { ordered_nodes_.push_back(n); }, GroupByDevice()); + + VLOG(1) << "Inferring graph shapes to fixpoint"; + + // The "changed" information from UpdateNode can give false positives, so we + // create a dedicated method to verify the shapes are not changed before and + // after the shape refine. + auto same_inferred_shape = [](shape_inference::InferenceContext* c, + shape_inference::ShapeHandle s0, + shape_inference::ShapeHandle s1) -> bool { + if (s0.SameHandle(s1) || (!c->RankKnown(s0) && !c->RankKnown(s1))) { + return true; + } + if (c->Rank(s0) != c->Rank(s1)) { + return false; + } + for (int i = 0; i < c->Rank(s0); ++i) { + if (!c->Dim(s0, i).SameHandle(c->Dim(s1, i))) { + int64_t val0 = c->Value(c->Dim(s0, i)); + int64_t val1 = c->Value(c->Dim(s1, i)); + // Negative value is treated as unknown so all negative values indicate + // the same dimension. + if (val0 >= 0 && val1 >= 0 && val0 != val1) return false; + } + } + return true; + }; + + bool changed = true; + int i = 0; + const int kMaxIterationCount = 2; + while (changed && i != kMaxIterationCount) { + changed = false; + for (const Node* node : ordered_nodes_) { + auto* shape_context = shape_refiner_->GetContext(node); + DCHECK(shape_context != nullptr); + absl::InlinedVector existing; + existing.reserve(shape_context->num_outputs()); + for (int o = 0; o < shape_context->num_outputs(); ++o) { + existing.push_back(shape_context->output(o)); + } + bool inferred = false; + shape_inference::ShapeHandle handle; + absl::Status status = + shape_refiner_->UpdateNode(node, /*relax=*/false, &inferred); + if (!status.ok()) { + return EmitErrorWithLocationStr(*node, status); + } + for (int o = 0; o < shape_context->num_outputs(); ++o) { + if (!same_inferred_shape(shape_context, shape_context->output(o), + existing[o])) { + changed = true; + break; + } + } + } + ++i; + } + if (i >= kMaxIterationCount) { + LOG(WARNING) << "Graph shapes did not converge to a fixpoint within " + << kMaxIterationCount + << " iterations. Graph shapes may be conservative."; + } + VLOG(1) << "Graph shapes were inferred with " << (i - 1) + << " extra rounds of analysis to reach a fixpoint."; + return absl::OkStatus(); +} + +absl::StatusOr ImporterBase::InferInputType(const Node& node, + int idx, + mlir::Builder builder) { + if (specs_.enable_shape_inference) { + // TODO(jpienaar): Remove this if shape inference on import flag is removed. + auto* context = shape_refiner_->GetContext(&node); + DataType dtype = node.input_type(idx); + return ConvertDataTypeAndShape(dtype, context->input(idx), + context->input_handle_shapes_and_types(idx), + context, builder); + } + DataType dtype = node.properties()->input_types[idx]; + mlir::Type element_type; + TF_RETURN_IF_ERROR(ConvertDataType(dtype, builder, &element_type)); + return mlir::UnrankedTensorType::get(element_type); +} + +absl::StatusOr ImporterBase::InferOutputType( + const Node& node, int idx, mlir::Builder builder) { + DataType dtype = node.properties()->output_types[idx]; + + // Returns output type given inference context. + auto shape_ic = + [&](shape_inference::InferenceContext* c) -> absl::StatusOr { + // TODO(b/200093974): Post triage, consider following + // GraphConstructor::ValidateShape in checking _output_shapes always. + if (specs_.unconditionally_use_set_output_shapes) { + if (const AttrValue* attr = node.attrs().Find(kOutputShapesAttrName)) { + auto& list = attr->list(); + if (list.shape_size() > idx) { + const TensorShapeProto& p = list.shape()[idx]; + shape_inference::ShapeHandle h; + absl::Status s = c->MakeShapeFromShapeProto(p, &h); + if (!s.ok()) + return absl::InvalidArgumentError( + absl::StrCat("Node '", node.name(), " has an invalid ", + kOutputShapesAttrName, " attribute (shape #", idx, + " error:'", s.message(), "')")); + c->set_output(idx, h); + } + } + } + + return ConvertDataTypeAndShape(dtype, c->output(idx), + c->output_handle_shapes_and_types(idx), c, + builder); + }; + + if (specs_.enable_shape_inference) { + // TODO(jpienaar): Remove this if shape inference on import flag is removed. + shape_inference::InferenceContext* shape_context = + shape_refiner_->GetContext(&node); + return shape_ic(shape_context); + } + + // Treat TensorList init ops specially here as the op requires knowing its + // element dtype. + // TODO(jpienaar): Reconsider post refactoring shape functions. + if (node.type_string() == "TensorListReserve" || + node.type_string() == "EmptyTensorList") { + mlir::Type etype; + if (auto element_dtype = node.attrs().Find("element_dtype")) { + TF_RETURN_IF_ERROR( + ConvertDataType(element_dtype->type(), builder, &etype)); + } + return GetTypeFromTFTensorShape( + {}, mlir::TF::VariantType::get({mlir::UnrankedTensorType::get(etype)}, + etype.getContext())); + } + + if (node.IsWhileNode()) { + auto* output_shapes = node.attrs().Find("output_shapes"); + auto* element_types = node.attrs().Find("T"); + if (output_shapes && !output_shapes->list().shape().empty()) { + const auto& output_shape = output_shapes->list().shape(idx); + const auto& element_type = element_types->list().type(idx); + return ConvertToMlirTensorType(output_shape, element_type, &builder); + } + } + + auto type_from_array_attr = [&node, &idx, &builder]( + absl::string_view output_shape_attr, + absl::string_view element_type_attr) { + auto* output_shapes = node.attrs().Find(output_shape_attr); + auto* element_types = node.attrs().Find(element_type_attr); + const auto& output_shape = output_shapes->list().shape(idx); + const auto& element_type = element_types->list().type(idx); + return ConvertToMlirTensorType(output_shape, element_type, &builder); + }; + + if (node.type_string() == "IteratorGetNext" || + node.type_string() == "IteratorGetNextSync" || + node.type_string() == "MultiDeviceIteratorGetNextFromShard") + return type_from_array_attr("output_shapes", "output_types"); + + if (node.type_string() == "InfeedDequeueTuple") + return type_from_array_attr("shapes", "dtypes"); + + if (node.type_string() == "InfeedDequeue") { + assert(idx == 0); + const auto& output_shape = node.attrs().Find("shape")->shape(); + const auto& element_type = node.attrs().Find("dtype")->type(); + return ConvertToMlirTensorType(output_shape, element_type, &builder); + } + + // Returns a simple, more conservative unranked tensor type. + auto default_type = [&]() -> absl::StatusOr { + mlir::Type element_type; + TF_RETURN_IF_ERROR(ConvertDataType(dtype, builder, &element_type)); + + // TODO(b/200093974): Post triage, consider following + // GraphConstructor::ValidateShape in checking _output_shapes. + if (specs_.unconditionally_use_set_output_shapes) { + if (const AttrValue* attr = node.attrs().Find(kOutputShapesAttrName)) { + auto& list = attr->list(); + if (list.shape_size() > idx) { + llvm::SmallVector shape; + const TensorShapeProto& shape_proto = list.shape()[idx]; + if (shape_proto.unknown_rank()) + return mlir::UnrankedTensorType::get(element_type); + TF_RETURN_IF_ERROR(ConvertToMlirShape(shape_proto, &shape)); + return GetTypeFromTFTensorShape(shape, element_type); + } + } + } + + return mlir::UnrankedTensorType::get(element_type); + }; + + // Below we only try and do some shape inference for "source" ops which have + // no inputs. + if (node.num_inputs() > 0) return default_type(); + + // Do some simply inference here to get the function arguments correct for + // this common case. + // TODO(jpienaar): Reconsider post refactoring shape functions. + if (node.IsArg()) { + if (dtype == DT_RESOURCE) { + const AttrValue* dtype_attr = node.attrs().Find("_handle_dtypes"); + const AttrValue* shape_attr = node.attrs().Find("_handle_shapes"); + if (dtype_attr && shape_attr) { + if (dtype_attr->list().type().empty()) { + return absl::InvalidArgumentError(absl::StrCat( + "Invalid \"_handle_dtypes\" attribute value for _Arg node: ", + shape_attr->DebugString())); + } + if (shape_attr->list().shape().empty()) { + return absl::InvalidArgumentError(absl::StrCat( + "Invalid \"_handle_shapes\" attribute value for _Arg node: ", + shape_attr->DebugString())); + } + DataType dtype = dtype_attr->list().type(0); + const TensorShapeProto& shape_proto = shape_attr->list().shape(0); + TF_ASSIGN_OR_RETURN( + auto etype, ConvertToMlirTensorType(shape_proto, dtype, &builder)); + return mlir::UnrankedTensorType::get(mlir::TF::ResourceType::get( + {mlir::cast(etype)}, builder.getContext())); + } else { + return mlir::UnrankedTensorType::get( + mlir::TF::ResourceType::get(builder.getContext())); + } + } else if (auto shape = node.attrs().Find("_output_shapes")) { + if (shape->has_list() && shape->list().shape_size() == 1) { + return ConvertToMlirTensorType(shape->list().shape().at(0), dtype, + &builder); + } + } + } + + const tensorflow::OpRegistrationData* op_reg_data; + TF_RETURN_IF_ERROR( + graph_->op_registry()->LookUp(node.type_string(), &op_reg_data)); + if (!op_reg_data) { + DVLOG(1) << "Skipping inference for unregistered op " << node.type_string(); + return default_type(); + } + if (op_reg_data->shape_inference_fn == nullptr) { + DVLOG(1) << "Skipping inference for op without shape function " + << node.type_string(); + return default_type(); + } + shape_inference::InferenceContext c(graph_->versions().producer(), + node.attrs(), op_reg_data->op_def, + std::vector{}, {}, + /*input_tensors_as_shapes=*/{}, {}); + TF_RETURN_IF_ERROR(c.Run(op_reg_data->shape_inference_fn)); + return shape_ic(&c); +} + +absl::StatusOr ImporterBase::ConvertDataTypeAndShape( + DataType dtype, const shape_inference::ShapeHandle& handle, + const std::vector* handle_subtypes, + shape_inference::InferenceContext* context, mlir::Builder builder) { + TF_ASSIGN_OR_RETURN(auto subtypes, + ConvertSubtypes(handle_subtypes, context, builder)); + + mlir::Type element_type; + if (dtype == DT_VARIANT) + element_type = mlir::TF::VariantType::get(subtypes, context_); + else if (dtype == DT_RESOURCE) + element_type = mlir::TF::ResourceType::get(subtypes, context_); + else + TF_RETURN_IF_ERROR( + ::tensorflow::ConvertDataType(dtype, builder, &element_type)); + + return ConvertElementTypeAndShape(element_type, handle, context, builder); +} + +absl::StatusOr ImporterBase::ConvertElementTypeAndShape( + mlir::Type element_type, const shape_inference::ShapeHandle& handle, + shape_inference::InferenceContext* context, mlir::Builder builder) { + if (!context->RankKnown(handle)) { + return mlir::UnrankedTensorType::get(element_type); + } + + // Sentinel for an unknown dimension size. getTensorType interprets any + // negative value as an unknown dimension. + // TODO(jmolloy): Ideally this shouldn't be a local sentinel. + const int64_t kUnknownDim = -1; + + absl::InlinedVector dimensions; + int32_t rank = context->Rank(handle); + dimensions.reserve(rank); + for (int i = 0; i < rank; ++i) { + auto dim_handle = context->Dim(handle, i); + if (!context->ValueKnown(dim_handle)) + dimensions.push_back(kUnknownDim); + else + dimensions.push_back(context->Value(dim_handle)); + } + + return GetTypeFromTFTensorShape( + llvm::ArrayRef(dimensions.begin(), dimensions.end()), element_type); +} + +absl::StatusOr ImporterBase::ConvertSubtypes( + const std::vector* handle_subtypes, + shape_inference::InferenceContext* context, mlir::Builder builder) { + ElementSubtypes subtypes; + if (!handle_subtypes) return subtypes; + + subtypes.reserve(handle_subtypes->size()); + for (const auto& subtype : *handle_subtypes) { + mlir::Type element_type; + TF_RETURN_IF_ERROR( + ::tensorflow::ConvertDataType(subtype.dtype, builder, &element_type)); + TF_ASSIGN_OR_RETURN(TensorType type, + ConvertElementTypeAndShape(element_type, subtype.shape, + context, builder)); + subtypes.push_back(type); + } + return subtypes; +} + +absl::Status ImporterBase::ConvertFunctionCallAttribute( + const std::string& base_name, const AttrValue& value, + NamedAttrList* attributes) { + TF_ASSIGN_OR_RETURN(auto func_attr, + ConvertFunctionCallName(value.func().name())); + if (!func_attr) return absl::OkStatus(); + attributes->push_back(builder_.getNamedAttr(base_name, func_attr)); + + for (const auto& it : value.func().attr()) { + auto name = absl::StrCat(base_name, ".", it.first); + TF_ASSIGN_OR_RETURN(auto value, ConvertAttributeValue(it.second)); + attributes->push_back(builder_.getNamedAttr(name, value)); + } + return absl::OkStatus(); +} + +absl::StatusOr ImporterBase::ConvertFunctionCallName( + const std::string& func_name) { + // Some ops like XlaHostCompute op uses empty value to represent missing + // functions. Such attribute values should be defined optional in MLIR + // definition. + if (func_name.empty()) return mlir::FlatSymbolRefAttr(); + + TF_RETURN_IF_ERROR(ConvertLibFunction(func_name)); + auto mlir_func_name = (*tf_name_to_mlir_name_)[func_name]; + return mlir::SymbolRefAttr::get(builder_.getContext(), mlir_func_name); +} + +absl::StatusOr ImporterBase::ConvertAttributeValue( + const AttrValue& value) { + switch (value.value_case()) { + case AttrValue::kFunc: { + // TODO(b/156546237): Unify kFunc/NameAttrList attribute representation. + // Currently kFunc/NameAttrList attributes in a kList/repeated AttrValue + // will not use this representation. This also doesn't handle empty + // function values like ConvertFunctionCallName method. + NamedAttrList attrs; + for (const auto& func_attr : value.func().attr()) { + TF_ASSIGN_OR_RETURN( + auto attr, ImporterBase::ConvertAttributeValue(func_attr.second)); + attrs.push_back(builder_.getNamedAttr(func_attr.first, attr)); + } + auto func_attrs = builder_.getDictionaryAttr(attrs); + return mlir::TF::FuncAttr::get(context_, value.func().name(), func_attrs); + } + case AttrValue::kList: { + if (!value.list().func().empty()) { + absl::InlinedVector attrs; + for (const auto& item : value.list().func()) { + TF_ASSIGN_OR_RETURN(auto attr, ConvertFunctionCallName(item.name())); + if (item.attr_size() != 0) + return absl::UnimplementedError( + "func attributes with non-zero attr.size()"); + if (attr) attrs.push_back(attr); + } + return builder_.getArrayAttr( + llvm::ArrayRef(attrs.begin(), attrs.end())); + } + return ConvertNonFuncAttributeValue(value, &builder_); + } + default: + return ConvertNonFuncAttributeValue(value, &builder_); + } +} + +void ImporterBase::GetArgsAndRetsFromFunctionBody( + const FunctionBody& fbody, absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes, + absl::InlinedVector* control_ret_nodes) { + arg_nodes->reserve(fbody.arg_nodes.size()); + ret_nodes->reserve(fbody.ret_nodes.size()); + for (auto arg : fbody.arg_nodes) { + arg_nodes->emplace_back(arg, 0); + } + for (auto ret : fbody.ret_nodes) { + ret_nodes->emplace_back(ret, 0); + } + *control_ret_nodes = fbody.control_ret_nodes; +} + +absl::Status ImporterBase::ConvertLibFunction(llvm::StringRef func_name) { + // If the library function has been converted already, nothing needs to be + // done. + if (tf_name_to_mlir_name_->find(std::string(func_name)) != + tf_name_to_mlir_name_->end()) + return absl::OkStatus(); + + std::string mlir_func_name( + function_name_uniquifier_->GetUniqueName(func_name)); + (*tf_name_to_mlir_name_)[std::string(func_name)] = mlir_func_name; + + const auto& func_lib = graph_flib_; + const auto* func_def = func_lib.Find(std::string(func_name)); + if (func_def == nullptr) { + return absl::FailedPreconditionError( + absl::StrCat("Failed to find function '", StringRefToView(func_name), + "'. The imported TensorFlow GraphDef is ill-formed.")); + } + + // Converts the argument and return types to MLIR types. + std::vector attributes; + attributes.reserve(func_def->attr_size()); + for (const auto& name_and_value : func_def->attr()) { + // This is a function definition attribute, so it shouldn't contain + // kFunc attribute and it is treated as normal one. + TF_ASSIGN_OR_RETURN(auto attr, + ConvertAttributeValue(name_and_value.second)); + std::string attr_name = + mangling_util::MangleAttributeName(name_and_value.first); + attributes.push_back(builder_.getNamedAttr(attr_name, attr)); + } + + // Checks opdef stateful attribute and import that as Function Attribute + if (func_def->signature().is_stateful()) { + auto stateful_str = mlir::TF::TensorFlowDialect::GetStatefulAttrName(); + attributes.push_back( + builder_.getNamedAttr(stateful_str, builder_.getUnitAttr())); + } + + // Checks for an associated custom gradient function. Adds it to the attribute + // list of this function. + auto grad_func_name = func_lib.FindGradient(std::string(func_name)); + if (!grad_func_name.empty()) { + TF_RETURN_IF_ERROR(ConvertLibFunction(grad_func_name)); + auto mlir_grad_func_name = (*tf_name_to_mlir_name_)[grad_func_name]; + auto gradient_attr = + mlir::SymbolRefAttr::get(builder_.getContext(), mlir_grad_func_name); + auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); + attributes.push_back(builder_.getNamedAttr(grad_string, gradient_attr)); + } + + deferred_functions_.emplace(func_name.str(), attributes); + return absl::OkStatus(); +} + +absl::Status ImporterBase::PruneUnreachableNodes( + std::unordered_map* node_name_map) { + std::unordered_set prune_start; + TF_RETURN_IF_ERROR(GetInputOutputNodes(*node_name_map, &prune_start)); + + if (!prune_start.empty()) { + if (PruneForReverseReachability(graph_.get(), prune_start)) { + VLOG(1) << "Pruned unused nodes in graphdef"; + } else { + VLOG(1) << "No unused nodes in graphdef to prune"; + } + } else { + VLOG(1) << "No output nodes specified, skipping pruning"; + } + return absl::OkStatus(); +} + +absl::Status ImporterBase::ConvertFeedsToPlaceholders( + std::unordered_map* node_name_map) { + // Feeds (edges) are converted into single-output placeholder nodes to + // simplify the conversion process. + TF_ASSIGN_OR_RETURN(auto feeds_by_node, GetFeedsByNode(specs_.inputs)); + for (const auto& it : feeds_by_node) { + TensorId tensor = ParseTensorName(it.first); + auto jt = node_name_map->find(std::string(tensor.node())); + if (jt == node_name_map->end()) { + return absl::FailedPreconditionError( + absl::StrCat("Graph does not contain node: ", tensor.node())); + } + + Node* node = jt->second; + auto op_name = node->op_def().name(); + if (op_name != "Placeholder" && op_name != "LegacyFedInput" && + op_name != FunctionLibraryDefinition::kArgOp) { + for (const auto& output_tensor : it.second) { + const int index = output_tensor.first; + const ArrayInfo& array_info = output_tensor.second->second; + + DataType dtype = array_info.imported_dtype; + // Uses the existing output type if it isn't specified by the user. + if (dtype == DT_INVALID) { + dtype = node->output_type(index); + } + + TF_ASSIGN_OR_RETURN( + auto placeholder_node_and_removed, + CreatePlaceholderNodeForFeed(array_info.shape, dtype, node, index, + *node_name_map)); + + Node* placeholder_node = placeholder_node_and_removed.first; + if (placeholder_node->in_edges().empty()) { + graph_->AddControlEdge(graph_->source_node(), placeholder_node, + true /* skip test for duplicates */); + } + if (placeholder_node->out_edges().empty()) { + graph_->AddControlEdge(placeholder_node, graph_->sink_node(), + true /* skip test for duplicates */); + } + remapped_feeds_[{it.first, index}] = placeholder_node->name(); + (*node_name_map)[placeholder_node->name()] = placeholder_node; + } + } + } + return absl::OkStatus(); +} + +absl::Status ImporterBase::PrepareConvert(const Graph& graph, + std::unique_ptr graph_def) { + // TODO(fengliuai): Converting to GraphDef and back is the easiest way to + // clone a graph. + // TODO(fengliuai): clone the graph without going to graph_def first. + if (graph_def == nullptr) { + graph_def = std::make_unique(); + graph.ToGraphDef(graph_def.get()); + } + graph_ = std::make_unique(graph.flib_def()); + GraphConstructorOptions opts; + opts.allow_internal_ops = true; + opts.add_default_attributes = true; + TF_RETURN_IF_ERROR(::tensorflow::ConvertGraphDefToGraph( + opts, std::move(*graph_def), graph_.get())); + + TF_RETURN_IF_ERROR(RemoveBackedges()); + + TF_RETURN_IF_ERROR(CopyStackTraces(graph, graph_.get())); + + auto node_name_map = graph_->BuildNodeNameIndex(); + + if (specs_.enable_shape_inference) { + // TODO(jpienaar): Remove once infer shapes on import flag is removed. + TF_RETURN_IF_ERROR(AddNodesToShapeRefiner(&node_name_map)); + } else { + TF_RETURN_IF_ERROR(ConvertFeedsToPlaceholders(&node_name_map)); + } + + // Prune nodes in the graph that are not reachable from the output. + if (specs_.prune_unused_nodes) { + TF_RETURN_IF_ERROR(PruneUnreachableNodes(&node_name_map)); + } + + if (!specs_.enable_shape_inference) { + // Re-initialize ordered_nodes_ since we might have modified the graph. + ordered_nodes_.clear(); + TopologicalOrdering( + *graph_, [&](Node* n) { ordered_nodes_.push_back(n); }, + GroupByDevice()); + } + + return absl::OkStatus(); +} + +absl::Status ImporterBase::Convert( + llvm::StringRef func_name, mlir::FunctionType func_type, + const absl::InlinedVector& arg_nodes, + const absl::InlinedVector& ret_nodes, + const absl::InlinedVector& control_ret_nodes, + llvm::ArrayRef attrs) { + // TODO(b/122040776): Uses debug info for FunctionDef. + auto function = mlir::func::FuncOp::create(mlir::UnknownLoc::get(context_), + func_name, func_type, attrs); + + module_.push_back(function); + // Seeds the builder with an initial block. + function.addEntryBlock(); + builder_ = mlir::OpBuilder(function.getBody()); + + // Create the graph operation in which we will convert the individual nodes. + auto graph = mlir::tf_executor::GraphOp::create(builder_, function.getLoc(), + func_type.getResults()); + builder_.createBlock(&graph.getBody()); + + for (const Node* node : ordered_nodes_) { + TF_RETURN_IF_ERROR(ConvertNode(*node)); + } + + // Adds the backedges back to the function by creating the source and sink + // pairs. + TF_RETURN_IF_ERROR(AddBackedges()); + + TF_RETURN_IF_ERROR(ConvertFunctionArgAndRets(function, graph, + func_type.getInputs(), arg_nodes, + ret_nodes, control_ret_nodes)); + + // TODO(jpienaar): Update post removing shape_refinier_. + if (!specs_.enable_shape_inference) { + // Refine graph's type given more precise fetch. + auto fetch = graph.GetFetch(); + bool all_equal = true; + for (auto it : + llvm::zip_first(graph.getResults(), fetch.getOperandTypes())) { + auto rt = std::get<1>(it); + if (rt == std::get<0>(it).getType()) continue; + std::get<0>(it).setType(rt); + all_equal = false; + } + if (!all_equal) { + function.setType(mlir::FunctionType::get(function.getContext(), + func_type.getInputs(), + graph.getResultTypes())); + } + } + + return absl::OkStatus(); +} + +absl::Status ImporterBase::ConvertFunctionArgAndRets( + mlir::func::FuncOp func, mlir::tf_executor::GraphOp graph_op, + llvm::ArrayRef arg_types, + const absl::InlinedVector& arg_nodes, + const absl::InlinedVector& ret_nodes, + const absl::InlinedVector& control_ret_nodes) { + // Store the arg/return attributes as a list rather than uniqueuing during + // construction. + llvm::SmallVector arg_attrs; + arg_attrs.resize(func.getNumArguments()); + llvm::SmallVector ret_attrs; + ret_attrs.resize(func.getNumResults()); + + auto set_attributes_on_func = [&](Node* node, int64_t index, bool is_arg) { + for (const auto& node_attr : node->attrs()) { + const auto& key = node_attr.first; + // Only import optional attributes (e.g., those starting with an + // underscore). + if (key.empty() || key[0] != '_') continue; + // Ignore shape inference attributes as shape information is already + // populated in the result type. + if (IsOutputShapesAttribute(node_attr.second, key) || + IsResourceOutputShapesAttribute(node_attr.second, key)) + continue; + TF_ASSIGN_OR_RETURN(auto converted_attr, + ConvertAttributeValue(node_attr.second)); + std::string dialect_attribute = "tf." + key; + if (is_arg) { + arg_attrs[index].set(dialect_attribute, converted_attr); + } else { + func.setResultAttr(index, dialect_attribute, converted_attr); + ret_attrs[index].set(dialect_attribute, converted_attr); + } + } + return absl::OkStatus(); + }; + + auto* bb = &func.front(); + llvm::SmallDenseMap, mlir::Value, 4> + arg_nodes_to_values; + for (int i = 0, e = arg_types.size(); i < e; ++i) { + auto& arg_node = arg_nodes[i]; + // The lookup can't fail here: otherwise some nodes in the function haven't + // be converted to mlir operations and don't have a mapping. + mlir::Operation* island = node_values_.find(arg_node.node->id())->second; + + auto bb_arg = bb->getArgument(i); + mlir::Value arg_def = bb_arg; + + if (island->getNumResults() != 2) + return absl::InvalidArgumentError( + "Only feed output tensors of single output nodes are supported"); + + // Collect mapping of OutputTensor to associated block arg. + arg_nodes_to_values.try_emplace({arg_node.node, arg_node.index}, arg_def); + island->getResult(0).replaceAllUsesWith(arg_def); + // Erase control outputs from feed. + auto control_uses = island->getResult(1).getUses(); + for (auto& control_use : llvm::make_early_inc_range(control_uses)) + control_use.getOwner()->eraseOperand(control_use.getOperandNumber()); + + if (!arg_node.node->requested_device().empty()) + arg_attrs[i].set("tf.device", builder_.getStringAttr( + arg_node.node->requested_device())); + + if (arg_node.node->IsArg()) { + TF_RETURN_IF_ERROR( + set_attributes_on_func(arg_node.node, i, /*is_arg=*/true)); + } + + island->dropAllReferences(); + island->erase(); + } + + llvm::SmallVector inst_to_return; + for (const auto& ret_and_idx : llvm::enumerate(ret_nodes)) { + const auto& ret = ret_and_idx.value(); + auto* inst = node_values_[ret.node->id()]; + if (ret.node->IsRetval()) { + if (!ret.node->requested_device().empty()) + ret_attrs[ret_and_idx.index()].set( + "tf.device", builder_.getStringAttr(ret.node->requested_device())); + TF_RETURN_IF_ERROR(set_attributes_on_func(ret.node, ret_and_idx.index(), + /*is_arg=*/false)); + // Lookup the instruction inside the island + auto island_op = llvm::cast(inst); + mlir::Operation* inner_op = &island_op.GetBody().front(); + // Remove kRetOp or kDeviceRetOp operation and return its operand. + // kRetOp and kDeviceRetOp should have just one operand unless they have + // control dependencies. + if (inner_op->getNumOperands() != 1) + return absl::UnimplementedError("Return node with multiple inputs."); + inst_to_return.push_back(inner_op->getOperand(0)); + inst->dropAllReferences(); + inst->erase(); + } else { + // Lookup and use block arg if fetch is a feed. + auto it = arg_nodes_to_values.find({ret.node, ret.index}); + if (it != arg_nodes_to_values.end()) + inst_to_return.push_back(it->second); + else + inst_to_return.push_back(inst->getResult(ret.index)); + } + } + + for (Node* control_ret : control_ret_nodes) { + auto* inst = node_values_[control_ret->id()]; + inst_to_return.push_back(*std::prev(inst->result_end())); + } + + // Terminate the function by adding a Fetch operation to terminate the graph + // and a return operation to return the Graph results. + builder_.setInsertionPointToEnd(&graph_op.getBody().front()); + mlir::tf_executor::FetchOp::create(builder_, graph_op.getLoc(), + inst_to_return); + builder_.setInsertionPointToEnd(bb); + mlir::func::ReturnOp::create(builder_, mlir::UnknownLoc::get(context_), + graph_op.getResults()); + + func.setAllArgAttrs( + llvm::to_vector<4>(llvm::map_range(arg_attrs, [&](NamedAttrList& list) { + return list.getDictionary(context_); + }))); + func.setAllResultAttrs( + llvm::to_vector<4>(llvm::map_range(ret_attrs, [&](NamedAttrList& list) { + return list.getDictionary(context_); + }))); + + return absl::OkStatus(); +} + +mlir::Location ImporterBase::GetLocation(const Node& node) { + DVLOG(1) << "Getting location for " << node.name() << " " << &node; + // TODO(b/142400497): What is the semantic contract for locations? + // Create a location for node `name` in function `function_name`. + auto create_location = [&](llvm::StringRef name, + llvm::StringRef function_name) -> mlir::Location { + // Use the catenation of function and node names as the lookup key into the + // debug info. This matches the way that the key is formed on the python + // side. + // + // We also use this as the name for the NameLoc for ops in function, since + // otherwise our names could collide across functions. + // For ops in the main graph, we omit the "@function_name" (which, would be + // just "@" since function_name would be empty) because some code seems to + // depend on the name being this way for correctness. + std::string debug_info_key = (name + "@" + function_name).str(); + std::string name_for_name_loc = + function_name.empty() ? name.str() : debug_info_key; + auto name_loc_id = mlir::StringAttr::get(context_, name_for_name_loc); + + std::shared_ptr stack_trace = node.GetStackTrace(); + + // Prefer stack traces if available, fallback to debug info if not, and then + // finally to just name. Older versions of debug info concatenated `@` onto + // the node name for the default graph, so we check both locations. + if (stack_trace != nullptr) { + } else if (stack_traces_.contains(name_for_name_loc)) { + stack_trace = stack_traces_.at(name_for_name_loc); + } else if (stack_traces_.contains(debug_info_key)) { + stack_trace = stack_traces_.at(debug_info_key); + } else { + DVLOG(1) << "No stack trace for " << node.name(); + } + + llvm::SmallVector locations; + + if (stack_trace != nullptr) { + DVLOG(1) << "Stack available for " << node.name(); + for (const StackFrame& frame : stack_trace->ToUncachedFrames()) { + auto file_name = mlir::StringAttr::get(context_, frame.file_name); + // Use col 1 as there is no column info in StackTrace. + auto file_line_loc = + mlir::FileLineColLoc::get(file_name, frame.line_number, 1); + locations.push_back(file_line_loc); + } + } + + // If there are no locations in the stack trace, fall back to just a + // NameLoc with no child. + if (locations.empty()) return mlir::NameLoc::get(name_loc_id); + + // Use the front FileLineColLoc to generate a NameLoc. + mlir::Location node_name_loc = + mlir::NameLoc::get(name_loc_id, locations.front()); + + // If there are more locations then generate a stack trace, otherwise just + // return the name loc. + auto callsite_locs = llvm::ArrayRef(locations).drop_front(); + return callsite_locs.empty() + ? node_name_loc + : mlir::CallSiteLoc::get(node_name_loc, callsite_locs); + }; + + // Create a location for node `name` in function `function_name`. + auto create_op_type_and_name_locations = [&]() { + return mlir::FusedLoc::get( + context_, + // Add the type operation for the propagation of op_type metadata. + {mlir::NameLoc::get( + mlir::StringAttr::get(context_, node.type_string() + ":")), + create_location(node.name(), function_name_for_debug_info_)}); + }; + + // For NextIteration nodes, location is used to pair source and sink nodes. + // Hence, we use node name as location to keep it unique. + // TODO(prakalps): In future the plan is to use tokens to pair source/sink + // nodes. Then NextIteration nodes would not need to be handled separately. + if (node.type_string() == "NextIteration") { + return create_op_type_and_name_locations(); + } + + const auto& node_def = node.def(); + auto original_nodes = + node_def.experimental_debug_info().original_node_names(); + auto original_funcs = + node_def.experimental_debug_info().original_func_names(); + + if (original_nodes.empty()) { + return create_op_type_and_name_locations(); + } else { + // If the original nodes are defined, then we use them to get a list of + // call sites, and then fuse them to a single fused location, with the name + // of the node_def. + llvm::SmallVector node_locations; + node_locations.reserve(original_nodes.size() + 2); + // Add the type operation for the propagation of op_type metadata. + node_locations.push_back(mlir::NameLoc::get( + mlir::StringAttr::get(context_, node.type_string() + ":"))); + // Retrieve the names from the experimental_debug_info. + for (int i = 0, e = original_nodes.size(); i != e; ++i) { + const auto& node_name = original_nodes[i]; + auto func_name = (i < original_funcs.size()) ? original_funcs[i] : ""; + node_locations.push_back(create_location(node_name, func_name)); + } + // Retrieve the name of the node_def. + node_locations.push_back( + create_location(node.name(), function_name_for_debug_info_)); + return mlir::FusedLoc::get(context_, node_locations); + } +} + +absl::Status ImporterBase::EmitErrorWithLocationStr( + const Node& node, const absl::Status& error_status) { + const mlir::Location location = GetLocation(node); + mlir::emitError(location); + return error_handler_.Combine(error_status); +} + +mlir::Operation* ImporterBase::CreateOperation( + const Node& node, llvm::StringRef node_type_name, + const mlir::OperationState& result, + const llvm::SmallVectorImpl& control_operands) { + // For the tf.executor specific operations (not wrapped in an island), we + // have an extra returned value for the control result, and we concatenate + // control and non-control operands. + mlir::SmallVector types(result.types); + types.push_back(mlir::tf_executor::ControlType::get(builder_.getContext())); + mlir::SmallVector operands(result.operands); + operands.append(control_operands.begin(), control_operands.end()); + + auto loc = result.location; + // Dispatch based on the name and create the appropriate operation. + if (node.IsSwitch()) { + // Switch and _SwitchN both are in switch class, differentiate based on + // op name. + if (node.op_def().name() == "_SwitchN") { + return mlir::tf_executor::SwitchNOp::create(builder_, loc, types, + operands, result.attributes); + } + return mlir::tf_executor::SwitchOp::create(builder_, loc, types, operands, + result.attributes); + } + if (node.IsMerge()) { + return mlir::tf_executor::MergeOp::create(builder_, loc, types, operands, + result.attributes); + } + if (node.IsNextIteration()) { + // NextIteration is a bit special, we create a pair of operations that are + // linked together through a token returned by the source. + // We make use of a separate builder to insert the source at the top of + // the block. + mlir::OpBuilder builder_at_begin(builder_.getBlock(), + builder_.getBlock()->begin()); + auto source_op = mlir::tf_executor::NextIterationSourceOp::create( + builder_at_begin, loc, operands[0].getType(), result.attributes); + return mlir::tf_executor::NextIterationSinkOp::create( + builder_, loc, source_op.getToken(), operands, result.attributes); + } + if (node.IsLoopCond()) { + return mlir::tf_executor::LoopCondOp::create(builder_, loc, types, operands, + result.attributes); + } + if (node.IsEnter()) { + return mlir::tf_executor::EnterOp::create(builder_, loc, types, operands, + result.attributes); + } + if (node.IsExit()) { + return mlir::tf_executor::ExitOp::create(builder_, loc, types, operands, + result.attributes); + } + if (node.IsControlTrigger()) { + return mlir::tf_executor::ControlTriggerOp::create( + builder_, loc, mlir::ValueRange(operands), result.attributes); + } + // Regular TensorFlow operation are wrapped in a tf_executor.island. + auto island = mlir::tf_executor::IslandOp::create( + builder_, result.location, types, control_operands, + mlir::ArrayRef{}); + island.getBody().push_back(new mlir::Block); + mlir::OpBuilder island_builder = + mlir::OpBuilder::atBlockEnd(&island.GetBody()); + + // Create the operation inside the island now. + mlir::Operation* inner_op = island_builder.create(result); + + // Sets operand_segment_sizes or result_segment_sizes attribute to the op. + const auto set_segment_sizes_attr = + [&](const NameRangeMap& arg_ranges, + const protobuf::RepeatedPtrField& args, + llvm::StringRef attr_name) { + std::vector values; + values.reserve(args.size()); + for (const auto& arg : args) { + auto range = arg_ranges.at(arg.name()); + values.push_back(range.second - range.first); + } + auto attr_value = + mlir::DenseI32ArrayAttr::get(inner_op->getContext(), values); + inner_op->setAttr(attr_name, attr_value); + }; + + if (inner_op->hasTrait() || + inner_op->hasTrait()) { + // The op has multiple variadic operands or results. + // Calculate operand and result segment sizes using the OpDef. + NameRangeMap input_ranges, output_ranges; + // This will fail only if the OpDef is syntactically invalid. + // TODO(jpienaar): Convert this CHECK into a properly propagated error. + CHECK_OK( + NameRangesForNode(node, node.op_def(), &input_ranges, &output_ranges)); + if (inner_op->hasTrait()) { + // Add derived "operand_segment_sizes" attr to the created operation. + // TODO(b/146937733): Don't use here. + set_segment_sizes_attr(input_ranges, node.op_def().input_arg(), + mlir::OpTrait::AttrSizedOperandSegments< + void>::getOperandSegmentSizeAttr()); + } + + if (inner_op->hasTrait()) { + // Add derived "result_segment_sizes" attr to the created operation. + // TODO(b/146937733): Don't use here. + set_segment_sizes_attr(output_ranges, node.op_def().output_arg(), + mlir::OpTrait::AttrSizedResultSegments< + void>::getResultSegmentSizeAttr()); + } + } + + if (VLOG_IS_ON(1)) { + mlir::OperationName name = inner_op->getName(); + if (!name.isRegistered() && + // Skip unmodelled ops that are handled differently. + (node_type_name != "_Arg" && node_type_name != "_Retval") && + !unmodelled_op_names_.count(name.getIdentifier())) { + if (node.op_def().is_stateful()) { + VLOG(1) << "[potentially conservative] Op type `" << node.type_string() + << "` is stateful but effects not modelled"; + } else { + // See if any resource type is used. + bool resource = false; + std::function record_resource; + record_resource = [&](mlir::Type type) { + type.walk([&](mlir::Type t) { + if (resource) return mlir::WalkResult::interrupt(); + if (mlir::isa(type)) { + resource = true; + return mlir::WalkResult::interrupt(); + } + + return mlir::WalkResult::advance(); + }); + + return resource; + }; + + for (mlir::Type t : inner_op->getResultTypes()) + if (record_resource(t)) break; + for (mlir::Type t : inner_op->getOperandTypes()) + if (record_resource(t)) break; + if (resource) { + unmodelled_op_names_.insert(name.getIdentifier()); + VLOG(1) << "[potentially conservative] Op type `" + << node.type_string() + << "` has resource operands/results but effects not modelled"; + } + } + } + } + + // Add the terminator for the island + mlir::tf_executor::YieldOp::create(island_builder, result.location, + inner_op->getResults()); + return island.getOperation(); +} + +// This function detects Identity nodes created by the XlaMetadataLayer in +// xla_metadata.py. It copies attributes prefixed with "tf.xla_metadata_" from +// the Identity node to the MLIR op corresponding to the Identity node's input. +// This allows propagating XLA metadata from the Python side to the MLIR +// representation. These attributes are later processed in +// core/tpu/tpu_compile.cc, where they are used to populate the +// frontend_attributes of the HLO instructions. +absl::Status ImporterBase::HandleXlaMetadata(const Node& node) { + absl::string_view node_name = node.name(); + if (absl::EndsWith(node_name, "xla_metadata_marker") && + node.op_def().name() == "Identity") { + const Edge* data_edge = nullptr; + for (const Edge* edge : node.in_edges()) { + if (!edge->IsControlEdge()) { + data_edge = edge; + break; + } + } + + if (data_edge == nullptr) { + return absl::FailedPreconditionError( + absl::StrCat("XLA metadata identity node ", node.name(), + " has no data input edge")); + } + + const Node& input_node = *data_edge->src(); + if (node_values_.find(input_node.id()) == node_values_.end()) { + return absl::FailedPreconditionError(absl::StrCat( + "XLA metadata identity node ", node.name(), "'s input node ", + input_node.name(), " hasn't been converted to MLIR yet.")); + } + + mlir::Operation* input_op = node_values_[input_node.id()]; + mlir::Operation* op_to_set_attr = input_op; + if (auto island_op = + llvm::dyn_cast(input_op)) { + op_to_set_attr = &island_op.getBody().front().front(); + } + for (const auto& [attr_name, attr_value] : node.attrs()) { + if (absl::StartsWith(attr_name, "tf.xla_metadata_")) { + std::string attr_value_str = attr_value.s(); + mlir::StringAttr attr_value = builder_.getStringAttr(attr_value_str); + op_to_set_attr->setAttr(attr_name, attr_value); + } + } + } + + return absl::OkStatus(); +} + +absl::Status ImporterBase::ConvertNode(const Node& node) { + if (!node.IsOp()) { + // Don't import the pseudo-nodes _SOURCE or _SINK. These are added by + // Graph and don't exist in GraphDef. + return absl::OkStatus(); + } + TF_RETURN_IF_ERROR(HandleXlaMetadata(node)); + + // If it is a custom OP, its definition should be found in the library. We + // create the MLIR function and insert it to the module if it doesn't exist. + std::string node_type_name = node.type_string(); + const auto* func_def = graph_flib_.Find(node_type_name); + bool convert_to_legacy_call = false; + if (func_def) { + TF_RETURN_IF_ERROR(ConvertLibFunction(node_type_name)); + node_type_name = (*tf_name_to_mlir_name_)[node_type_name]; + convert_to_legacy_call = true; + } + + auto get_full_op_name = [&](const std::string& op_name) { + const char* kTfPrefix = "tf."; + return kTfPrefix + op_name; + }; + + std::string op_name = get_full_op_name(node_type_name); + if (back_edge_node_output_.contains(&node)) { + op_name = op_name + ".sink"; + } + + mlir::OperationState result(GetLocation(node), op_name); + for (int i = 0; i < node.num_outputs(); ++i) { + // The backedge has been removed, so we shouldn't count the corresponding + // output from the src node when converting to an operation. + if (back_edge_node_output_.contains(&node) && + back_edge_node_output_[&node] == i) { + continue; + } + TF_ASSIGN_OR_RETURN(auto type, InferOutputType(node, i, builder_)); + result.types.push_back(type); + } + + // Surprisingly input edges can be nondeterministically ordered. This + // particularly seems to be the case for the control edges between _SOURCE + // and _SINK that the Graph constructor inserts. Copy the input edges and + // sort the edges, but only the control edges, not data edges! + // TODO(jmolloy): We should probably just ignore _SOURCE and _SINK nodes. + // They'll break roundtripping anyway unless we strip them when converting + // back to graphdef. + absl::InlinedVector in_edges(node.in_edges().size()); + absl::c_copy(node.in_edges(), in_edges.begin()); + absl::c_stable_sort(in_edges, [](const Edge* e1, const Edge* e2) { + if (e1->IsControlEdge() && !e2->IsControlEdge()) return false; + if (!e1->IsControlEdge() && e2->IsControlEdge()) return true; + if (e1->IsControlEdge() && e2->IsControlEdge()) + return e1->src()->id() < e2->src()->id(); + return e1->dst_input() < e2->dst_input(); + }); + + result.operands.reserve(in_edges.size()); + + // Collect the control operands separately, they will be held by the island. + mlir::SmallVector control_operands; + + for (const auto* input_edge : in_edges) { + const Node& input_node = *input_edge->src(); + if (input_node.IsSource()) { + if (in_edges.size() != 1) { + return absl::FailedPreconditionError( + "The node has other inputs besides the _Source node"); + } + // We don't import the _SOURCE node. + continue; + } + if (input_node.IsArg() && input_edge->IsControlEdge()) { + // Currently we have not reached consensus as to what TF function + // semantics are (b/133509504). Here we assume that all arguments to a + // function should be available before we start execution of any internal + // node. This makes the control dependencies between function arguments + // and internal nodes redundant, and so we do not import them. The TF + // inliner however assumes no such dependency between function args and + // internal nodes exists, unless explicitly stated. Since we drop control + // dependencies here, it leads to loss of information. If the function is + // inlined later, the inliner would not know of these explicit control + // dependencies present in the original graph. + continue; + } + if (node_values_.find(input_node.id()) == node_values_.end()) + return absl::FailedPreconditionError( + "Graph not traversed in reverse post order; use seen before def!"); + mlir::Operation* inst = node_values_[input_node.id()]; + if (input_edge->IsControlEdge()) + control_operands.push_back(inst->getResult(inst->getNumResults() - 1)); + else + result.operands.push_back(inst->getResult(input_edge->src_output())); + } + + using FuncPairType = std::pair; + std::vector funcs; + result.attributes.reserve(node.attrs().size() + 2); + auto abstract_op = result.name.getRegisteredInfo(); + auto derived_op = + abstract_op + ? abstract_op->getInterface() + : nullptr; + for (const auto& name_and_value : node.attrs()) { + const auto& attr_name = name_and_value.first; + // Skip adding derived attributes to the generated op. + if (derived_op && derived_op->isDerivedAttribute(attr_name)) continue; + const AttrValue& attr_value = name_and_value.second; + + // Remove _output_shapes attribute that will be added by the exporter. + if (IsOutputShapesAttribute(attr_value, attr_name)) continue; + + if (attr_value.value_case() == AttrValue::kFunc) { + // Attribute iteration order is not defined for protocol buffer Map. + // Process function attributes separately in the lexicographical order to + // have deterministic order of functions in the constructed IR. + funcs.emplace_back(&attr_name, &attr_value); + } else { + TF_ASSIGN_OR_RETURN(auto attr, ConvertAttributeValue(attr_value)); + result.attributes.push_back(builder_.getNamedAttr(attr_name, attr)); + } + } + + auto comparator = [](const FuncPairType& a, const FuncPairType& b) { + return *a.first < *b.first; + }; + std::sort(funcs.begin(), funcs.end(), comparator); + for (const auto& func : funcs) { + TF_RETURN_IF_ERROR(ConvertFunctionCallAttribute(*func.first, *func.second, + &result.attributes)); + } + + const auto& node_def = node.def(); + // NodeDef can contain partial TF device names. In such cases, canonicalize + // it. Note that in current TF, placer will place full device name to each + // node. + DeviceNameUtils::ParsedName parsed_name; + if (!DeviceNameUtils::ParseFullName(node_def.device(), &parsed_name)) { + return absl::InvalidArgumentError(absl::StrCat( + "Op ", op_name, " has invalid device name: ", node_def.device())); + } + // Keep the parsed name untouched if the device name is empty. + if (!node_def.device().empty()) { + if (!parsed_name.has_type) { + parsed_name.type = "CPU"; + parsed_name.has_type = true; + } + if (!parsed_name.has_id) { + parsed_name.id = 0; + parsed_name.has_id = true; + } + } + result.attributes.push_back(builder_.getNamedAttr( + "device", builder_.getStringAttr( + DeviceNameUtils::ParsedNameToString(parsed_name)))); + + // Map user function calls to LegacyCall ops and add the user function name + // as an attribute. + if (convert_to_legacy_call) { + result.name = mlir::OperationName(get_full_op_name("LegacyCall"), context_); + mlir::SymbolRefAttr val = + mlir::SymbolRefAttr::get(builder_.getContext(), node_type_name); + result.addAttribute("f", val); + + if (!result.attributes.get("_disable_call_shape_inference")) { + result.addAttribute("_disable_call_shape_inference", + builder_.getBoolAttr(false)); + } + } + + auto composite_control_flow_op = [&](const std::string& name) { + result.name = mlir::OperationName(get_full_op_name(name), context_); + bool stateless = absl::StartsWith(node_type_name, "Stateless"); + mlir::BoolAttr val = builder_.getBoolAttr(stateless); + result.attributes.push_back(builder_.getNamedAttr("is_stateless", val)); + }; + + // Map Case/If/While and StatelessCase/If/While op in TensorFlow to the common + // Case/If/While op in MLIR and add the differentiating attribute. + if (node.IsCaseNode()) composite_control_flow_op("Case"); + if (node.IsIfNode()) composite_control_flow_op("If"); + if (node.IsWhileNode()) { + composite_control_flow_op("While"); + auto* output_shapes = node.attrs().Find("output_shapes"); + if (output_shapes && !output_shapes->list().shape().empty()) + result.attributes.push_back( + builder_.getNamedAttr("shape_invariant", builder_.getUnitAttr())); + } + + // Register the mapping between the TF node and the newly created operation. + node_values_[node.id()] = + CreateOperation(node, node_type_name, result, control_operands); + return absl::OkStatus(); +} + +// Add the backedges to the CFG. Given a backedge, we replace the original +// source and destination operations by two new operations. Most of the +// fields of the replacements are copied from the original operations. +// However, +// - for the src operation, one output is inserted to the front of the output +// list. The type of the output is set to the type of the non-control result +// of the dst operation, and +// - for the dst operation, one operand is inserted to the front of the +// operand list. This operand is using the first result of the src +// operation. +// TODO(fengliuai): Preserve the order of the results and operands if +// necessary. +absl::Status ImporterBase::AddBackedges() { + for (auto it : back_edge_dst_inputs_) { + BackEdge& edge = it.second; + if (!edge.src->IsNextIteration() || !edge.dst->IsMerge()) { + return absl::FailedPreconditionError( + "Invalid backedge; should be from NextIteration to Merge!"); + } + auto* sink = node_values_[edge.src->id()]; + auto* dst = node_values_[edge.dst->id()]; + TF_RETURN_IF_ERROR(AddBackedge(sink, dst, edge.dst_input)); + } + return absl::OkStatus(); +} + +absl::Status ImporterBase::AddBackedge(mlir::Operation* sink, + mlir::Operation* dst, int dst_input) { + // Get the NextIteration.Source operation from the token operand of the sink. + mlir::Operation* source = sink->getOperand(0).getDefiningOp(); + + // Adds the "source" to the operands of the dst by creating a new dst + // operation. + mlir::OperationState state(dst->getLoc(), dst->getName()); + auto num_operands = dst->getNumOperands(); + state.operands.reserve(num_operands + 1); + for (int input = 0, e = num_operands + 1; input != e; ++input) { + if (input < dst_input) { + state.operands.push_back(dst->getOperand(input)); + } else if (input == dst_input) { + state.operands.push_back(source->getResult(0)); + } else { + state.operands.push_back(dst->getOperand(input - 1)); + } + } + state.attributes.assign(dst->getAttrs().begin(), dst->getAttrs().end()); + state.types.assign(dst->getResultTypes().begin(), + dst->getResultTypes().end()); + builder_.setInsertionPoint(dst); + auto* new_dst = builder_.create(state); + + // Replaces the output uses of the old operation by the corresponding + // result of the new operation, and deletes the old operation. + for (unsigned i = 0, e = dst->getNumResults(); i != e; ++i) { + auto new_output = new_dst->getResult(i); + dst->getResult(i).replaceAllUsesWith(new_output); + } + dst->dropAllReferences(); + dst->erase(); + return absl::OkStatus(); +} + +absl::StatusOr ImporterBase::InferLibFunctionType( + const FunctionBody& fbody) { + mlir::Builder builder(context_); + + // The FunctionBody contains a graph with a single-output _Arg node for each + // function argument and a single-input _Retval node for each function return + // value. + // + // We already populated the ShapeRefiner with all the information about the + // shapes of these graph edges, so we just query it to build the corresponding + // MLIR function type signature. + + llvm::SmallVector arg_types; + if (specs_.inputs.empty()) { + arg_types.reserve(fbody.arg_types.size()); + for (auto arg : fbody.arg_nodes) { + // Find node in the graph using the node id instead of using `arg` + // directly because the graph has been cloned. + auto* node = graph_->FindNodeId(arg->id()); + TF_ASSIGN_OR_RETURN(auto type, + InferOutputType(*node, /*idx=*/0, builder)); + arg_types.push_back(type); + } + } else { + arg_types.reserve(fbody.arg_types.size()); + for (const auto& it : llvm::enumerate(specs_.inputs)) { + mlir::Type element_type; + const auto& node_info = it.value().second; + DataType dtype = node_info.imported_dtype; + // Uses the existing output type of the arg node if the data type of the + // the node isn't specified through the import configuration. + if (dtype == DT_INVALID) { + auto arg = fbody.arg_nodes[it.index()]; + auto* node = graph_->FindNodeId(arg->id()); + dtype = node->output_type(0); + if (dtype == DT_INVALID) { + return absl::InvalidArgumentError( + absl::StrCat("Input ", it.index(), "has invalid data type")); + } + } + TF_RETURN_IF_ERROR( + ::tensorflow::ConvertDataType(dtype, builder, &element_type)); + if (node_info.shape.unknown_rank()) { + arg_types.push_back(mlir::UnrankedTensorType::get(element_type)); + } else { + llvm::SmallVector shape; + TF_RETURN_IF_ERROR(ConvertToMlirShape(node_info.shape, &shape)); + arg_types.push_back(GetTypeFromTFTensorShape(shape, element_type)); + } + } + } + + llvm::SmallVector ret_types; + ret_types.reserve(fbody.ret_types.size()); + for (auto ret : fbody.ret_nodes) { + // Find node in the graph using the node id instead of using `ret` directly + // because the graph has been cloned. + auto* node = graph_->FindNodeId(ret->id()); + TF_ASSIGN_OR_RETURN(auto type, InferInputType(*node, /*idx=*/0, builder)); + ret_types.push_back(type); + } + + return builder.getFunctionType(arg_types, ret_types); +} + +// Stateful helper class to import a TensorFlow model expressed in GraphDef into +// an MLIR Module. +// +// The nodes defined in the graph are converted to a function called +// 'func_name'. All library function definitions are converted to MLIR functions +// in the module. +class GraphDefImporter : public ImporterBase { + public: + // Main entry point: converts the given graph to an MLIR Module. + static absl::StatusOr> Convert( + mlir::MLIRContext* context, const Graph& graph, + const GraphDebugInfo& debug_info, + const FunctionLibraryDefinition& flib_def, const GraphImportConfig& specs, + std::unordered_map* tf_name_to_mlir_name, + bool disable_crash_analysis = false, + std::optional module_name = std::nullopt); + + private: + explicit GraphDefImporter( + const FunctionLibraryDefinition& flib, const GraphDebugInfo& debug_info, + const GraphImportConfig& specs, mlir::ModuleOp module, + std::unordered_map* tf_name_to_mlir_name, + NameUniquifier* function_name_uniquifier) + : ImporterBase(flib, debug_info, specs, module, tf_name_to_mlir_name, + function_name_uniquifier) {} + + // Returns the function signature of the main function of converted MLIR + // module, the input nodes and output nodes. The type and shape information + // for the function arguments are read from `specs`, but the type and shape + // information for the function returns are inferred by the shape refiner in + // ImporterBase. + absl::StatusOr InferMainFunctionType( + const GraphImportConfig& specs, mlir::MLIRContext* context, + absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes); + + // Returns the function signature of the main function, alongside input and + // output nodes, for function graphs. Arguments and return values are + // determined by node op type. Type and shape information of the function are + // inferred by the shape refiner in ImporterBase. + absl::StatusOr GetArgsRetsAndTypesFromFunctionGraph( + mlir::MLIRContext* context, + absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes); + + // Finds the graph's target nodes/function's control ret nodes based on + // supplied node names in `control_outputs`. If `control_outputs` are not + // unique or a control ret node is missing, an error will be returned. + absl::Status GetControlRetsFromGraph( + llvm::ArrayRef control_outputs, + absl::InlinedVector* control_ret_nodes); +}; + +absl::StatusOr> GraphDefImporter::Convert( + mlir::MLIRContext* context, const Graph& graph, + const GraphDebugInfo& debug_info, const FunctionLibraryDefinition& flib_def, + const GraphImportConfig& specs, + std::unordered_map* tf_name_to_mlir_name, + bool disable_crash_analysis, std::optional module_name) { + LoadImporterDialects(*context); + mlir::OwningOpRef module = + mlir::ModuleOp::create(mlir::UnknownLoc::get(context), module_name); + NameUniquifier function_name_uniquifier(flib_def); + + // importer.PrepareConvert below will attemp to clone the original `graph` + // via conversion to the graph def first. Convert graph to graph_def here + // first and avoid extra copies later. + auto graph_def = std::make_unique(); + graph.ToGraphDef(graph_def.get(), /*include_flib_def=*/false); + + auto scope_exit = [&]() { + std::function cleanup = []() {}; + if (!disable_crash_analysis) { + static std::atomic counter(0); + uint32_t current_file_prefix = counter++; + const auto* graph_crash_handle = crash_analysis::ReportProtoDataOnCrash( + absl::StrCat(current_file_prefix, "_mlir_import_graph.pbtxt"), + *graph_def); + auto reachable_flib = flib_def.ReachableDefinitions(*graph_def); + const auto* flib_crash_handle = crash_analysis::ReportProtoDataOnCrash( + absl::StrCat(current_file_prefix, "_mlir_import_flib.pbtxt"), + reachable_flib.ToProto()); + cleanup = [=]() { + crash_analysis::RemoveReportData(graph_crash_handle); + crash_analysis::RemoveReportData(flib_crash_handle); + }; + } + + return llvm::make_scope_exit(std::move(cleanup)); + }(); + + VLOG(2) << "Importing: " + << ::tensorflow::DumpGraphToFile("tf_mlir_importer_base", graph, + &flib_def); + + GraphDefImporter importer(flib_def, debug_info, specs, module.get(), + tf_name_to_mlir_name, &function_name_uniquifier); + + TF_RETURN_IF_ERROR(importer.PrepareConvert(graph, std::move(graph_def))); + + mlir::FunctionType func_type; + absl::InlinedVector arg_nodes; + absl::InlinedVector ret_nodes; + absl::InlinedVector control_ret_nodes; + llvm::SmallVector attrs; + if (specs.graph_as_function) { + if (specs.prune_unused_nodes || !specs.inputs.empty() || + !specs.outputs.empty()) + return absl::InvalidArgumentError( + "Pruning of graph is currently unsupported when the main graph is " + "converted to a function."); + + TF_ASSIGN_OR_RETURN(func_type, + importer.GetArgsRetsAndTypesFromFunctionGraph( + context, &arg_nodes, &ret_nodes)); + + TF_RETURN_IF_ERROR(importer.GetControlRetsFromGraph(specs.control_outputs, + &control_ret_nodes)); + + mlir::Builder b(context); + std::string s; + llvm::raw_string_ostream ss(s); + auto node_name = [&](const OutputTensor& tensor) { + ss << tensor.node->name(); + }; + llvm::interleave(arg_nodes, ss, node_name, ","); + auto inputs = b.getNamedAttr("inputs", b.getStringAttr(ss.str())); + s.clear(); + llvm::interleave(ret_nodes, ss, node_name, ","); + auto outputs = b.getNamedAttr("outputs", b.getStringAttr(ss.str())); + s.clear(); + llvm::interleave(specs.control_outputs, ss, ","); + auto control_outputs = + b.getNamedAttr("control_outputs", b.getStringAttr(ss.str())); + + // Under `graph_as_function` mode, `tf.entry_function` is always set as it + // is assumed feed, fetch, and target nodes are set correctly. + attrs.push_back(b.getNamedAttr( + "tf.entry_function", + b.getDictionaryAttr({inputs, outputs, control_outputs}))); + if (!specs.xla_compile_device_type.empty()) { + attrs.push_back( + b.getNamedAttr("_xla_compile_device_type", + b.getStringAttr(specs.xla_compile_device_type))); + } + attrs.push_back(b.getNamedAttr("allow_soft_placement", + b.getBoolAttr(specs.enable_soft_placement))); + } else { + // Collects the argument and return nodes by looking up the node names + // specified by the user. + TF_ASSIGN_OR_RETURN(func_type, importer.InferMainFunctionType( + specs, context, &arg_nodes, &ret_nodes)); + + TF_RETURN_IF_ERROR(importer.GetControlRetsFromGraph(specs.control_outputs, + &control_ret_nodes)); + + // TODO(prakalps): Refactor to keep tf.entry_function attribute encoding and + // decoding in a centralized place. + // Record the input and output mapping. + if (!specs.inputs.empty() || !specs.outputs.empty() || + !specs.control_outputs.empty()) { + mlir::Builder b(context); + std::string s; + llvm::raw_string_ostream ss(s); + llvm::interleave( + specs.inputs, ss, + [&](const std::pair& v) { ss << v.first; }, + ","); + auto inputs = b.getNamedAttr("inputs", b.getStringAttr(ss.str())); + s.clear(); + llvm::interleave(specs.outputs, ss, ","); + auto outputs = b.getNamedAttr("outputs", b.getStringAttr(ss.str())); + s.clear(); + llvm::interleave(specs.control_outputs, ss, ","); + auto control_outputs = + b.getNamedAttr("control_outputs", b.getStringAttr(ss.str())); + + attrs.push_back(b.getNamedAttr( + "tf.entry_function", + b.getDictionaryAttr({inputs, outputs, control_outputs}))); + } + } + + // Record version info. + PopulateTfVersions(module.get(), graph.versions()); + + const llvm::StringRef& graph_func_name = + specs.graph_func_name.empty() ? kImportModelDefaultGraphFuncName + : specs.graph_func_name; + TF_RETURN_IF_ERROR(importer.ImporterBase::Convert(graph_func_name, func_type, + arg_nodes, ret_nodes, + control_ret_nodes, attrs)); + if (specs.convert_all_functions_to_mlir) { + auto fn_names = graph.flib_def().ListFunctionNames(); + for (const auto& fn_name : fn_names) { + TF_RETURN_IF_ERROR(importer.ConvertLibFunction(fn_name)); + } + } + TF_RETURN_IF_ERROR(importer.ImporterBase::ConvertDeferredFunctions()); + + // Mark main function public, others private. + for (auto function : module.get().getOps()) { + auto visibility = function.getName() == graph_func_name + ? mlir::func::FuncOp::Visibility::Public + : mlir::func::FuncOp::Visibility::Private; + function.setVisibility(visibility); + } + VLOG(2) << "Imported: " + << tensorflow::DumpMlirOpToFile("tf_mlir_imported_base", + module.get()); + return module; +} + +absl::StatusOr GraphDefImporter::InferMainFunctionType( + const GraphImportConfig& specs, mlir::MLIRContext* context, + absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes) { + // Find all the input nodes and output nodes. + // Feeds have been remapped to single output nodes (Placeholder), so an exact + // name match is sufficient. + absl::flat_hash_map inputs; + for (const auto& input_and_idx : llvm::enumerate(specs.inputs)) { + TensorId tensor = ParseTensorName(input_and_idx.value().first); + auto remapped_it = remapped_feeds_.find(tensor); + if (remapped_it != remapped_feeds_.end()) { + inputs.insert({remapped_it->second, input_and_idx.index()}); + } else { + inputs.insert({tensor.node(), input_and_idx.index()}); + } + } + + absl::flat_hash_set output_node_names; + std::vector outputs; + output_node_names.reserve(specs.outputs.size()); + for (const auto& output : specs.outputs) { + TensorId tensor = ParseTensorName(output); + auto remapped_it = remapped_feeds_.find(tensor); + if (remapped_it != remapped_feeds_.end()) { + output_node_names.insert(remapped_it->second); + outputs.push_back({remapped_it->second, 0}); + } else { + output_node_names.insert(tensor.node()); + outputs.push_back(tensor); + } + } + + if (!inputs.empty() || !outputs.empty()) { + arg_nodes->resize(inputs.size()); + ret_nodes->resize(outputs.size()); + + for (Node* n : GetOrderedNodes()) { + // Handle inputs/arguments. + auto input_it = inputs.find(n->name()); + if (input_it != inputs.end()) { + (*arg_nodes)[input_it->second] = {n, 0}; + } + + // Handle outputs/returns. + if (output_node_names.contains(n->name())) { + for (int i = 0, e = outputs.size(); i != e; ++i) { + TensorId tensor = outputs[i]; + if (n->name() != tensor.node()) continue; + (*ret_nodes)[i] = {n, tensor.index()}; + } + } + } + } + + // Starts to construct the function type. + mlir::Builder builder(context); + llvm::SmallVector arg_types; + arg_types.reserve(specs.inputs.size()); + int i = 0; + for (const auto& it : specs.inputs) { + Node* arg_node = arg_nodes->at(i).node; + if (arg_node == nullptr) { + return absl::InvalidArgumentError( + absl::StrCat("Input ", it.first, " was not found in graph")); + } + mlir::Type element_type; + const auto& node_info = it.second; + DataType imported_dtype = node_info.imported_dtype; + // Uses the existing output type of the arg node if the data type of the + // the node isn't specified through the import configuration. + if (imported_dtype == DT_INVALID) { + imported_dtype = arg_node->output_type(0); + if (imported_dtype == DT_INVALID) { + return absl::InvalidArgumentError( + absl::StrCat("Input ", i, "has invalid data type")); + } + } + // Check if we have subtypes first + if (!node_info.subtypes.empty()) { + std::vector subtypes; + for (const auto& st : node_info.subtypes) { + mlir::Type st_data_type; + llvm::SmallVector shape; + TF_RETURN_IF_ERROR(ConvertToMlirShape(st.shape, &shape)); + TF_RETURN_IF_ERROR( + ConvertDataType(st.imported_dtype, builder, &st_data_type)); + subtypes.push_back(GetTypeFromTFTensorShape(shape, st_data_type)); + } + if (imported_dtype == DT_RESOURCE) { + element_type = + mlir::TF::ResourceType::get(subtypes, builder.getContext()); + } else if (imported_dtype == DT_VARIANT) { + element_type = + mlir::TF::VariantType::get(subtypes, builder.getContext()); + } else { + return absl::InvalidArgumentError( + absl::StrCat(DataType_Name(imported_dtype), " takes no subtypes.")); + } + } else { + TF_RETURN_IF_ERROR( + ConvertDataType(imported_dtype, builder, &element_type)); + } + if (node_info.shape.unknown_rank()) { + arg_types.push_back(mlir::UnrankedTensorType::get(element_type)); + } else { + llvm::SmallVector shape; + TF_RETURN_IF_ERROR(ConvertToMlirShape(node_info.shape, &shape)); + arg_types.push_back(GetTypeFromTFTensorShape(shape, element_type)); + } + i++; + } + + llvm::SmallVector ret_types; + ret_types.reserve(specs.outputs.size()); + for (int i = 0, e = specs.outputs.size(); i != e; ++i) { + if (ret_nodes->at(i).node == nullptr) { + return absl::InvalidArgumentError( + absl::StrCat("Output ", specs.outputs[i], " was not found in graph")); + } + } + for (const auto& ret : *ret_nodes) { + if (ret.node->num_outputs() <= ret.index) { + return absl::InvalidArgumentError( + absl::StrCat("Invalid output index ", ret.index, + " specified for node: ", ret.node->name())); + } + TF_ASSIGN_OR_RETURN(auto type, + InferOutputType(*ret.node, ret.index, builder)); + ret_types.push_back(type); + } + + return builder.getFunctionType(arg_types, ret_types); +} + +absl::StatusOr +GraphDefImporter::GetArgsRetsAndTypesFromFunctionGraph( + mlir::MLIRContext* context, absl::InlinedVector* arg_nodes, + absl::InlinedVector* ret_nodes) { + auto add_node = [](Node* node, absl::InlinedVector* nodes) { + auto* attr = node->attrs().Find("index"); + if (!attr) + return absl::InvalidArgumentError( + absl::StrCat(node->type_string(), " node '", node->name(), + "' is missing attribute 'index'")); + + auto index = attr->i(); + const int num_nodes = nodes->size(); + if (num_nodes < index + 1) nodes->resize(index + 1); + + if ((*nodes)[index].node != nullptr) + return absl::InvalidArgumentError(absl::StrCat( + node->type_string(), " node '", node->name(), + "' has attribute 'index' ", index, " that conflicts with node '", + (*nodes)[index].node->name(), "'")); + (*nodes)[index] = {node, 0}; + + return absl::OkStatus(); + }; + + // Collect arg and ret nodes from graph. + for (auto* node : GetOrderedNodes()) + if (node->IsArg()) + TF_RETURN_IF_ERROR(add_node(node, arg_nodes)); + else if (node->IsRetval()) + TF_RETURN_IF_ERROR(add_node(node, ret_nodes)); + + // Collect arg and ret types and create function type. + mlir::Builder builder(context); + llvm::SmallVector arg_types; + arg_types.reserve(arg_nodes->size()); + for (const auto& arg_node_and_idx : llvm::enumerate(*arg_nodes)) { + auto& arg_node = arg_node_and_idx.value(); + if (arg_node.node == nullptr) + return absl::InvalidArgumentError(absl::StrCat( + "Graph missing _Arg at index ", arg_node_and_idx.index())); + + TF_ASSIGN_OR_RETURN(auto type, + InferOutputType(*arg_node.node, /*idx=*/0, builder)); + arg_types.push_back(type); + } + + llvm::SmallVector ret_types; + ret_types.reserve(ret_nodes->size()); + for (const auto& ret_node_and_idx : llvm::enumerate(*ret_nodes)) { + auto& ret_node = ret_node_and_idx.value(); + if (ret_node.node == nullptr) + return absl::InvalidArgumentError(absl::StrCat( + "Graph missing _Retval at index ", ret_node_and_idx.index())); + + TF_ASSIGN_OR_RETURN(auto type, + InferInputType(*ret_node.node, /*idx=*/0, builder)); + ret_types.push_back(type); + } + + return builder.getFunctionType(arg_types, ret_types); +} + +absl::Status GraphDefImporter::GetControlRetsFromGraph( + llvm::ArrayRef control_outputs, + absl::InlinedVector* control_ret_nodes) { + if (control_outputs.empty()) return absl::OkStatus(); + + llvm::SmallDenseMap controls_to_idx; + for (const auto& control_and_idx : llvm::enumerate(control_outputs)) + controls_to_idx.insert({control_and_idx.value(), control_and_idx.index()}); + + if (controls_to_idx.size() != control_outputs.size()) + return absl::InvalidArgumentError("Control outputs must be unique"); + + control_ret_nodes->resize(controls_to_idx.size()); + + for (auto* node : GetOrderedNodes()) { + auto it = controls_to_idx.find(node->name()); + if (it != controls_to_idx.end()) (*control_ret_nodes)[it->second] = node; + } + + for (auto node_and_name : llvm::zip(*control_ret_nodes, control_outputs)) + if (std::get<0>(node_and_name) == nullptr) + return absl::InvalidArgumentError(absl::StrCat( + "Control output '", std::get<1>(node_and_name), "' is missing")); + + return absl::OkStatus(); +} + +bool IsCompiledNode(const Node* n) { + return n->attrs().Find(tensorflow::kTpuReplicateAttr) || + n->attrs().Find(tensorflow::kCompileDeviceTypeAttr); +} + +absl::StatusOr> ConvertGraphToTfExecutor( + const Graph& graph, const GraphDebugInfo& debug_info, + const FunctionLibraryDefinition& flib_def, const GraphImportConfig& specs, + mlir::MLIRContext* context, + std::unordered_map* tf_name_to_mlir_name, + const ConfigProto& config_proto, + tensorflow::TF2XLABridgeVersion bridge_version, + std::optional module_name) { + if (bridge_version != tensorflow::TF2XLABridgeVersion::kNotBridgeUseCase) { + bool has_unsupported_features_in_mlir_bridge = + GraphHasUnsupportedFeaturesInMlirBridge( + graph, &flib_def, config_proto, + tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false); + if (has_unsupported_features_in_mlir_bridge) { + LOG(WARNING) + << "Graph contains unsupported features in MLIR bridge. " + << "Use MLIR bridge at your own risk or disable MLIR bridge, e.g., " + << "tf.config.experimental.disable_mlir_bridge."; + } + } + + // TODO(jpienaar): Remove need to const_cast. + if (specs.upgrade_legacy) { + NodeFilter node_filter = specs.restrict_functionalization_to_compiled_nodes + ? IsCompiledNode + : NodeFilter{}; + TF_RETURN_WITH_CONTEXT_IF_ERROR( + FunctionalizeControlFlow( + const_cast(&graph), + const_cast(&flib_def), node_filter, + /*include_functions=*/true), + tensorflow::kFunctionalizeControlFlowFailureMessage); + } + + std::unordered_map local_tf_name_to_mlir_name; + TF_ASSIGN_OR_RETURN( + auto module, + GraphDefImporter::Convert(context, graph, debug_info, flib_def, specs, + tf_name_to_mlir_name == nullptr + ? &local_tf_name_to_mlir_name + : tf_name_to_mlir_name, + /*disable_crash_analysis=*/false, + /*module_name=*/module_name)); + + if (specs.set_original_tf_func_name) { + // Set up the original function names in the imported TF MLIR. + mlir::Builder builder(module->getContext()); + mlir::SymbolTable symbol_table(*module); + for (const auto& [tf_name, mlir_name] : + (tf_name_to_mlir_name == nullptr ? local_tf_name_to_mlir_name + : *tf_name_to_mlir_name)) { + auto func_op = symbol_table.lookup(mlir_name); + TF_RET_CHECK(func_op) + << "Graphdef importer should have created a function named " + << mlir_name << "."; + func_op->setAttr("tf._original_func_name", + builder.getStringAttr(tf_name)); + } + } + return module; +} + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h new file mode 100644 index 0000000000000000000000000000000000000000..2e6a32ef402887cbee6d3d84ea894a518bce76b5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h @@ -0,0 +1,54 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_GRAPH_TO_TF_EXECUTOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_GRAPH_TO_TF_EXECUTOR_H_ + +#include +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_debug_info.pb.h" +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +inline constexpr absl::string_view kImportModelDefaultGraphFuncName = "main"; + +// Given a Graph, returns a MLIR module containing the graph, expressed with +// tf_executor dialect. +absl::StatusOr> ConvertGraphToTfExecutor( + const Graph& graph, const GraphDebugInfo& debug_info, + const FunctionLibraryDefinition& flib_def, const GraphImportConfig& specs, + mlir::MLIRContext* context, + std::unordered_map* tf_name_to_mlir_name = + nullptr, + const ConfigProto& config_proto = {}, + tensorflow::TF2XLABridgeVersion bridge_version = + tensorflow::TF2XLABridgeVersion::kNotBridgeUseCase, + std::optional module_name = std::nullopt); + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_GRAPH_TO_TF_EXECUTOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..750c8936303747ef6dcfb8d04a5ee549b6c2e045 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor_test.cc @@ -0,0 +1,205 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/graph_to_tf_executor.h" + +#include + +#include +#include +#include + +#include +#include "absl/status/statusor.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "riegeli/bytes/fd_reader.h" // from @riegeli +#include "riegeli/bytes/read_all.h" // from @riegeli +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace { + +using mlir::DialectRegistry; +using mlir::MLIRContext; + +constexpr char kGraphWithFlibDefFileName[] = "graph_with_flib_def.txt"; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/api/v2/testdata/"); +} + +class GraphToTfExecutorTest : public ::testing::Test { + public: + GraphToTfExecutorTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + GraphDef CreateGraphDef(std::string graphdef_filename) { + std::string file_path = TestDataPath() + graphdef_filename; + std::string contents; + GraphDef graph_def; + auto status = riegeli::ReadAll(riegeli::FdReader(file_path), contents); + if (!status.ok()) { + return graph_def; + } + tsl::protobuf::TextFormat::ParseFromString(contents, &graph_def); + return graph_def; + } + + int CountNumberOfFunctionsInModule(mlir::ModuleOp module) { + int count = 0; + for (auto unused : module.getOps()) { + (void)unused; // Avoid unused variable warning + count++; + } + return count; + } + + DialectRegistry registry_; + MLIRContext context_; +}; + +TEST_F(GraphToTfExecutorTest, BasicConvertGraphToTfExecutorPasses) { + Graph graph(OpRegistry::Global()); + GraphDebugInfo debug_info; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphImportConfig specs; + GraphDef graph_def = CreateGraphDef("valid_graph.txt"); + GraphConstructorOptions opts; + TF_ASSERT_OK(ConvertGraphDefToGraph(opts, graph_def, &graph)); + + TF_ASSERT_OK( + ConvertGraphToTfExecutor(graph, debug_info, flib_def, specs, &context_)); +} + +TEST_F( + GraphToTfExecutorTest, + ConvertGraphToTfExecutorConvertAllFunctionsTrueConvertsAllFunctionsInFlibDef) { + Graph graph(OpRegistry::Global()); + GraphDebugInfo debug_info; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphDef graph_def = CreateGraphDef(kGraphWithFlibDefFileName); + GraphConstructorOptions opts; + TF_ASSERT_OK(ConvertGraphDefToGraph(opts, graph_def, &graph)); + GraphImportConfig specs; + specs.convert_all_functions_to_mlir = true; + + absl::StatusOr> result = + ConvertGraphToTfExecutor(graph, debug_info, graph.flib_def(), specs, + &context_); + + // should equal main + 4 functions in flib_def + ASSERT_EQ(CountNumberOfFunctionsInModule(result->get()), 5); +} + +TEST_F( + GraphToTfExecutorTest, + ConvertGraphToTfExecutorConvertAllFunctionsFalseOnlyConvertsFunctionsReferencedInGraph) { + Graph graph(OpRegistry::Global()); + GraphDebugInfo debug_info; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphDef graph_def = CreateGraphDef(kGraphWithFlibDefFileName); + GraphConstructorOptions opts; + TF_ASSERT_OK(ConvertGraphDefToGraph(opts, graph_def, &graph)); + GraphImportConfig specs; + specs.convert_all_functions_to_mlir = false; + + absl::StatusOr> result = + ConvertGraphToTfExecutor(graph, debug_info, graph.flib_def(), specs, + &context_); + + // should equal main + 2 functions referenced by nodes in the graph via the + // "f" attr. + ASSERT_EQ(CountNumberOfFunctionsInModule(result->get()), 3); +} + +TEST_F(GraphToTfExecutorTest, + ConvertGraphToTfExecutorPopulatesTfNameToMlirNameMap) { + Graph graph(OpRegistry::Global()); + GraphDebugInfo debug_info; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphDef graph_def = CreateGraphDef(kGraphWithFlibDefFileName); + GraphConstructorOptions opts; + TF_ASSERT_OK(ConvertGraphDefToGraph(opts, graph_def, &graph)); + GraphImportConfig specs; + std::unordered_map tf_name_to_mlir_name; + + TF_ASSERT_OK(ConvertGraphToTfExecutor(graph, debug_info, graph.flib_def(), + specs, &context_, + &tf_name_to_mlir_name)); + + std::unordered_set result_set; + for (const auto& pair : tf_name_to_mlir_name) { + result_set.insert(pair.first); + } + // Converted functions referenced by nodes in the graph via the + // "f" attr. These are before they are converted to their corresponding MLIR + // name. + std::unordered_set expected_set = { + "__inference__traced_save_45", "__inference__traced_restore_57"}; + EXPECT_EQ(result_set, expected_set); +} + +TEST_F(GraphToTfExecutorTest, ConvertGraphToTfExecutorSetsModuleName) { + Graph graph(OpRegistry::Global()); + GraphDebugInfo debug_info; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + GraphImportConfig specs; + GraphDef graph_def = CreateGraphDef("valid_graph.txt"); + GraphConstructorOptions opts; + TF_ASSERT_OK(ConvertGraphDefToGraph(opts, graph_def, &graph)); + + const std::string kModuleName = "my_module_name"; + absl::StatusOr> result = + ConvertGraphToTfExecutor( + graph, debug_info, flib_def, specs, &context_, + /*tf_name_to_mlir_name=*/nullptr, + /*config_proto=*/{}, + /*bridge_version=*/ + tensorflow::TF2XLABridgeVersion::kNotBridgeUseCase, + /*module_name=*/kModuleName); + + TF_ASSERT_OK(result.status()); + EXPECT_EQ(result->get().getName().value_or(""), kModuleName); +} + +} // namespace +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..bfda14b99b25eeaf21f462245792b41ad28c4e88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.cc @@ -0,0 +1,206 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h" + +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/variant.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/reproducer.pb.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/compile_only_client.h" +#include "xla/hlo/ir/hlo_module.h" +#include "xla/shape.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/lib/monitoring/sampler.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using tpu::FunctionToHloArgs; +using tpu::MlirToHloArgs; +using tpu::ShardingAndIndex; + +auto* phase2_bridge_compilation_time = tsl::monitoring::Sampler<1>::New( + {"/tensorflow/core/tf2xla/api/v2/phase2_compilation_time", + "The wall-clock time spent on executing graphs in milliseconds.", + "configuration"}, + // Power of 1.5 with bucket count 45 (> 23 hours) + {tsl::monitoring::Buckets::Exponential(1, 1.5, 45)}); + +// Name of component for error logging. This name is fixed and required to +// enable logging. +constexpr char kBridgeComponent[] = "TFXLABridge"; +constexpr char kFullBridge[] = "full_bridge"; + +namespace { + +bool ShouldFallbackToGraphCompiler( + const std::variant& computation) { + if (computation.index() == 1) return true; + + return std::get<0>(computation).rollout_state == + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_DISABLED; +} + +std::string ComputationToString(const MlirToHloArgs computation) { + if (computation.mlir_module_op.has_value()) { + return SerializeMlirModule(computation.mlir_module_op.value()); + } + return std::string(computation.mlir_module); +} + +void DumpComputationInput( + const tpu::TPUCompileMetadataProto& metadata, + const std::vector& arg_shapes, + const std::variant + computation) { + if (!VLOG_IS_ON(2)) { + return; + } + + tensorflow::mlir::tf2xla::internal::LegalizeMlirToHloReproducer reproducer; + *reproducer.mutable_compile_metadata() = metadata; + for (const auto& shape : arg_shapes) { + shape.AsProto(reproducer.add_input_shapes()); + } + + switch (computation.index()) { + case 0: + reproducer.set_mlir_module(ComputationToString(std::get<0>(computation))); + break; + case 1: { + auto input = std::get<1>(computation); + *reproducer.mutable_function_def_library() = input.flib_def->ToProto(); + } break; + default: + VLOG(2) << "LegalizeMlirToHlo computation input: unknown"; + break; + } + + std::string string_reproducer; + tensorflow::protobuf::TextFormat::PrintToString(reproducer, + &string_reproducer); + DumpRawStringToFile("legalize_tf_reproducer.textproto", string_reproducer); +} + +absl::Status DumpHloCompilationResult( + absl::string_view name, XlaCompilationResult* compilation_result) { + if (!VLOG_IS_ON(2) && + !DEBUG_DATA_DUMPER()->ShouldDump(std::string(name), kDebugGroupMain)) { + return absl::OkStatus(); + } + + TF_ASSIGN_OR_RETURN( + auto hlo_module_config, + xla::HloModule::CreateModuleConfigFromProto( + compilation_result->computation->proto(), xla::DebugOptions())); + + TF_ASSIGN_OR_RETURN( + std::unique_ptr hlo_module, + xla::HloModule::CreateFromProto(compilation_result->computation->proto(), + hlo_module_config)); + + std::string all_computations; + for (auto computation : hlo_module->computations()) { + all_computations += computation->ToString() + "\n\n"; + } + + tensorflow::DumpRawStringToFile(name, all_computations); + + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr LegalizeMlirToHlo( + const std::variant& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + llvm::StringRef device_type, + std::vector>& custom_legalization_passes, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client) { + CompilationTimer timer; + auto record_time = llvm::make_scope_exit([&timer] { + phase2_bridge_compilation_time->GetCell(kFullBridge) + ->Add(timer.ElapsedCyclesInMilliseconds()); + }); + + auto compilation_result = std::make_unique(); + + DumpComputationInput(metadata, arg_shapes, computation); + + // If there are no MLIR args, compile the given function in the library. + if (ShouldFallbackToGraphCompiler(computation)) { + TF_RETURN_IF_ERROR(tf2xla::v1::CompileTensorflowGraphToHlo( + computation, metadata, use_tuple_args, shape_determination_fns, + arg_shapes, tsl::DeviceType(device_type.str()), arg_core_mapping, + per_core_arg_shapes, client, compilation_result.get())); + + DumpHloCompilationResult("legalize_tf_fallback.hlo", + compilation_result.get()) + .IgnoreError(); + return *compilation_result; + } + + auto combined_bridge_status = internal::LegalizeTfToHlo( + std::get<0>(computation), metadata, use_tuple_args, device_type, + shape_determination_fns, arg_shapes, arg_core_mapping, + per_core_arg_shapes, custom_legalization_passes, client, + compilation_result.get()); + + if (combined_bridge_status.ok()) { + VLOG(1) << "Successfully compiled MLIR computation to XLA HLO using " + "Combined MLIR and XlaBuilder Bridge."; + + DumpHloCompilationResult("legalize_tf_combined_bridge.hlo", + compilation_result.get()) + .IgnoreError(); + return *compilation_result; + } + + return combined_bridge_status.status(); +} + +}; // namespace v2 +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h new file mode 100644 index 0000000000000000000000000000000000000000..4c25578175262ba8cba388782b128d986840810f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h @@ -0,0 +1,67 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_LEGALIZE_TF_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_LEGALIZE_TF_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/types/variant.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/api/v2/device_type.pb.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/compile_only_client.h" +#include "xla/pjrt/proto/compile_options.pb.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +// Legalizes the given mlir::Module into XLA HLO. If successful, returns the +// compiled XLA HLO. V1 of the tf2xla uses MLIR whereas V0 does not use MLIR. +// +// Inputs: +// computation - The MLIR module op. It currently takes in +// tpu::FunctionToHloArgs but this is deprecated. arg_shapes - The shapes of +// the arguments in module_op. device_type - The device type to compile for. +// use_tuple_args - Pack the incoming arg shapes into a single tuple. +// custom_legalization_passes - Extra passes to lower from TF -> MHLO. +// arg_shapes - The shapes of the args. +// arg_core_mapping - Which args go on which cores. +// per_core_arg_shapes - For each core, the shapes for each argument. +// client - The Xla Compilation client. +absl::StatusOr LegalizeMlirToHlo( + const std::variant& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + llvm::StringRef device_type, + std::vector>& custom_legalization_passes, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + xla::CompileOnlyClient* client); + +}; // namespace v2 +}; // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_LEGALIZE_TF_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..4964d20a6e3ccda224128b8d580f468b661d06ea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test.cc @@ -0,0 +1,367 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h" + +#include +#include +#include +#include + +#include +#include +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "xla/client/client_library.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/lib/monitoring/test_utils.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/tsl/platform/test.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/lib/monitoring/test_utils.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using ::tensorflow::monitoring::testing::CellReader; +using tensorflow::tf2xla::v2::testing::CompileMlirModule; +using ::testing::Not; +using ::testing::TestWithParam; +using tpu::FunctionToHloArgs; +using tpu::ShardingAndIndex; +using tpu::TPUCompileMetadataProto; +using tsl::testing::TmpDir; + +static constexpr char kCompilationTimeStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/phase2_compilation_time"; +static constexpr char kFullBridge[] = "full_bridge"; +static constexpr char kCompilationStatusStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/phase2_compilation_status"; +static const char kMlirWithFallbackModeSuccess[] = + "kMlirWithFallbackModeSuccess"; +static const char kMlirWithFallbackModeFailure[] = + "kMlirWithFallbackModeFailure"; +static const char kOldBridgeMlirFilteredFailure[] = + "kOldBridgeMlirFilteredFailure"; +static const char kOldBridgeWithFallbackModeFailure[] = + "kOldBridgeWithFallbackModeFailure"; +static const char kOldBridgeMlirFilteredSuccess[] = + "kOldBridgeMlirFilteredSuccess"; +static const char kOldBridgeWithFallbackModeSuccess[] = + "kOldBridgeWithFallbackModeSuccess"; +static const char kMlirCombinedMlirSuccess[] = "kMlirCombinedMlirSuccess"; +static const char kMlirCombinedMlirFailure[] = "kMlirCombinedMlirFailure"; +static const char kMlirCombinedOldSuccess[] = "kMlirCombinedOldSuccess"; +static const char kMlirCombinedOldFailure[] = "kMlirCombinedOldFailure"; + +static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } +})"; + +// MLIR which should not legalize at all +static constexpr char kBadMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %0 = tf.Unknown() -> () + func.return %0 + } +})"; + +// MLIR which should be filtered by the MLIR bridge but fully legalize with the +// combined bridge. +static constexpr char kUnsupportedMlirBridgeModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %cst0 = "tf.Const"(){ value = dense<0> : tensor<3x5xi1>} : () -> tensor<3x5xi1> + %0 = "tf.Where"(%cst0) : (tensor<3x5xi1>) -> tensor + func.return + } +})"; + +TEST(LegalizeTFTest, RecordsStreamzForSuccessfulLegalizeWithMlirBridge) { + CellReader compilation_status(kCompilationStatusStreamzName); + + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + kMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED)); + + // May have been filtered so check for lack of failure instead of success. + EXPECT_EQ(compilation_status.Delta(kMlirWithFallbackModeFailure), 0); +} + +TEST(LegalizeTFTest, MatMul) { + static constexpr char kMatMulModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> (tensor<5x11xf32>) { + %arg0 = "tf.Const"() {value = dense<-3.0> : tensor<5x7xf32>} : () -> tensor<5x7xf32> + %arg1 = "tf.Const"() {value = dense<-3.0> : tensor<11x7xf32>} : () -> tensor<11x7xf32> + + %1 = "tf.MatMul"(%arg0, %arg1) {transpose_a = false, transpose_b = true} : (tensor<5x7xf32>, tensor<11x7xf32>) -> tensor<5x11xf32> + + func.return %1 : tensor<5x11xf32> + } + })"; + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + kMatMulModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED)); +} + +struct MatMulTestCase { + std::string mat_mul_method; +}; + +using BatchMatMulTest = TestWithParam; + +TEST_P(BatchMatMulTest, BatchMatMul) { + const MatMulTestCase& test_case = GetParam(); + static constexpr char kMatMulModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> (tensor<1x4x4xf32>) { + %%arg0 = "tf.Const"() {value = dense<-3.0> : tensor<1x4x2xf32>} : () -> tensor<1x4x2xf32> + %%arg1 = "tf.Const"() {value = dense<-3.0> : tensor<1x2x4xf32>} : () -> tensor<1x2x4xf32> + + %%1 = "tf.%s"(%%arg0, %%arg1) {T = f32, adj_x = false, adj_y = false, grad_x = false, grad_y = false, device = ""} : (tensor<1x4x2xf32>, tensor<1x2x4xf32>) -> tensor<1x4x4xf32> + + func.return %%1 : tensor<1x4x4xf32> + } + })"; + std::string mat_mul_method = + absl::StrFormat(kMatMulModuleStr, test_case.mat_mul_method); + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + mat_mul_method.c_str(), + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED)); +} + +INSTANTIATE_TEST_SUITE_P( + BatchMatMulTest, BatchMatMulTest, + ::testing::ValuesIn({ + {"BatchMatMul"}, + {"BatchMatMulV2"}, + {"BatchMatMulV3"}, + }), + [](const ::testing::TestParamInfo& info) { + return info.param.mat_mul_method; + }); + +TEST(LegalizeTFTest, DumpsProducedHLO) { + Env* env = Env::Default(); + std::string test_dir = TmpDir(); + setenv("TF_DUMP_GRAPH_PREFIX", test_dir.c_str(), /*overwrite=*/1); + setenv("TF_DUMP_GRAPH_NAME_FILTER", "*", 1); + DEBUG_DATA_DUMPER()->LoadEnvvars(); + + std::vector files; + TF_ASSERT_OK(env->GetChildren(test_dir, &files)); + int original_files_size = files.size(); + + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + kMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED)); + + // Due to the shared test of this infrastructure, we just need to make sure + // that the dumped file size is greater than what was originally inside + // the test directory. + TF_ASSERT_OK(env->GetChildren(test_dir, &files)); + EXPECT_THAT(files.size(), ::testing::Gt(original_files_size)); + setenv("TF_DUMP_GRAPH_PREFIX", test_dir.c_str(), /*overwrite=*/0); +} + +TEST(LegalizeTFTest, RecordsStreamzForFailedLegalizeWithMlirBridge) { + CellReader compilation_status(kCompilationStatusStreamzName); + + auto result = CompileMlirModule( + kBadMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED); + + EXPECT_FALSE(result.ok()); + EXPECT_EQ(compilation_status.Delta(kMlirCombinedMlirFailure), 1); +} + +TEST(LegalizeTFTest, RecordsStreamzForSuccessWithCombinedBridge) { + CellReader compilation_status(kCompilationStatusStreamzName); + + auto result = CompileMlirModule( + kUnsupportedMlirBridgeModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED); + + // MLIR Bridge will filter this unsupported MLIR, Combined will succeed. + EXPECT_TRUE(result.ok()); + EXPECT_EQ(compilation_status.Delta(kMlirCombinedMlirSuccess), 1); + EXPECT_EQ(compilation_status.Delta(kMlirCombinedMlirFailure), 0); + EXPECT_EQ(compilation_status.Delta(kMlirCombinedOldSuccess), 1); + EXPECT_EQ(compilation_status.Delta(kMlirCombinedOldFailure), 0); + // Old bridge should never be called at all. + EXPECT_EQ(compilation_status.Delta(kOldBridgeMlirFilteredFailure), 0); + EXPECT_EQ(compilation_status.Delta(kOldBridgeWithFallbackModeFailure), 0); + EXPECT_EQ(compilation_status.Delta(kOldBridgeMlirFilteredSuccess), 0); + EXPECT_EQ(compilation_status.Delta(kOldBridgeWithFallbackModeSuccess), 0); +} + +TEST(LegalizeTFTest, RecordsStreamzForNoMlirFallback) { + FunctionDef my_func = + tensorflow::FunctionDefHelper::Create("empty", {}, {}, {}, {}, {}); + + tensorflow::FunctionDefLibrary fdef; + *(fdef.add_function()) = my_func; + tensorflow::FunctionLibraryDefinition flib_def( + tensorflow::OpRegistry::Global(), fdef); + + OpInputList guaranteed_constants; + NameAttrList function; + FunctionToHloArgs function_to_hlo_args{&function, + &flib_def, + /*graph_def_version=*/0, + {&guaranteed_constants}}; + + se::Platform* cpu_platform = + se::PlatformManager::PlatformWithName("Host").value(); + auto client = + xla::ClientLibrary::GetOrCreateCompileOnlyClient(cpu_platform).value(); + + std::vector arg_shapes; + TPUCompileMetadataProto metadata_proto; + bool use_tuple_args = true; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + std::vector> custom_legalization_passes; + + // This doesn't actually compile correctly. + absl::StatusOr compile_result = + LegalizeMlirToHlo(function_to_hlo_args, metadata_proto, use_tuple_args, + /*device_type=*/"XLA_CPU_JIT", + custom_legalization_passes, + /*shape_determination_fns=*/{}, arg_shapes, + &arg_core_mapping, &per_core_arg_shapes, client); + + EXPECT_FALSE(compile_result.ok()); +} + +TEST(LegalizeTFTest, RecordsCompilationTimeForSuccessfulCompilation) { + CellReader compilation_time( + kCompilationTimeStreamzName); + + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + kMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_ENABLED)); + + // Compilation time should have been updated. + EXPECT_GT(compilation_time.Delta(kFullBridge).num(), 0); +} + +TEST(LegalizeTFTest, SuccessfullyCompilesModulesWithReturnValues) { + static constexpr char kHasReturnValuesAndNoMetadataRetvals[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> (tensor<2xi32>) { + %cst = "tf.Const"() {value = dense<[524170, 523952]> : tensor<2xi32>} : () -> tensor<2xi32> + return %cst : tensor<2xi32> + } + })"; + + auto compilation_result = CompileMlirModule( + kHasReturnValuesAndNoMetadataRetvals, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED); + EXPECT_TRUE(compilation_result.ok()); + + // Ensure that the compilation result contains a constant. + EXPECT_THAT(compilation_result, + ComputationProtoContains("opcode:.*constant")); +} + +TEST(LegalizeTFTest, SkipsTensorListSetItemIfDimensionsTooLarge) { + static constexpr char kTensorListSetItemDimensionTooLarge[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor>> { + // unknown rank + %elem_shape = "tf.Const"() <{value = dense<-1> : tensor}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor + // zero reserved elements + %num_elements = "tf.Const"() <{value = dense<0> : tensor}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor + + %list = "tf.TensorListReserve"(%elem_shape, %num_elements) : (tensor, tensor) -> tensor>> + + %index = "tf.Const"() <{value = dense<0> : tensor}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor + %element = "tf.Const"() <{value = dense<0.0> : tensor<64x1xbf16>}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor<64x1xbf16> + // Results in a bad mismatch of shapes. + %updated_list = "tf.TensorListSetItem"(%list, %index, %element) : (tensor>>, tensor, tensor<64x1xbf16>) -> tensor>> + + return %updated_list : tensor>> + } + })"; + + auto compilation_result = CompileMlirModule( + kTensorListSetItemDimensionTooLarge, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED); + + // Ensure that it compile + ASSERT_TRUE(compilation_result.ok()); + // Assert that the tensor list operation is lowered to something. + ASSERT_THAT(compilation_result, + Not(ComputationProtoContains("%.*= \"tf.TensorListSetItem"))); + // Assert that the tensor list operation is lowered to something that doesn't + // get stuck on a broken dynamic update slice. + ASSERT_THAT(compilation_result, + Not(ComputationProtoContains("%.*=.*DynamicUpdateSlice"))); +} + +TEST(LegalizeTFTest, LegalizesFunctionWithBoundedDynamicArg) { + static constexpr char kMlirModuleWithBoundedDynamicArgStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor> ) -> (tensor>) { + func.return %arg0 : tensor> + } +})"; + + auto compilation_result = CompileMlirModule( + kMlirModuleWithBoundedDynamicArgStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED); + + ASSERT_TRUE(compilation_result.ok()); + EXPECT_THAT(compilation_result, + ComputationProtoContains("element_type:.S32\n.*dimensions: 3")); +} + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test_gpu.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test_gpu.cc new file mode 100644 index 0000000000000000000000000000000000000000..edf85905390fa04efa1427fc9b625a8ec2fca244 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf_test_gpu.cc @@ -0,0 +1,82 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +// These test are in a separate file because they requires separate build +// environments (--config=cuda). +using tensorflow::tf2xla::v2::testing::CompileMlirModule; + +// MLIR which is legalize only with the right device type. +// The mlir is generated by running +// tensorflow/compiler/mlir/tf-opt -tf-xla-call-module-serialization +// +// module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, +// producer = 268 : i32}} { +// func.func private @_jit_sin(%arg0: tensor) -> tensor { +// %0 = stablehlo.sine %arg0 : tensor +// return %0 : tensor +// } +// func.func @main(%arg0: tensor) -> tensor<*xf32> { +// %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = +// "", dim_args_spec = [], _entry_function = @_jit_sin, +// _stablehlo_version = "1.0.0", module = "", platforms = ["CUDA"], +// version = 6 : i64} : (tensor) -> tensor<*xf32> +// func.return %0 : tensor<*xf32> +// } +// } +// +static constexpr char kGpuMlirModuleStr[] = R"( +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor) -> tensor<*xf32> { + %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], module = "ML\EFR\03StableHLOv1.0.0\00\01\17\05\01\05\01\03\05\03\07\07\t\0B\03K5\07\01\1B\07\0B\13\0B3\0B\0B\0B\0B\0F\0B\13\0B\03\1B\0F\1B\0B\0B\0B\0B\0B\0F\13\0B\0B\0B\0B\03\07\0F\17\07\02\A7\1F\05\0D\03\03\03\07\05\0F\03\0B\0B\1B\0D'\0F)\031\113\05\11\05\13\05\15\05\17\1D\15\17\05\19\17\19\EF\01\05\1B\03\03\1D\0D\05\1F!#%\1D\1D\1D\1F\1D!\1D##\03\03\03+\0D\03-/\1D%\1D'\1D)\1D+)\01\05\11\03\01\03\01\t\04A\05\01\11\01\05\07\03\01\05\03\11\01\t\05\03\05\0B\03\01\01\05\06\13\03\01\03\01\07\04\01\03\03\06\03\01\05\01\00\9A\04-\0F\0B\03!\1B\1D\05\1B\83/\1F\15\1D\15\11\13\15\11\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00sine_v1\00return_v1\00sym_name\00jit_sin\00arg_attrs\00function_type\00res_attrs\00sym_visibility\00jit(sin)/jit(main)/sin\00third_party/py/jax/experimental/jax2tf/tests/back_compat_test.py\00jax.arg_info\00x\00mhlo.sharding\00{replicated}\00jax.result_info\00\00main\00public\00", platforms = ["CUDA"], version = 6 : i64} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } +})"; + +TEST(LegalizeTFTest, RightDeviceTypeShallPass) { + TF_ASSERT_OK_AND_ASSIGN( + XlaCompiler::CompilationResult result, + CompileMlirModule( + kGpuMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED, + "XLA_GPU_JIT")); +} + +TEST(LegalizeTFTest, WrongDeviceTypeShallFail) { + absl::StatusOr result = CompileMlirModule( + kGpuMlirModuleStr, + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED, + "XLA_TPU_JIT"); + EXPECT_THAT(result, absl_testing::StatusIs(absl::StatusCode::kNotFound)); +} + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/mlir_roundtrip_flags.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/mlir_roundtrip_flags.h new file mode 100644 index 0000000000000000000000000000000000000000..86831d3ca952f862cb3d9289819cf33958a73bb7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/mlir_roundtrip_flags.h @@ -0,0 +1,119 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_MLIR_ROUNDTRIP_FLAGS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_MLIR_ROUNDTRIP_FLAGS_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/StringMap.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace tensorflow { + +struct ArrayInfoBase { + // The node type when the input node is imported. Typically needs to be + // specified when passing arbitrary nodes (some node attributes are removed). + DataType imported_dtype; + + // Node "shape" attribute value. + TensorShapeProto shape; +}; + +struct ArrayInfo : public ArrayInfoBase { + using SubTypeInfo = ArrayInfoBase; + // DT_RESOURCE and DT_VARIANT have subtypes + std::vector subtypes; +}; + +struct GraphImportConfig { + // Returns string representation of config. + std::string str() const; + + using InputArrays = + llvm::MapVector>; + // The name assigned to the function which is the import result of the given + // graph. If empty, a default one will be used. + std::string graph_func_name; + // Maps input node names to node data types and shapes. + InputArrays inputs; + // name:index strings for the data outputs. + std::vector outputs; + // name strings for the control outputs. + std::vector control_outputs; + // Setting prune_unused_nodes to true, would prune unreachable nodes if + // output_arrays is specified. + bool prune_unused_nodes = false; + // If true, inputs of type LegacyFedInput are replaced with Placeholder ops. + // LegacyFedInput ops have two outputs unlike Placeholder which has only one + // output, so if both outputs of the LegacyFedInput ops are used then returns + // an error. + bool convert_legacy_fed_inputs = false; + // If true, the main graph will be treated as a function. + bool graph_as_function = false; + // If true, upgrade legacy features of the graph (for instance, functionalize + // control-flow). + bool upgrade_legacy = false; + // If true, functionalization is restricted to nodes that will be + // XLA-compiled. This is only needed if + // - `upgrade_legacy` is true + // - upgrading legacy features of the graph (which includes functionalization) + // runs before compilation cluster extraction (as for MLIR-based TPU bridge) + // - session runtime is used (session runtime has issues with function names + // rewritten by functionalization). + // Otherwise, this parameter should be set to false. + bool restrict_functionalization_to_compiled_nodes = false; + // If true, enables shape inference on input. + // TODO(jpienaar): This will be removed shortly. + bool enable_shape_inference = true; + // _output_shapes is an unregistered attribute which is used during + // GraphConstructor::ConvertGraph to override shapes. It is unfortunately + // not always set correctly (which is undesirable and should be addressed) + // so make it opt-in to consider it unconditionally also when importing the + // graph. + bool unconditionally_use_set_output_shapes = false; + // If set, use the value as the device type and mark the function graph for + // XLA compilation. + std::string xla_compile_device_type; + // If true, enables moving ops to different devices or moving unsupported ops + // out of a compilation cluster. + bool enable_soft_placement = false; + // If true, a function attribute, `tf._original_func_name`, will be set in + // functions which contains the corresponding original TF function name. + bool set_original_tf_func_name = false; + + // If true, all functions in the graph will be converted to MLIR regardless of + // whether the functions are referenced by the nodes. This is needed if + // aliases and saved model object graph function matching is needed. + bool convert_all_functions_to_mlir = false; +}; + +struct GraphExportConfig { + // Whether to export the entry function to function library instead of the + // graph. + bool export_entry_func_to_flib = false; + // Whether to export functions using the name set in the attribute + // `tf._original_func_name` if it exists. + bool export_original_tf_func_name = false; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_MLIR_ROUNDTRIP_FLAGS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/empty_func.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/empty_func.mlir new file mode 100644 index 0000000000000000000000000000000000000000..826ce6955e709a9285f4c917ab8004923075c4ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/empty_func.mlir @@ -0,0 +1,5 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/func_with_dead_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/func_with_dead_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..14ec74b0edc40cd346862562e6423ef8117a8864 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/func_with_dead_ops.mlir @@ -0,0 +1,62 @@ +module attributes {tf.devices = {"/job:tpu_host_worker/replica:0/task:0/device:CPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:0", "/job:tpu_host_worker/replica:0/task:0/device:TPU:1", "/job:tpu_host_worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:1/device:CPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:0", "/job:tpu_host_worker/replica:0/task:1/device:TPU:1", "/job:tpu_host_worker/replica:0/task:1/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:2/device:CPU:0", "/job:tpu_host_worker/replica:0/task:2/device:TPU:0", "/job:tpu_host_worker/replica:0/task:2/device:TPU:1", "/job:tpu_host_worker/replica:0/task:2/device:TPU_SYSTEM:0", "/job:tpu_host_worker/replica:0/task:3/device:CPU:0", "/job:tpu_host_worker/replica:0/task:3/device:TPU:0", "/job:tpu_host_worker/replica:0/task:3/device:TPU:1", "/job:tpu_host_worker/replica:0/task:3/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1847 : i32}} { + func.func @main(%arg0: tensor {tf._user_specified_name = "steps", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg1: tensor<*x!tf_type.resource>> {tf._user_specified_name = "899", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg2: tensor<*x!tf_type.resource>> {tf._user_specified_name = "901", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg3: tensor<*x!tf_type.resource>> {tf._user_specified_name = "903", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg4: tensor<*x!tf_type.resource>> {tf._user_specified_name = "905", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg5: tensor<*x!tf_type.resource>> {tf._user_specified_name = "907", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg6: tensor<*x!tf_type.resource>> {tf._user_specified_name = "909", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg7: tensor<*x!tf_type.resource>> {tf._user_specified_name = "911", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg8: tensor<*x!tf_type.resource>> {tf._user_specified_name = "913", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg9: tensor<*x!tf_type.resource>> {tf._user_specified_name = "915", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg10: tensor<*x!tf_type.resource>> {tf._user_specified_name = "917", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg11: tensor<*x!tf_type.resource>> {tf._user_specified_name = "919", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg12: tensor<*x!tf_type.resource>> {tf._user_specified_name = "921", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg13: tensor<*x!tf_type.resource>> {tf._user_specified_name = "923", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg14: tensor<*x!tf_type.resource>> {tf._user_specified_name = "925", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg15: tensor<*x!tf_type.resource>> {tf._user_specified_name = "927", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg16: tensor<*x!tf_type.resource>> {tf._user_specified_name = "929", tf.device = "/job:tpu_host_worker/replica:0/task:1/device:CPU:0"}, %arg17: tensor<*x!tf_type.resource>> {tf._user_specified_name = "931", tf.device = "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"}, %arg18: tensor<*x!tf_type.resource>> {tf._user_specified_name = "933", tf.device = "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"}, %arg19: tensor<*x!tf_type.resource>> {tf._user_specified_name = "935", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg20: tensor<*x!tf_type.resource>> {tf._user_specified_name = "937", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}, %arg21: tensor<*x!tf_type.resource>> {tf._user_specified_name = "939", tf.device = "/job:tpu_host_worker/replica:0/task:0/device:CPU:0"}) -> tensor attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "steps,unknown,unknown_0,unknown_1,unknown_2,unknown_3,unknown_4,unknown_5,unknown_6,unknown_7,unknown_8,unknown_9,unknown_10,unknown_11,unknown_12,unknown_13,unknown_14,unknown_15,unknown_16,unknown_17,unknown_18,unknown_19", outputs = "statefulpartitionedcall_RetVal"}} { + %0 = "tf.ReadVariableOp"(%arg19) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %1 = "tf.ReadVariableOp"(%arg1) : (tensor<*x!tf_type.resource>>) -> tensor + %2 = "tf.ReadVariableOp"(%arg2) : (tensor<*x!tf_type.resource>>) -> tensor + %3 = "tf.ReadVariableOp"(%arg4) : (tensor<*x!tf_type.resource>>) -> tensor<1024xf32> + %4 = "tf.ReadVariableOp"(%arg3) : (tensor<*x!tf_type.resource>>) -> tensor<128x1024xf32> + %5 = "tf.ReadVariableOp"(%arg5) : (tensor<*x!tf_type.resource>>) -> tensor<1024x1xf32> + %6 = "tf.ReadVariableOp"(%arg20) : (tensor<*x!tf_type.resource>>) -> tensor<1024xf32> + %7 = "tf.ReadVariableOp"(%arg21) : (tensor<*x!tf_type.resource>>) -> tensor<1024x1xf32> + %8 = "tf.ReadVariableOp"(%arg6) : (tensor<*x!tf_type.resource>>) -> tensor + %9 = "tf.Const"() <{value = dense<"test"> : tensor<3x!tf_type.string>}> : () -> tensor<3x!tf_type.string> + %cst = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %11:4 = "tf.Split"(%cst, %0) {num_split = 4 : i32} : (tensor, tensor<128x1024xf32>) -> (tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>) + %cst_0 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %12:4 = "tf.Split"(%cst_0, %4) {num_split = 4 : i32} : (tensor, tensor<128x1024xf32>) -> (tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>) + %cst_1 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %cst_2 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %13:20 = tf_device.replicate {devices = {TPU_REPLICATED_CORE_0 = ["/job:tpu_host_worker/replica:0/task:0/device:TPU:0", "/job:tpu_host_worker/replica:0/task:2/device:TPU:0"], TPU_REPLICATED_CORE_1 = ["/job:tpu_host_worker/replica:0/task:0/device:TPU:1", "/job:tpu_host_worker/replica:0/task:2/device:TPU:1"], TPU_REPLICATED_CORE_2 = ["/job:tpu_host_worker/replica:0/task:1/device:TPU:0", "/job:tpu_host_worker/replica:0/task:3/device:TPU:0"], TPU_REPLICATED_CORE_3 = ["/job:tpu_host_worker/replica:0/task:1/device:TPU:1", "/job:tpu_host_worker/replica:0/task:3/device:TPU:1"], TPU_REPLICATED_HOST_0 = ["/job:tpu_host_worker/replica:0/task:0/device:CPU:0", "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"], TPU_REPLICATED_HOST_1 = ["/job:tpu_host_worker/replica:0/task:0/device:CPU:0", "/job:tpu_host_worker/replica:0/task:2/device:CPU:0"], TPU_REPLICATED_HOST_2 = ["/job:tpu_host_worker/replica:0/task:1/device:CPU:0", "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"], TPU_REPLICATED_HOST_3 = ["/job:tpu_host_worker/replica:0/task:1/device:CPU:0", "/job:tpu_host_worker/replica:0/task:3/device:CPU:0"]}, n = 2 : i32} { + %16:40 = "tf_device.parallel_execute"() ({ + %19:10 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_0"}> ({ + %20:10 = "tf.TPUExecute"(%arg0, %11#0, %1, %2, %3, %12#0, %5, %6, %7, %8, %9) : (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor<3x!tf_type.string>) -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %20#0, %20#1, %20#2, %20#3, %20#4, %20#5, %20#6, %20#7, %20#8, %20#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }) : () -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %19#0, %19#1, %19#2, %19#3, %19#4, %19#5, %19#6, %19#7, %19#8, %19#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }, { + %19:10 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_1"}> ({ + %20:10 = "tf.TPUExecute"(%arg0, %11#1, %1, %2, %3, %12#1, %5, %6, %7, %8, %9) : (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor<3x!tf_type.string>) -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %20#0, %20#1, %20#2, %20#3, %20#4, %20#5, %20#6, %20#7, %20#8, %20#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }) : () -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %19#0, %19#1, %19#2, %19#3, %19#4, %19#5, %19#6, %19#7, %19#8, %19#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }, { + %19:10 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_2"}> ({ + %20:10 = "tf.TPUExecute"(%arg0, %11#2, %1, %2, %3, %12#2, %5, %6, %7, %8, %9) : (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor<3x!tf_type.string>) -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %20#0, %20#1, %20#2, %20#3, %20#4, %20#5, %20#6, %20#7, %20#8, %20#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }) : () -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %19#0, %19#1, %19#2, %19#3, %19#4, %19#5, %19#6, %19#7, %19#8, %19#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }, { + %19:10 = "tf_device.launch"() <{device = "TPU_REPLICATED_CORE_3"}> ({ + %20:10 = "tf.TPUExecute"(%arg0, %11#3, %1, %2, %3, %12#3, %5, %6, %7, %8, %9) : (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor<3x!tf_type.string>) -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %20#0, %20#1, %20#2, %20#3, %20#4, %20#5, %20#6, %20#7, %20#8, %20#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }) : () -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + tf_device.return %19#0, %19#1, %19#2, %19#3, %19#4, %19#5, %19#6, %19#7, %19#8, %19#9 : tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor + }) : () -> (tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor, tensor, tensor<32x1024xf32>, tensor, tensor, tensor<1024xf32>, tensor<32x1024xf32>, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor) + %17 = "tf.Concat"(%cst_1, %16#5, %16#15, %16#25, %16#35) : (tensor, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>) -> tensor<128x1024xf32> + %18 = "tf.Concat"(%cst_2, %16#1, %16#11, %16#21, %16#31) : (tensor, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>, tensor<32x1024xf32>) -> tensor<128x1024xf32> + tf_device.return %16#0, %16#9, %16#8, %16#7, %16#6, %17, %16#4, %16#3, %16#2, %18 : tensor, tensor, tensor<1024x1xf32>, tensor<1024xf32>, tensor<1024x1xf32>, tensor<128x1024xf32>, tensor<1024xf32>, tensor, tensor, tensor<128x1024xf32> + } + "tf.AssignVariableOp"(%arg19, %13#18) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<128x1024xf32>) -> () + "tf.AssignVariableOp"(%arg1, %13#16) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg2, %13#14) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor) -> () + "tf.AssignVariableOp"(%arg4, %13#12) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<1024xf32>) -> () + "tf.AssignVariableOp"(%arg3, %13#10) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<128x1024xf32>) -> () + "tf.AssignVariableOp"(%arg5, %13#8) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<1024x1xf32>) -> () + "tf.AssignVariableOp"(%arg20, %13#6) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<1024xf32>) -> () + "tf.AssignVariableOp"(%arg21, %13#4) <{validate_shape = false}> : (tensor<*x!tf_type.resource>>, tensor<1024x1xf32>) -> () + "tf.AssignVariableOp"(%arg6, %13#2) <{validate_shape = true}> {_has_manual_control_dependencies = true} : (tensor<*x!tf_type.resource>>, tensor) -> () + %14 = "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>>) -> tensor + %15 = "tf.Identity"(%14) {device = ""} : (tensor) -> tensor + return %15 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/graph_with_flib_def.txt b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/graph_with_flib_def.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe52b071c08c7271fdcef2ae9385d869563ff67f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/graph_with_flib_def.txt @@ -0,0 +1,1535 @@ +node { + name: "Variable" + op: "VarHandleOp" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "debug_name" + value { + s: "Variable/" + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "shape" + value { + shape { + } + } + } + attr { + key: "shared_name" + value { + s: "Variable" + } + } +} +node { + name: "Variable/Read/ReadVariableOp" + op: "ReadVariableOp" + input: "Variable" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } +} +node { + name: "NoOp" + op: "NoOp" +} +node { + name: "Const" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "\n3\n\005\010\001\022\001v\n\n\010\002\022\006callee\n\n\010\003\022\006caller\n\016\010\004\022\nsignatures*\002\010\001\n>\0228\n\016VARIABLE_VALUE\022\010Variable\032\034v/.ATTRIBUTES/VARIABLE_VALUE*\002\010\001\n\017\n\013\010\005\022\007trace_0*\000\n\017\n\013\010\006\022\007trace_0*\000\n\002*\000\n\002*\000\n\002*\000" + } + } + } +} +node { + name: "saver_filename" + op: "Placeholder" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "shape" + value { + shape { + } + } + } +} +node { + name: "StatefulPartitionedCall" + op: "StatefulPartitionedCall" + input: "saver_filename" + input: "Variable" + input: "Const" + attr { + key: "Tin" + value { + list { + type: DT_STRING + type: DT_RESOURCE + type: DT_STRING + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_STRING + } + } + } + attr { + key: "_collective_manager_ids" + value { + list { + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + } + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000\222\001\002J\000" + } + } + attr { + key: "f" + value { + func { + name: "__inference__traced_save_45" + } + } + } +} +node { + name: "StatefulPartitionedCall_1" + op: "StatefulPartitionedCall" + input: "saver_filename" + input: "Variable" + attr { + key: "Tin" + value { + list { + type: DT_STRING + type: DT_RESOURCE + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_STRING + } + } + } + attr { + key: "_collective_manager_ids" + value { + list { + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + } + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000\222\001\002J\000" + } + } + attr { + key: "f" + value { + func { + name: "__inference__traced_restore_57" + } + } + } +} +library { + function { + signature { + name: "__inference_callee_10" + input_arg { + name: "x" + type: DT_FLOAT + } + input_arg { + name: "readvariableop_resource" + type: DT_RESOURCE + handle_data { + dtype: DT_FLOAT + shape { + } + } + } + output_arg { + name: "identity" + type: DT_FLOAT + } + output_arg { + name: "identity_1" + type: DT_FLOAT + } + is_stateful: true + control_output: "ReadVariableOp" + } + node_def { + name: "ReadVariableOp" + op: "ReadVariableOp" + input: "readvariableop_resource" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "x" + input: "^NoOp" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Identity_1" + op: "Identity" + input: "ReadVariableOp:value:0" + input: "^NoOp" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "NoOp" + op: "NoOp" + input: "^ReadVariableOp" + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + ret { + key: "identity" + value: "Identity:output:0" + } + ret { + key: "identity_1" + value: "Identity_1:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + attr { + key: "_input_shapes" + value { + list { + shape { + } + shape { + } + } + } + } + control_ret { + key: "ReadVariableOp" + value: "ReadVariableOp" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + } + } + arg_attr { + key: 1 + value { + attr { + key: "_user_specified_name" + value { + s: "resource" + } + } + } + } + } + function { + signature { + name: "__inference__traced_save_45" + input_arg { + name: "file_prefix" + type: DT_STRING + } + input_arg { + name: "read_disablecopyonread_variable" + type: DT_RESOURCE + handle_data { + dtype: DT_FLOAT + shape { + } + } + } + input_arg { + name: "savev2_const" + type: DT_STRING + } + output_arg { + name: "identity_3" + type: DT_STRING + } + is_stateful: true + control_output: "MergeV2Checkpoints" + control_output: "Read/DisableCopyOnRead" + control_output: "Read/ReadVariableOp" + } + node_def { + name: "StaticRegexFullMatch" + op: "StaticRegexFullMatch" + input: "file_prefix" + device: "/device:CPU:*" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "pattern" + value { + s: "^s3://.*" + } + } + } + node_def { + name: "Const" + op: "Const" + device: "/device:CPU:*" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: ".part" + } + } + } + } + node_def { + name: "Const_1" + op: "Const" + device: "/device:CPU:*" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + } + string_val: "_temp/part" + } + } + } + } + node_def { + name: "Select" + op: "Select" + input: "StaticRegexFullMatch:output:0" + input: "Const:output:0" + input: "Const_1:output:0" + device: "/device:CPU:*" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "StringJoin" + op: "StringJoin" + input: "file_prefix" + input: "Select:output:0" + device: "/device:CPU:*" + attr { + key: "N" + value { + i: 2 + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Read/DisableCopyOnRead" + op: "DisableCopyOnRead" + input: "read_disablecopyonread_variable" + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + node_def { + name: "Read/ReadVariableOp" + op: "ReadVariableOp" + input: "read_disablecopyonread_variable" + input: "^Read/DisableCopyOnRead" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "Read/ReadVariableOp:value:0" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Identity_1" + op: "Identity" + input: "Identity:output:0" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "num_shards" + op: "Const" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } + } + node_def { + name: "ShardedFilename/shard" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } + } + node_def { + name: "ShardedFilename" + op: "ShardedFilename" + input: "StringJoin:output:0" + input: "ShardedFilename/shard:output:0" + input: "num_shards:output:0" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "SaveV2/tensor_names" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "v/.ATTRIBUTES/VARIABLE_VALUE" + string_val: "_CHECKPOINTABLE_OBJECT_GRAPH" + } + } + } + } + node_def { + name: "SaveV2/shape_and_slices" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "" + string_val: "" + } + } + } + } + node_def { + name: "SaveV2" + op: "SaveV2" + input: "ShardedFilename:filename:0" + input: "SaveV2/tensor_names:output:0" + input: "SaveV2/shape_and_slices:output:0" + input: "Identity_1:output:0" + input: "savev2_const" + device: "/device:CPU:0" + attr { + key: "_has_manual_control_dependencies" + value { + b: true + } + } + attr { + key: "_output_shapes" + value { + list { + } + } + } + attr { + key: "dtypes" + value { + list { + type: DT_FLOAT + type: DT_STRING + } + } + } + } + node_def { + name: "MergeV2Checkpoints/checkpoint_prefixes" + op: "Pack" + input: "ShardedFilename:filename:0" + input: "^SaveV2" + device: "/device:CPU:0" + attr { + key: "N" + value { + i: 1 + } + } + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 1 + } + } + } + } + } + } + node_def { + name: "MergeV2Checkpoints" + op: "MergeV2Checkpoints" + input: "MergeV2Checkpoints/checkpoint_prefixes:output:0" + input: "file_prefix" + device: "/device:CPU:0" + attr { + key: "_has_manual_control_dependencies" + value { + b: true + } + } + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + node_def { + name: "Identity_2" + op: "Identity" + input: "file_prefix" + input: "^MergeV2Checkpoints" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Identity_3" + op: "Identity" + input: "Identity_2:output:0" + input: "^NoOp" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "NoOp" + op: "NoOp" + input: "^MergeV2Checkpoints" + input: "^Read/DisableCopyOnRead" + input: "^Read/ReadVariableOp" + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + ret { + key: "identity_3" + value: "Identity_3:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + attr { + key: "_input_shapes" + value { + list { + shape { + } + shape { + } + shape { + } + } + } + } + control_ret { + key: "MergeV2Checkpoints" + value: "MergeV2Checkpoints" + } + control_ret { + key: "Read/DisableCopyOnRead" + value: "Read/DisableCopyOnRead" + } + control_ret { + key: "Read/ReadVariableOp" + value: "Read/ReadVariableOp" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "file_prefix" + } + } + } + } + arg_attr { + key: 1 + value { + attr { + key: "_user_specified_name" + value { + s: "Variable" + } + } + } + } + arg_attr { + key: 2 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "Const" + } + } + } + } + } + function { + signature { + name: "__inference_caller_19" + input_arg { + name: "x" + type: DT_FLOAT + } + input_arg { + name: "unknown" + type: DT_RESOURCE + handle_data { + dtype: DT_FLOAT + shape { + } + } + } + output_arg { + name: "identity" + type: DT_FLOAT + } + output_arg { + name: "identity_1" + type: DT_FLOAT + } + is_stateful: true + control_output: "StatefulPartitionedCall" + } + node_def { + name: "StatefulPartitionedCall" + op: "StatefulPartitionedCall" + input: "x" + input: "unknown" + attr { + key: "Tin" + value { + list { + type: DT_FLOAT + type: DT_RESOURCE + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_FLOAT + type: DT_FLOAT + } + } + } + attr { + key: "_collective_manager_ids" + value { + list { + } + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + shape { + } + } + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + i: 1 + } + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000\222\001\002J\000" + } + } + attr { + key: "f" + value { + func { + name: "__inference_callee_10" + } + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "StatefulPartitionedCall:output:0" + input: "^NoOp" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Identity_1" + op: "Identity" + input: "StatefulPartitionedCall:output:1" + input: "^NoOp" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "NoOp" + op: "NoOp" + input: "^StatefulPartitionedCall" + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + ret { + key: "identity" + value: "Identity:output:0" + } + ret { + key: "identity_1" + value: "Identity_1:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + attr { + key: "_input_shapes" + value { + list { + shape { + } + shape { + } + } + } + } + control_ret { + key: "StatefulPartitionedCall" + value: "StatefulPartitionedCall" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + } + } + arg_attr { + key: 1 + value { + attr { + key: "_user_specified_name" + value { + s: "13" + } + } + } + } + } + function { + signature { + name: "__inference__traced_restore_57" + input_arg { + name: "file_prefix" + type: DT_STRING + } + input_arg { + name: "assignvariableop_variable" + type: DT_RESOURCE + handle_data { + dtype: DT_FLOAT + shape { + } + } + } + output_arg { + name: "identity_2" + type: DT_STRING + } + is_stateful: true + control_output: "AssignVariableOp" + } + node_def { + name: "RestoreV2/tensor_names" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "v/.ATTRIBUTES/VARIABLE_VALUE" + string_val: "_CHECKPOINTABLE_OBJECT_GRAPH" + } + } + } + } + node_def { + name: "RestoreV2/shape_and_slices" + op: "Const" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_STRING + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_STRING + tensor_shape { + dim { + size: 2 + } + } + string_val: "" + string_val: "" + } + } + } + } + node_def { + name: "RestoreV2" + op: "RestoreV2" + input: "file_prefix" + input: "RestoreV2/tensor_names:output:0" + input: "RestoreV2/shape_and_slices:output:0" + device: "/device:CPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + shape { + unknown_rank: true + } + } + } + } + attr { + key: "dtypes" + value { + list { + type: DT_FLOAT + type: DT_STRING + } + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "RestoreV2:tensors:0" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + unknown_rank: true + } + } + } + } + } + node_def { + name: "AssignVariableOp" + op: "AssignVariableOp" + input: "assignvariableop_variable" + input: "Identity:output:0" + device: "/device:CPU:0" + attr { + key: "_has_manual_control_dependencies" + value { + b: true + } + } + attr { + key: "_output_shapes" + value { + list { + } + } + } + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + } + node_def { + name: "NoOp" + op: "NoOp" + device: "/device:CPU:0" + attr { + key: "_has_manual_control_dependencies" + value { + b: true + } + } + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + node_def { + name: "Identity_1" + op: "Identity" + input: "file_prefix" + input: "^AssignVariableOp" + input: "^NoOp" + device: "/device:CPU:0" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "Identity_2" + op: "Identity" + input: "Identity_1:output:0" + input: "^NoOp_1" + attr { + key: "T" + value { + type: DT_STRING + } + } + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + node_def { + name: "NoOp_1" + op: "NoOp" + input: "^AssignVariableOp" + attr { + key: "_output_shapes" + value { + list { + } + } + } + } + ret { + key: "identity_2" + value: "Identity_2:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + attr { + key: "_input_shapes" + value { + list { + shape { + } + shape { + } + } + } + } + control_ret { + key: "AssignVariableOp" + value: "AssignVariableOp" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "file_prefix" + } + } + } + } + arg_attr { + key: 1 + value { + attr { + key: "_user_specified_name" + value { + s: "Variable" + } + } + } + } + } +} +versions { + producer: 2009 + min_consumer: 12 +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/invalid_executor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/invalid_executor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..79974124efbe109c4ed5856e2dc062e8e279fe5e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/invalid_executor.mlir @@ -0,0 +1,17 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() { + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + tf_executor.graph { + %control = tf_executor.island { + tf_executor.yield + } + tf_executor.fetch %control : !tf_executor.control + } + return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/outside_compilation.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/outside_compilation.mlir new file mode 100644 index 0000000000000000000000000000000000000000..002240da48328ab4e8f503c2d83bda92953acb52 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/outside_compilation.mlir @@ -0,0 +1,135 @@ +module attributes {tf.devices = {"/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0"}, tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1654 : i32}} { + func.func @main(%arg0: tensor<*x!tf_type.resource> {tf._user_specified_name = "input_1", tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) attributes {allow_soft_placement = true, tf.entry_function = {control_outputs = "while,image_sample/write_summary/summary_cond", inputs = "image_sample_write_summary_summary_cond_input_1", outputs = ""}} { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.TPUReplicatedInput"(%outputs, %outputs_0) {device = "", index = -1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor<*xi32> + %outputs_4, %control_5 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_6, %control_7 = tf_executor.island wraps "tf.Const"() {_post_device_rewrite = true, device = "", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %outputs_8, %control_9 = tf_executor.island wraps "tf.Const"() {_post_device_rewrite = true, device = "", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %outputs_10, %control_11 = tf_executor.island wraps "tf.Pack"(%outputs_6, %outputs_8) {axis = 0 : i64, device = ""} : (tensor<0xi32>, tensor<0xi32>) -> tensor<*xi32> + %outputs_12, %control_13 = tf_executor.island wraps "tf.Max"(%outputs_10, %outputs_4) {device = "", keep_dims = false} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %control_14 = tf_executor.island wraps "tf.NoOp"() {_pivot_for_cluster = "cluster_sample_sequence", device = ""} : () -> () + %control_15 = tf_executor.island(%control_14) wraps "tf.NoOp"() {_has_manual_control_dependencies = true, _tpu_replicate = "cluster_sample_sequence", device = ""} : () -> () + %control_16 = tf_executor.island(%control_15) wraps "tf.NoOp"() {device = ""} : () -> () + %control_17 = tf_executor.island(%control_15) wraps "tf.NoOp"() {device = ""} : () -> () + %control_18 = tf_executor.island(%control_14) wraps "tf.TPUReplicateMetadata"() {_has_manual_control_dependencies = true, _tpu_replicate = "cluster_sample_sequence", allow_soft_placement = true, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 2 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + %outputs_19, %control_20 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<0> : tensor} : () -> tensor + %outputs_21, %control_22 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<5> : tensor} : () -> tensor + %outputs_23, %control_24 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<[3, 32, 32, 1]> : tensor<4xi32>} : () -> tensor<4xi32> + %outputs_25, %control_26 = tf_executor.island(%control_18) wraps "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster_sample_sequence", device = ""} : () -> tensor + %outputs_27, %control_28 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_29, %control_30 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<[1, 1, 1, 3]> : tensor<4xi32>} : () -> tensor<4xi32> + %outputs_31, %control_32 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<0> : tensor} : () -> tensor + %outputs_33, %control_34 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense : tensor} : () -> tensor + %outputs_35, %control_36 = tf_executor.island(%control_18) wraps "tf.Identity"(%outputs_2) {_tpu_input_identity = true, _tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_37, %control_38 = tf_executor.island wraps "tf.Equal"(%outputs_35, %outputs_31) {_tpu_replicate = "cluster_sample_sequence", device = "", incompatible_shape_error = true} : (tensor<*xi32>, tensor) -> tensor<*xi1> + %outputs_39, %control_40 = tf_executor.island wraps "tf.LogicalAnd"(%outputs_37, %outputs_33) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xi1>, tensor) -> tensor<*xi1> + %outputs_41, %control_42 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_43, %control_44 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<1024> : tensor} : () -> tensor + %outputs_45, %control_46 = tf_executor.island wraps "tf.TensorListReserve"(%outputs_41, %outputs_43) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<1xi32>, tensor) -> tensor>> + %outputs_47, %control_48 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_49, %control_50 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<0> : tensor} : () -> tensor + %outputs_51, %control_52 = tf_executor.island(%control_18) wraps "tf.Const"() {_tpu_replicate = "cluster_sample_sequence", device = "", value = dense<-1> : tensor} : () -> tensor + %outputs_53:4, %control_54 = tf_executor.island wraps "tf.While"(%outputs_49, %outputs_51, %outputs_19, %outputs_45) {_num_original_outputs = 4 : i64, _read_only_resource_inputs = [], _tpu_replicate = "cluster_sample_sequence", _xla_propagate_compile_time_consts = true, body = @while_body_260, cond = @while_cond_250, device = "", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor>>) -> (tensor, tensor, tensor, tensor) + %outputs_55, %control_56 = tf_executor.island wraps "tf.TensorListStack"(%outputs_53#3, %outputs_27) {_tpu_replicate = "cluster_sample_sequence", device = "", num_elements = 1024 : i64} : (tensor, tensor<1xi32>) -> tensor<*xf32> + %outputs_57, %control_58 = tf_executor.island wraps "tf.Transpose"(%outputs_55, %outputs_47) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xf32>, tensor<2xi32>) -> tensor<*xf32> + %outputs_59, %control_60 = tf_executor.island wraps "tf.Reshape"(%outputs_57, %outputs_23) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xf32>, tensor<4xi32>) -> tensor<*xf32> + %outputs_61, %control_62 = tf_executor.island wraps "tf.Tile"(%outputs_59, %outputs_29) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xf32>, tensor<4xi32>) -> tensor<*xf32> + %outputs_63, %control_64 = tf_executor.island wraps "tf.If"(%outputs_39, %outputs_61, %arg0, %outputs_21) {_read_only_resource_inputs = [], _tpu_replicate = "cluster_sample_sequence", _xla_propagate_compile_time_consts = true, device = "", else_branch = @image_sample_write_summary_summary_cond_false_710, is_stateless = false, then_branch = @image_sample_write_summary_summary_cond_true_700} : (tensor<*xi1>, tensor<*xf32>, tensor<*x!tf_type.resource>, tensor) -> tensor<*xi1> + %outputs_65, %control_66 = tf_executor.island wraps "tf.Identity"(%outputs_63) {_tpu_replicate = "cluster_sample_sequence", device = ""} : (tensor<*xi1>) -> tensor<*xi1> + tf_executor.fetch %control_54, %control_64 : !tf_executor.control, !tf_executor.control + } + return + } + func.func private @while_body_260(%arg0: tensor {tf._user_specified_name = "while/loop_counter"}, %arg1: tensor {tf._user_specified_name = "while/maximum_iterations"}, %arg2: tensor, %arg3: tensor) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.variant>) attributes {tf._construction_context = "kEagerRuntime", tf.signature.is_stateful} { + %0:4 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_4, %control_5 = tf_executor.island wraps "tf.RandomUniform"(%outputs_2) {device = "", seed = 87654321 : i64, seed2 = 0 : i64} : (tensor<1xi32>) -> tensor<*xf32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.AddV2"(%arg2, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi32> + %outputs_8, %control_9 = tf_executor.island wraps "tf.Identity"(%outputs_6) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_10, %control_11 = tf_executor.island wraps "tf.TensorListSetItem"(%arg3, %arg2, %outputs_4) {device = "", resize_if_index_out_of_bounds = false} : (tensor, tensor, tensor<*xf32>) -> tensor<*x!tf_type.variant> + %outputs_12, %control_13 = tf_executor.island wraps "tf.Identity"(%outputs_10) {device = ""} : (tensor<*x!tf_type.variant>) -> tensor<*x!tf_type.variant> + %outputs_14, %control_15 = tf_executor.island wraps "tf.AddV2"(%arg0, %outputs_0) {device = ""} : (tensor, tensor) -> tensor<*xi32> + %outputs_16, %control_17 = tf_executor.island wraps "tf.Identity"(%outputs_14) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_18, %control_19 = tf_executor.island wraps "tf.Identity"(%arg1) {device = ""} : (tensor) -> tensor<*xi32> + tf_executor.fetch %outputs_16, %outputs_18, %outputs_8, %outputs_12 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.variant> + } + return %0#0, %0#1, %0#2, %0#3 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.variant> + } + func.func private @while_cond_250(%arg0: tensor {tf._user_specified_name = "while/loop_counter"}, %arg1: tensor {tf._user_specified_name = "while/maximum_iterations"}, %arg2: tensor, %arg3: tensor) -> tensor<*xi1> attributes {tf._construction_context = "kEagerRuntime"} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1024> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Less"(%arg2, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi1> + %outputs_2, %control_3 = tf_executor.island wraps "tf.Identity"(%outputs_0) {device = ""} : (tensor<*xi1>) -> tensor<*xi1> + tf_executor.fetch %outputs_2 : tensor<*xi1> + } + return %0 : tensor<*xi1> + } + func.func private @image_sample_write_summary_summary_cond_false_710(%arg0: tensor<3x32x32x3xf32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor) -> tensor<*xi1> attributes {tf._construction_context = "kEagerRuntime"} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Identity"(%outputs) {device = ""} : (tensor) -> tensor<*xi1> + tf_executor.fetch %outputs_0 : tensor<*xi1> + } + return %0 : tensor<*xi1> + } + func.func private @image_sample_write_summary_summary_cond_true_700(%arg0: tensor<3x32x32x3xf32> {tf._user_specified_name = "Tile"}, %arg1: tensor<*x!tf_type.resource> {tf._user_specified_name = "writer"}, %arg2: tensor {tf._user_specified_name = "Const_3"}) -> tensor<*xi1> attributes {tf._construction_context = "kEagerRuntime", tf.signature.is_stateful} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<[3, 32, 32, 3]> : tensor<4xi32>} : () -> tensor<4xi32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<""> : tensor} : () -> tensor + %outputs_4, %control_5 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"Condition x >= 0 did not hold element-wise:"> : tensor} : () -> tensor + %outputs_6, %control_7 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"x (image_sample/write_summary/summary_cond/assert_non_negative/x:0) = "> : tensor} : () -> tensor + %outputs_8, %control_9 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<""> : tensor} : () -> tensor + %outputs_10, %control_11 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"Condition x >= 0 did not hold element-wise:"> : tensor} : () -> tensor + %outputs_12, %control_13 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"x (image_sample/write_summary/summary_cond/assert_non_negative/x:0) = "> : tensor} : () -> tensor + %outputs_14, %control_15 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_16, %control_17 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %outputs_18, %control_19 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_20, %control_21 = tf_executor.island wraps "tf.Range"(%outputs_18, %outputs_14, %outputs_16) {device = "/device:CPU:0"} : (tensor, tensor, tensor) -> tensor<*xi32> + %outputs_22, %control_23 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<3> : tensor} : () -> tensor + %outputs_24, %control_25 = tf_executor.island wraps "tf.LessEqual"(%outputs_0, %outputs_22) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor<*xi1> + %outputs_26, %control_27 = tf_executor.island wraps "tf.All"(%outputs_24, %outputs_20) {device = "/device:CPU:0", keep_dims = false} : (tensor<*xi1>, tensor<*xi32>) -> tensor<*xi1> + %control_28 = tf_executor.island wraps "tf.Assert"(%outputs_26, %outputs_2, %outputs_4, %outputs_6, %outputs_22) {device = "/device:CPU:0", summarize = 3 : i64} : (tensor<*xi1>, tensor, tensor, tensor, tensor) -> () + %outputs_29, %control_30 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<[3, 32, 32, 3]> : tensor<4xi32>} : () -> tensor<4xi32> + %control_31 = tf_executor.island wraps "tf.NoOp"() {device = "/device:CPU:0"} : () -> () + %outputs_32, %control_33 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<4> : tensor} : () -> tensor + %control_34 = tf_executor.island wraps "tf.NoOp"() {device = "/device:CPU:0"} : () -> () + %outputs_35, %control_36 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_37, %control_38 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0.000000e+00> : tensor} : () -> tensor + %outputs_39, %control_40 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<2.550000e+02> : tensor} : () -> tensor + %outputs_41, %control_42 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<2.555000e+02> : tensor} : () -> tensor + %outputs_43, %control_44 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_45, %control_46 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_47, %control_48 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_49, %control_50 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_51, %control_52 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_53, %control_54 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_55, %control_56 = tf_executor.island wraps "tf.StridedSlice"(%outputs, %outputs_49, %outputs_51, %outputs_53) {begin_mask = 0 : i64, device = "/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<*xi32> + %outputs_57, %control_58 = tf_executor.island wraps "tf.AsString"(%outputs_55) {device = "/device:CPU:0", fill = "", precision = -1 : i64, scientific = false, shortest = false, width = -1 : i64} : (tensor<*xi32>) -> tensor<*x!tf_type.string> + %outputs_59, %control_60 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_61, %control_62 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_63, %control_64 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_65, %control_66 = tf_executor.island wraps "tf.StridedSlice"(%outputs, %outputs_59, %outputs_61, %outputs_63) {begin_mask = 0 : i64, device = "/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<*xi32> + %outputs_67, %control_68 = tf_executor.island wraps "tf.AsString"(%outputs_65) {device = "/device:CPU:0", fill = "", precision = -1 : i64, scientific = false, shortest = false, width = -1 : i64} : (tensor<*xi32>) -> tensor<*x!tf_type.string> + %outputs_69, %control_70 = tf_executor.island wraps "tf.Pack"(%outputs_57, %outputs_67) {axis = 0 : i64, device = "/device:CPU:0"} : (tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) -> tensor<*x!tf_type.string> + %outputs_71, %control_72 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"\0A\08\0A\06images"> : tensor} : () -> tensor + %outputs_73, %control_74 = tf_executor.island wraps "tf.Const"() {device = "/device:CPU:0", value = dense<"image_sample"> : tensor} : () -> tensor + %outputs_75, %control_76 = tf_executor.island wraps "tf.Mul"(%arg0, %outputs_41) {device = "/device:CPU:0"} : (tensor<3x32x32x3xf32>, tensor) -> tensor<*xf32> + %outputs_77, %control_78 = tf_executor.island wraps "tf.ClipByValue"(%outputs_75, %outputs_37, %outputs_39) {device = "/device:CPU:0"} : (tensor<*xf32>, tensor, tensor) -> tensor<*xf32> + %outputs_79, %control_80 = tf_executor.island wraps "tf.Cast"(%outputs_77) {Truncate = false, device = "/device:CPU:0"} : (tensor<*xf32>) -> tensor<*xui8> + %outputs_81, %control_82 = tf_executor.island wraps "tf.StridedSlice"(%outputs_79, %outputs_43, %outputs_45, %outputs_47) {begin_mask = 1 : i64, device = "/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 0 : i64} : (tensor<*xui8>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<*xui8> + %outputs_83, %control_84 = tf_executor.island wraps "tf.EncodePng"(%outputs_81) {compression = -1 : i64, device = "/device:CPU:0"} : (tensor<*xui8>) -> tensor<*x!tf_type.string> + %outputs_85, %control_86 = tf_executor.island wraps "tf.ConcatV2"(%outputs_69, %outputs_83, %outputs_35) {device = "/device:CPU:0"} : (tensor<*x!tf_type.string>, tensor<*x!tf_type.string>, tensor) -> tensor<*x!tf_type.string> + %control_87 = tf_executor.island wraps "tf.WriteSummary"(%arg1, %arg2, %outputs_85, %outputs_73, %outputs_71) {_has_manual_control_dependencies = true, device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>, tensor, tensor<*x!tf_type.string>, tensor, tensor) -> () + %outputs_88, %control_89 = tf_executor.island(%control_87) wraps "tf.Const"() {device = "/device:CPU:0", value = dense : tensor} : () -> tensor + %control_90 = tf_executor.island(%control_28, %control_87) wraps "tf.NoOp"() {device = ""} : () -> () + %outputs_91, %control_92 = tf_executor.island(%control_90) wraps "tf.Identity"(%outputs_88) {device = ""} : (tensor) -> tensor<*xi1> + tf_executor.fetch %outputs_91, %control_28, %control_87 : tensor<*xi1>, !tf_executor.control, !tf_executor.control + } + return %0 : tensor<*xi1> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_executor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_executor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5680bbcf317b53261943211de18efb80620086fc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_executor.mlir @@ -0,0 +1,10 @@ + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() { + tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:TPU:0", dtype = "tfdtype$DT_INT32", value = #tf_type : tensor<2xi32>} : () -> tensor<2xi32> loc("Empty/shape") + tf_executor.fetch + } + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_graph.txt b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_graph.txt new file mode 100644 index 0000000000000000000000000000000000000000..d059e13c5a4d32af21fc1ef0cf71db2f73e801bf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testdata/valid_graph.txt @@ -0,0 +1,44 @@ + node { + name: "Empty/shape" + op: "Const" + device: "/job:localhost/replica:0/task:0/device:TPU:0" + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 2 + } + } + } + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + tensor_content: "\200\000\000\000\200\000\000\000" + } + } + } + experimental_debug_info { + } +} +library { +} +versions { + producer: 268 +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..db18e7e9258ca8f0f3f122c11beaac903ee9dc7d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/BUILD @@ -0,0 +1,53 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tf2xla/api/v2:__subpackages__", + ], +) + +cc_library( + name = "compile_mlir", + testonly = True, + srcs = ["compile_mlir.cc"], + hdrs = ["compile_mlir.h"], + deps = [ + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir/tf2xla/api/v2:legalize_tf", + "//tensorflow/compiler/mlir/tf2xla/internal/utils:test_metadata_config", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//mlir:Pass", + "@xla//xla:shape_util", + "@xla//xla/client:client_library", + "@xla//xla/stream_executor:platform", + "@xla//xla/stream_executor:platform_manager", + ], +) + +cc_library( + name = "utils", + testonly = True, + srcs = ["utils.cc"], + hdrs = ["utils.h"], + deps = ["//tensorflow/core/platform:resource_loader"], +) + +tf_cc_test( + name = "utils_test", + srcs = ["utils_test.cc"], + deps = [ + ":utils", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.cc new file mode 100644 index 0000000000000000000000000000000000000000..a5889e94a5b547288152fc4180bfea01d2076ee2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.cc @@ -0,0 +1,81 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "xla/client/client_library.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +using tpu::MlirToHloArgs; +using tpu::ShardingAndIndex; +using tpu::TPUCompileMetadataProto; + +absl::StatusOr CompileMlirModule( + const char* mlir_module_str, + ConfigProto::Experimental::MlirBridgeRollout rollout_state, + absl::string_view device_type) { + MlirToHloArgs mlir_to_hlo_args; + mlir_to_hlo_args.rollout_state = rollout_state; + mlir_to_hlo_args.mlir_module = mlir_module_str; + + TF_ASSIGN_OR_RETURN(se::Platform * platform, + se::PlatformManager::PlatformWithName("Host")); + TF_ASSIGN_OR_RETURN( + auto client, xla::ClientLibrary::GetOrCreateCompileOnlyClient(platform)); + + std::vector arg_shapes; + TPUCompileMetadataProto metadata_proto; + // Configure metadata requires parsing the module and if we are testing a + // failure, we ignore this particular set up error assuming we'll not get + // far enough to need valid metadata. + tensorflow::tf2xla::internal::ConfigureMetadata(mlir_module_str, arg_shapes, + metadata_proto) + .IgnoreError(); + bool use_tuple_args = true; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + std::vector> custom_legalization_passes; + + return LegalizeMlirToHlo(mlir_to_hlo_args, metadata_proto, use_tuple_args, + device_type, custom_legalization_passes, + /*shape_determination_fns=*/{}, arg_shapes, + &arg_core_mapping, &per_core_arg_shapes, client); +} + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h new file mode 100644 index 0000000000000000000000000000000000000000..9aaa7e06eec019eb91d971940b7e77ceba693dba --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/compile_mlir.h @@ -0,0 +1,40 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_COMPILE_MLIR_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_COMPILE_MLIR_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +// Compiles the given MLIR module to XLA HLO. +absl::StatusOr CompileMlirModule( + const char* mlir_module_str, + ConfigProto::Experimental::MlirBridgeRollout rollout_state, + absl::string_view device_type = "XLA_TPU_JIT"); + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_COMPILE_MLIR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..2020e5d3511ce480ff06d82cf48d309f6bcfb0cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.cc @@ -0,0 +1,35 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h" + +#include + +#include "tensorflow/core/platform/resource_loader.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/api/v2/testdata/"); +} + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..e90acfce0264ed050e49c67fdc7d9477a70a872b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h @@ -0,0 +1,34 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_UTILS_H_ + +#include + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +// Returns the path to the testdata directory. +std::string TestDataPath(); + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TESTING_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..fcabc7618d8ca6caff1315b183ce313ddda542b4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils_test.cc @@ -0,0 +1,47 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h" + +#include + +#include + +#include +#include + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { +namespace { + +class UtilsTest : public ::testing::Test {}; + +TEST_F(UtilsTest, TestDataPathSucceeds) { + std::string expected_test_data_path_regex = + ".*tensorflow/compiler/mlir/tf2xla/api/v2/testdata/"; + + std::string result_test_data_path = TestDataPath(); + + EXPECT_THAT(result_test_data_path, + ::testing::ContainsRegex(expected_test_data_path_regex)); +} + +} // namespace +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.cc new file mode 100644 index 0000000000000000000000000000000000000000..423c9a87a55e1fd883960b862c4894fa6a9d4b95 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.cc @@ -0,0 +1,184 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" +#include "xla/tsl/lib/monitoring/counter.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/platform/error_payloads.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/util/debug_data_dumper.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::OpPassManager; +using mlir::PassManager; +using mlir::func::FuncOp; + +auto *tf_dialect_to_executor_dialect_status = tsl::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/api/v2/tf_dialect_to_executor_dialect_status", + "Counts how often a successful export from TF Dialect to Executor Dialect " + "is", + "status"); + +constexpr char kExportSuccess[] = "success"; +constexpr char kExportFailed[] = "failed"; + +namespace { + +void AddTfDialectToExecutorPasses(OpPassManager &pm) { + pm.addPass(mlir::TF::CreateTFRegionControlFlowToFunctional()); + + // First, we need to convert from functional, to executor dialect. + pm.addNestedPass( + mlir::CreateFunctionalToExecutorDialectConversionPass()); + + // Do a single pass to split the graph's single island op into an island per + // op as expected by the following passes. + pm.addNestedPass(mlir::TF::CreateSplitIntoIslandPerOpPass()); + + pm.addNestedPass(mlir::TFDevice::CreateReplicateToIslandPass( + /*legacy_graph_export=*/false)); + pm.addNestedPass( + mlir::TFDevice::CreateReplicaIDToDeviceOrdinalPass()); + pm.addNestedPass(mlir::TFDevice::CreateParallelExecuteToIslandsPass( + /*legacy_graph_export=*/false)); + pm.addNestedPass(mlir::TFDevice::CreateLaunchToDeviceAttributePass( + /*legacy_graph_export=*/false)); + + // Do a single pass to encode necessary control deps in the IR according to + // the results of side effect analysis. + pm.addPass( + mlir::tf_executor::CreateTFExecutorUpdateControlDependenciesPass()); + + pm.addNestedPass(mlir::TFTPU::CreateTPUDevicePropagationPass()); + pm.addNestedPass(mlir::TFTPU::CreateTPUColocateSplitsPass()); + pm.addPass(mlir::createSymbolDCEPass()); + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass()); + if (tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_convert_control_to_data_outputs_pass) { + bool composite_tpuexecute_side_effects = + tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_composite_tpuexecute_side_effects; + pm.addPass( + mlir::tf_executor::CreateTFExecutorConvertControlToDataOutputsPass( + composite_tpuexecute_side_effects)); + } + pm.addPass(mlir::TF::CreateVerifySuitableForExportPass()); +} + +absl::Status RecordStatusIfError(absl::Status status) { + if (status.ok()) { + return absl::OkStatus(); + } + + tf_dialect_to_executor_dialect_status->GetCell(kExportFailed)->IncrementBy(1); + VLOG(1) << "Failed to export from TF Dialect to TF Executor Dialect. " + << status; + + constexpr char bridge_subcomponent[] = + "TFXLA_TF_FUNCTIONAL_TO_EXECUTOR_EXPORT_v2"; + constexpr char kBridgeComponent[] = "TFXLABridge"; + + tsl::OkOrSetErrorCounterPayload( + tensorflow::core::platform::ErrorSourceProto::MLIR_BRIDGE_PHASE_1, + status); + + tsl::error_logging::Log(kBridgeComponent, bridge_subcomponent, + status.ToString()) + .IgnoreError(); + + return status; +} + +} // namespace + +absl::Status ExportFromTensorflowDialectToExecutor( + ModuleOp module, llvm::StringRef module_name) { + PassManager tf_to_executor(module.getContext()); + ::tensorflow::applyTensorflowAndCLOptions(tf_to_executor); + tf_to_executor.enableVerifier(); + AddTfDialectToExecutorPasses(tf_to_executor); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "tfxla_bridge_v2_tfdialect_to_executor_before"), + module, llvm::StringRef(), &tf_to_executor); + + if (VLOG_IS_ON(2) || + DEBUG_DATA_DUMPER()->ShouldDump( + module_name.str(), kDebugGroupBridgePhase1ExecutorExport)) { + internal::EnablePassIRPrinting( + tf_to_executor, kDebugGroupBridgePhase1ExecutorExport, module_name); + } + } + + LogicalResult result = tf_to_executor.run(module); + + if (VLOG_IS_ON(1) || + DEBUG_DATA_DUMPER()->ShouldDump(module_name.str(), kDebugGroupMain)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), kDebugGroupMain, + "tfxla_bridge_v2_tfdialect_to_executor_after"), + module, llvm::StringRef(), &tf_to_executor); + } + + if (result.failed()) { + return RecordStatusIfError( + absl::InternalError("Failed to export from TF Dialect to TF Executor " + "Dialect. Read LLVM Pipeline Error")); + } + + tf_dialect_to_executor_dialect_status->GetCell(kExportSuccess) + ->IncrementBy(1); + return absl::OkStatus(); +} + +mlir::PassPipelineRegistration<> tf_dialect_to_executor_pipeline( + "tf-dialect-to-executor-v2", + "Run passes to convert from TF Dialect to Executor in preparation for " + "exporting module back to TF Graph.", + AddTfDialectToExecutorPasses); + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h new file mode 100644 index 0000000000000000000000000000000000000000..50ec6694e305e9dd5bcb79ae6e63c58cf3d409c3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h @@ -0,0 +1,51 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_DIALECT_TO_EXECUTOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_DIALECT_TO_EXECUTOR_H_ + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/platform/status.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +// Given the input Module op that's in the Tensorflow Dialect, convert the MLIR +// module in place to the Tensorflow Executor Dialect. Returns an OK Status if +// success, otherwise failure with an error message. +// The Tensorflow Executor Dialect is required to export an MLIR module to a +// Tensorflow GraphDef. This API will add control dependencies and verify that +// the conversion was successful. +// +// This also converts the Tensorflow Dialect MLIR into the Tensorflow Executor +// dialect that is suitable to be exported to GraphDef. Graph -> MLIR -> Graph +// is not perfectly round trippable, so this API will attempt to make the module +// exportable and verify some properties of the Tensorflow Executor MLIR that +// are required by Graph Export. It will return an error if it cannot. +// +// Input: A MLIR Module in the Tensorflow Dialect with no +// `tf_device.cluster_func` ops. +// Output: A MLIR module in the Tensorflow Executor Dialect. +ABSL_DEPRECATED("Use tensorflow::tf2xla::v2::ConvertGraphToTfExecutor instead.") +absl::Status ExportFromTensorflowDialectToExecutor( + mlir::ModuleOp module, llvm::StringRef module_name = llvm::StringRef()); + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_DIALECT_TO_EXECUTOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..641ab76bd39c6cae6b56851f4e2e0334055c492c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor_test.cc @@ -0,0 +1,133 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h" + +#include + +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/testing/utils.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/platform/resource_loader.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace { + +constexpr char kExportStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/tf_dialect_to_executor_dialect_status"; +constexpr char kExportSuccess[] = "success"; +constexpr char kExportFailed[] = "failed"; + +using mlir::DialectRegistry; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; +using ::tensorflow::monitoring::testing::CellReader; +using ::tensorflow::tf2xla::v2::testing::TestDataPath; + +size_t CountSubstring(absl::string_view str, absl::string_view substr) { + size_t count = 0; + size_t idx = str.find(substr); + while (idx != std::string::npos) { + count++; + idx = str.find(substr, idx + 1); + } + return count; +} + +class TensorflowDialectToExecutorTest : public ::testing::Test { + public: + TensorflowDialectToExecutorTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(TensorflowDialectToExecutorTest, ConvertsToExecutor) { + CellReader compilation_status(kExportStreamzName); + + TF_ASSERT_OK(CreateMlirModule("empty_func.mlir")); + + TF_EXPECT_OK(ExportFromTensorflowDialectToExecutor(*mlir_module_)); + + EXPECT_EQ(compilation_status.Delta(kExportSuccess), 1); + EXPECT_EQ(compilation_status.Delta(kExportFailed), 0); +} + +TEST_F(TensorflowDialectToExecutorTest, ErrorsWhenCannotConvert) { + CellReader compilation_status(kExportStreamzName); + + TF_ASSERT_OK(CreateMlirModule("invalid_executor.mlir")); + + EXPECT_FALSE(ExportFromTensorflowDialectToExecutor(*mlir_module_).ok()); + + EXPECT_EQ(compilation_status.Delta(kExportSuccess), 0); + EXPECT_EQ(compilation_status.Delta(kExportFailed), 1); +} + +TEST_F(TensorflowDialectToExecutorTest, PrunesDeadOps) { + CellReader compilation_status(kExportStreamzName); + + TF_ASSERT_OK(CreateMlirModule("func_with_dead_ops.mlir")); + + TF_EXPECT_OK(ExportFromTensorflowDialectToExecutor(*mlir_module_)); + + std::string module_dump; + llvm::raw_string_ostream raw_stream(module_dump); + mlir_module_->print(raw_stream); + + EXPECT_EQ(compilation_status.Delta(kExportSuccess), 1); + EXPECT_EQ(compilation_status.Delta(kExportFailed), 0); + EXPECT_EQ( + CountSubstring(module_dump, "tf_executor.island wraps \"tf.Concat\""), 2); +} + +} // namespace +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.cc new file mode 100644 index 0000000000000000000000000000000000000000..31e58f757eb7ab2c8779dcee210ded68fb591f4d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.cc @@ -0,0 +1,842 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/inlined_vector.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/export_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/verify_suitable_for_graph_export.h" +#include "tensorflow/compiler/mlir/utils/name_utils.h" +#include "xla/status_macros.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/graph_to_functiondef.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/regularization/util.h" +#include "tensorflow/core/graph/tensor_id.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +using mlir::BlockArgument; +using mlir::Dialect; +using mlir::Operation; +using mlir::SymbolTable; +using mlir::Value; +using mlir::func::FuncOp; +using tsl::StatusOr; + +namespace { + +constexpr char kDeviceAttr[] = "tf.device"; +constexpr char kResourceArgUniqueIdAttr[] = "tf._resource_arg_unique_id"; +constexpr char kEntryFuncAttr[] = "tf.entry_function"; +constexpr char kAliasingAttr[] = "tf.aliasing_output"; + +// OpOrArgLocNameMapper that legalizes the returned name. +class LegalizedOpOrValLocNameMapper : public OpOrArgLocNameMapper { + private: + std::string GetName(OpOrVal op_or_val) override { + std::string name = OpOrArgLocNameMapper::GetName(op_or_val); + assert(!name.empty() && "expected non-empty name"); + mlir::LegalizeNodeName(name); + return name; + } +}; + +// Finds first inner op if `op` is a tf_executor.island. Otherwise `op` is +// returned. +Operation* GetIslandInnerOpOrSelf(mlir::Operation* op) { + auto island = llvm::dyn_cast(op); + if (island) return &island.GetBody().front(); + return op; +} + +// Stateful helper class to export a function into a Graph. +class Exporter { + public: + // Converts the given Module to a Graph. The given module should only contain + // one entry function, which is identified by name "main". This entry function + // is converted to the base of the graph graph. The rest of the functions are + // converted to the library functions in that graph. + static absl::Status Convert(mlir::ModuleOp module, + const GraphExportConfig& configs, + std::unique_ptr* graph, + FunctionLibraryDefinition* flib_def, + absl::flat_hash_set* control_ret_nodes); + + // Converts a given FuncOp to a FunctionDef and adds it to the function + // definition library + static absl::Status ConvertLibFunction( + const GraphExportConfig& configs, const Dialect* tf_dialect, + const SymbolTable& symbol_table, FuncOp function, + FunctionLibraryDefinition* flib_def, + llvm::SmallDenseSet& visited_functions); + + // Converts the given FuncOp to a Graph. The arguments and returns of + // function are added to the graph with special op names kArgOp and kRetOp. + // Later on, this graph can be converted a function definition and added to + // another graph. + static absl::StatusOr> Convert( + const GraphExportConfig& configs, const Dialect* tf_dialect, + const SymbolTable& symbol_table, FuncOp function, + FunctionLibraryDefinition* flib_def, + llvm::SmallDenseSet& visited_functions, + absl::flat_hash_set* control_ret_nodes); + + private: + explicit Exporter(const GraphExportConfig* configs, Graph* graph, + const Dialect* tf_dialect, const SymbolTable* symbol_table) + : configs_(*configs), + graph_(graph), + tf_dialect_(tf_dialect), + symbol_table_(*symbol_table) { + graph_->ToGraphDef(&graphdef_); + } + + absl::Status AddArgumentNode(BlockArgument arg, unsigned index, + llvm::StringRef name); + absl::Status AddFetchNode(FuncOp function, mlir::tf_executor::FetchOp fetch, + llvm::ArrayRef names); + absl::Status AddInstructionNode(Operation* inst); + void UseOriginalFunctionNames(NodeDef& node_def); + + absl::Status AddEdge(Operation* inst); + + absl::StatusOr> GetArgumentNode( + BlockArgument arg, unsigned index, llvm::StringRef name); + absl::StatusOr> GetReturnNode(FuncOp function, + Value operand, + unsigned index, + llvm::StringRef name); + absl::Status GetControlRetNodes( + mlir::tf_executor::FetchOp fetch, + absl::flat_hash_set* control_ret_nodes); + // Adds one edge between src_node and dst_node. If it is not a control edge, + // an index is used to find out the right operand of the dst_node. + absl::Status AddEdgeBetweenNodes(Value src, Node* dst_node, + unsigned dst_index); + + const GraphExportConfig& configs_; + Graph* graph_; + GraphDef graphdef_; + LegalizedOpOrValLocNameMapper op_to_name_; + absl::flat_hash_map nodes_; + llvm::DenseMap args_; + // One single return operation can return multiple results, and each of them + // will be converted to one node in the graph. + typedef absl::InlinedVector NodeVector; + absl::flat_hash_map returns_; + const mlir::Dialect* tf_dialect_; + const SymbolTable& symbol_table_; +}; + +std::string FindFunctionName(const GraphExportConfig& configs, FuncOp func) { + if (auto original_func_name = + func->getAttrOfType("tf._original_func_name"); + configs.export_original_tf_func_name && original_func_name) { + return original_func_name.str(); + } + + return func.getName().str(); +} + +absl::StatusOr> Exporter::GetArgumentNode( + BlockArgument arg, unsigned index, llvm::StringRef name) { + auto func = arg.getParentRegion()->getParentOfType(); + + auto node_def = std::make_unique(); + if (!name.empty()) + node_def->set_name(ParseTensorName(name.str()).node()); + else + node_def->set_name( + std::string(op_to_name_.GetUniqueName(func.getName().str()))); + + node_def->set_op(FunctionLibraryDefinition::kArgOp); + + mlir::TensorType arg_type = mlir::cast(arg.getType()); + if (auto resource_type = + mlir::dyn_cast(arg_type.getElementType())) { + llvm::ArrayRef subtypes = resource_type.getSubtypes(); + if (!subtypes.empty()) { + AttrValue handle_dtypes_attr; + AttrValue handle_shapes_attr; + for (mlir::TensorType subtype : subtypes) { + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(subtype.getElementType(), &dtype)); + handle_dtypes_attr.mutable_list()->add_type(dtype); + + SetTensorShapeProto(subtype, + handle_shapes_attr.mutable_list()->add_shape()); + } + + (*node_def->mutable_attr())["_handle_dtypes"] = handle_dtypes_attr; + (*node_def->mutable_attr())["_handle_shapes"] = handle_shapes_attr; + } + } + + TF_RETURN_IF_ERROR( + SetShapeAttribute("_output_shapes", arg_type, node_def->mutable_attr())); + + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(arg_type.getElementType(), &dtype)); + AttrValue type_attr; + type_attr.set_type(dtype); + (*node_def->mutable_attr())["T"] = type_attr; + + AttrValue index_attr; + index_attr.set_i(index); + (*node_def->mutable_attr())["index"] = index_attr; + + if (auto device_attr = + func.getArgAttrOfType(index, kDeviceAttr)) + *node_def->mutable_device() = device_attr.getValue().str(); + + llvm::ArrayRef func_arg_i_attrs = + mlir::function_interface_impl::getArgAttrs(func, index); + absl::flat_hash_set attrs_to_ignore = {kDeviceAttr, + kAliasingAttr}; + TF_RETURN_IF_ERROR(ConvertAttributes(func_arg_i_attrs, attrs_to_ignore, + /*remove_ref_type=*/false, + node_def->mutable_attr())); + + return node_def; +} + +absl::StatusOr> Exporter::GetReturnNode( + FuncOp function, Value operand, unsigned index, llvm::StringRef name) { + auto node_def = std::make_unique(); + if (!name.empty()) + node_def->set_name(ParseTensorName(name.str()).node()); + else + node_def->set_name( + std::string(op_to_name_.GetUniqueName(function.getName().str()))); + + node_def->set_op(FunctionLibraryDefinition::kRetOp); + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType( + mlir::cast(operand.getType()).getElementType(), + &dtype)); + AttrValue type_attr; + type_attr.set_type(dtype); + (*node_def->mutable_attr())["T"] = type_attr; + AttrValue index_attr; + index_attr.set_i(index); + (*node_def->mutable_attr())["index"] = index_attr; + + if (auto device_attr = + function.getResultAttrOfType(index, kDeviceAttr)) + *node_def->mutable_device() = device_attr.getValue().str(); + + llvm::ArrayRef func_res_i_attrs = + function.getResultAttrs(index); + absl::flat_hash_set attrs_to_ignore = {kDeviceAttr}; + TF_RETURN_IF_ERROR(ConvertAttributes(func_res_i_attrs, attrs_to_ignore, + /*remove_ref_type=*/false, + node_def->mutable_attr())); + + return node_def; +} + +absl::Status Exporter::AddEdgeBetweenNodes(Value src, Node* dst_node, + unsigned dst_index) { + if (auto input_result = mlir::dyn_cast(src)) { + auto* input_inst = GetIslandInnerOpOrSelf(input_result.getOwner()); + // Replaces the input node with NextIteration sink if it is a NextIteration + // source. + if (auto next_iter_source = + llvm::dyn_cast( + input_inst)) + input_inst = next_iter_source.GetSink(); + + auto node_it = nodes_.find(input_inst); + TF_RET_CHECK(node_it != nodes_.end()) + << "Use of OpResult encountered before def!"; + if (mlir::isa(input_result.getType())) { + graph_->AddControlEdge(node_it->second, dst_node, + /*allow_duplicates=*/true); + } else { + graph_->AddEdge(node_it->second, input_result.getResultNumber(), dst_node, + dst_index); + } + return absl::OkStatus(); + } + + auto input_arg = mlir::cast(src); + auto input_node_it = args_.find(input_arg); + TF_RET_CHECK(input_node_it != args_.end()) + << "Use of BlockArgument encounted before def!"; + // For argument, there is only one result output, so the index is always 0. + graph_->AddEdge(input_node_it->second, 0, dst_node, dst_index); + return absl::OkStatus(); +} + +absl::Status Exporter::AddEdge(Operation* inst) { + // For tf_executor.fetch, add only its data edges. Control edges are captured + // later. + if (auto fetch = llvm::dyn_cast(inst)) { + for (auto operand_and_idx : llvm::enumerate(fetch.getOperands())) { + Value operand = operand_and_idx.value(); + if (mlir::isa(operand.getType())) break; + + auto* dst_node = returns_[fetch][operand_and_idx.index()]; + TF_RETURN_IF_ERROR(AddEdgeBetweenNodes(operand, dst_node, 0)); + } + + return absl::OkStatus(); + } + + // For tf_executor.NextIteration.Sink, skip its token operand and add data and + // control edges with their index offset by 1. + if (auto next_iter_sink = + llvm::dyn_cast(inst)) { + auto* dst_node = nodes_[inst]; + TF_RETURN_IF_ERROR( + AddEdgeBetweenNodes(next_iter_sink.getInput(), dst_node, 0)); + for (auto control_and_idx : + llvm::enumerate(next_iter_sink.getControlInputs())) + TF_RETURN_IF_ERROR(AddEdgeBetweenNodes(control_and_idx.value(), dst_node, + control_and_idx.index() + 1)); + + return absl::OkStatus(); + } + + // For tf_executor.NextIteration.Source, op can be skipped as it is assumed + // there are no operands. + if (llvm::isa(inst)) { + assert(inst->getNumOperands() == 0); + return absl::OkStatus(); + } + + Operation* op = GetIslandInnerOpOrSelf(inst); + auto* dst_node = nodes_[op]; + int operand_offset = 0; + // For tf_executor.island, add data edges from its wrapped op before control + // edges. + if (auto island = llvm::dyn_cast(inst)) { + for (auto operand_and_idx : llvm::enumerate(op->getOperands())) + TF_RETURN_IF_ERROR(AddEdgeBetweenNodes(operand_and_idx.value(), dst_node, + operand_and_idx.index())); + + operand_offset = op->getNumOperands(); + } + + // For all other ops (including tf_executor.island), add remaining edges. + for (auto operand_and_idx : llvm::enumerate(inst->getOperands())) + TF_RETURN_IF_ERROR( + AddEdgeBetweenNodes(operand_and_idx.value(), dst_node, + operand_and_idx.index() + operand_offset)); + + return absl::OkStatus(); +} + +void Exporter::UseOriginalFunctionNames(NodeDef& node_def) { + if (!configs_.export_original_tf_func_name) return; + + auto& attrs = *node_def.mutable_attr(); + + auto try_use_original_func_name = [this](std::string* name) { + if (auto func = symbol_table_.lookup(*name)) { + if (auto original_func_name = + func->getAttrOfType("tf._original_func_name")) { + *name = original_func_name.str(); + } + } + }; + + // Change its op name if it is a legacy call. + try_use_original_func_name(node_def.mutable_op()); + + // Change any function attributes in the attrs. + for (auto& iter : attrs) { + auto& attr = iter.second; + if (attr.has_func()) { + try_use_original_func_name(attr.mutable_func()->mutable_name()); + } else if (attr.has_list()) { + for (auto& func_attr : *attr.mutable_list()->mutable_func()) { + try_use_original_func_name(func_attr.mutable_name()); + } + } + } +} + +absl::Status Exporter::AddInstructionNode(Operation* inst) { + std::unique_ptr node_def; + int graph_hash_value = graph_regularization::ComputeHash(graphdef_); + auto name = op_to_name_.GetUniqueName(inst, graph_hash_value); + // Convert registered TF ops to NodeDef. Only registered ops are handled to + // ensure that PopulateDerivedAttrs adds the correct attributes. + TF_ASSIGN_OR_RETURN(node_def, + ConvertTFDialectOpToNodeDef( + inst, name, /*ignore_unregistered_attrs=*/false)); + UseOriginalFunctionNames(*node_def); + + TF_ASSIGN_OR_RETURN(Node * node, graph_->AddNode(std::move(*node_def))); + DCHECK(node != nullptr); + nodes_[inst] = node; + return absl::OkStatus(); +} + +bool IsEntryFunctionArg(BlockArgument arg) { + return arg.getParentRegion()->getParentOfType().getName() == "main"; +} + +// Creates argument nodes from Block argument. If a name is supplied, that +// name will be used instead of generating a unique name. +absl::Status Exporter::AddArgumentNode(BlockArgument arg, unsigned index, + llvm::StringRef name) { + TF_ASSIGN_OR_RETURN(auto node_def, GetArgumentNode(arg, index, name)); + TF_ASSIGN_OR_RETURN(Node * node, graph_->AddNode(std::move(*node_def))); + args_[arg] = node; + return absl::OkStatus(); +} + +// Creates return nodes per operand of a FetchOp. If names is supplied, those +// names will be used per node in order instead of generating a unique name. +absl::Status Exporter::AddFetchNode(FuncOp function, + mlir::tf_executor::FetchOp fetch, + llvm::ArrayRef names) { + auto& return_nodes = returns_[fetch]; + for (auto operand_and_idx : llvm::enumerate(fetch.getOperands())) { + if (mlir::isa( + operand_and_idx.value().getType())) + break; + + TF_ASSIGN_OR_RETURN( + auto node_def, + GetReturnNode(function, operand_and_idx.value(), + operand_and_idx.index(), + names.empty() ? "" : names[operand_and_idx.index()])); + TF_ASSIGN_OR_RETURN(Node * node, graph_->AddNode(std::move(*node_def))); + return_nodes.push_back(node); + } + return absl::OkStatus(); +} + +// Collects control ret Nodes based on tf_executor.graph's associated +// tf_executor.fetch control inputs. +absl::Status Exporter::GetControlRetNodes( + mlir::tf_executor::FetchOp fetch, + absl::flat_hash_set* control_ret_nodes) { + for (Value fetch_operand : fetch.getOperands()) { + if (mlir::isa(fetch_operand.getType())) { + Operation* defining_op = + GetIslandInnerOpOrSelf(fetch_operand.getDefiningOp()); + auto node_it = nodes_.find(defining_op); + TF_RET_CHECK(node_it != nodes_.end()); + control_ret_nodes->insert(node_it->second); + } + } + return absl::OkStatus(); +} + +// After conversion from MLIR the input names are all blank which causes +// graph compilation to fail. This uses the edges to fix up the input names. +void FixupInputNamesFromEdges(Graph* graph) { + for (Node* n : graph->nodes()) { + if (n->IsOp()) { + NodeDef* node_def = n->mutable_def(); + node_def->clear_input(); + for (const Edge* e : n->in_edges()) { + Node* src = e->src(); + if (src->IsOp()) { + Graph::AddInput(node_def, src->name(), e->src_output()); + } + } + } + } +} +absl::StatusOr> Exporter::Convert( + const GraphExportConfig& configs, const Dialect* tf_dialect, + const SymbolTable& symbol_table, FuncOp function, + FunctionLibraryDefinition* flib_def, + llvm::SmallDenseSet& visited_functions, + absl::flat_hash_set* control_ret_nodes) { + mlir::Block& block = function.front(); + + // Extract input & output names if set. + llvm::SmallVector input_names; + llvm::SmallVector output_names; + llvm::SmallVector unique_output_names; + auto dict_attr = + function->getAttrOfType(kEntryFuncAttr); + if (dict_attr) { + TF_RET_CHECK(mlir::isa(dict_attr.get("inputs"))) + << "inputs missing in entry function attribute"; + TF_RET_CHECK(mlir::isa(dict_attr.get("outputs"))) + << "outputs missing in entry function attribute"; + mlir::cast(dict_attr.get("inputs")) + .getValue() + .split(input_names, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + mlir::cast(dict_attr.get("outputs")) + .getValue() + .split(output_names, ',', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + } + + auto graph = std::make_unique(OpRegistry::Global()); + + // Extract version info. + VersionDef versions; + auto module = function->getParentOfType(); + if (mlir::succeeded(ExtractTfVersions(module, &versions))) { + graph->set_versions(versions); + } + + Exporter exporter(&configs, graph.get(), tf_dialect, &symbol_table); + + auto graph_op = llvm::cast(block.front()); + + // Set input and output names and increment the use counter for them to help + // generate unique names. + if (!output_names.empty()) { + const int num_data_results = graph_op.getNumResults(); + const int64_t output_names_size = output_names.size(); + TF_RET_CHECK(output_names_size == num_data_results) + << "output names (" << output_names.size() + << ") != terminator operands (" << num_data_results << ")"; + llvm::DenseMap output_op_to_name; + llvm::StringMap name_to_op; + for (const auto& it : llvm::enumerate(graph_op.GetFetch().getOperands())) { + // Skip control rets. + const int64_t index = it.index(); + if (index >= num_data_results) break; + // TODO(jpienaar): If there is a result index specified, ensure only one + // and that it matches the result index of the op. + std::string name(output_names[index]); + auto tensor_id = ParseTensorName(name); + std::string tensor_id_node(tensor_id.node()); + assert(!tensor_id_node.empty() && "expected non-empty name"); + mlir::LegalizeNodeName(tensor_id_node); + + // Ensure name does not get reused. + unique_output_names.push_back( + exporter.op_to_name_.GetUniqueName(tensor_id_node)); + } + } + + if (!input_names.empty()) { + TF_RET_CHECK(input_names.size() == block.getNumArguments()); + for (const auto& it : llvm::enumerate(function.getArguments())) { + // TODO(lyandy): Update when changing feed/fetch import. + std::string name(input_names[it.index()]); + assert(!name.empty() && "expected non-empty name"); + mlir::LegalizeNodeName(name); + auto tensor_id = ParseTensorName(name); + TF_RET_CHECK(tensor_id.index() == 0) + << "input port designation not supported"; + // Only assign user of argument the input name if the main graph did not + // have its _Arg nodes lifted into the functions arguments. + // Ensure name does not get reused. + (void)exporter.op_to_name_.GetUniqueName(name); + } + } + + // Adds nodes for basic block (function) arguments. + for (auto it : llvm::enumerate(block.getArguments())) { + int index = it.index(); + auto arg = it.value(); + mlir::Type type = arg.getType(); + if (!mlir::isa(type)) { + return absl::InvalidArgumentError(absl::StrCat( + "FuncOps arguments must have tensor types. Found ", + mlir::debugString(type), " in function ", function.getName().str())); + } + + TF_RETURN_IF_ERROR(exporter.AddArgumentNode( + arg, index, !input_names.empty() ? input_names[index] : "")); + } + + auto convert_called_function = [&](llvm::StringRef name) { + auto func = symbol_table.lookup(name); + if (func != nullptr) { + TF_RETURN_IF_ERROR(ConvertLibFunction(configs, tf_dialect, symbol_table, + func, flib_def, visited_functions)); + // TODO(prakalps): Optimize to only add the requested function to graph + // library rather than the all the functions exported so far. + TF_RETURN_IF_ERROR(graph->mutable_flib_def()->AddLibrary(*flib_def)); + } + return absl::OkStatus(); + }; + + // Adds nodes for operations. + for (Operation& inst : graph_op.GetBody()) { + for (auto type : inst.getResultTypes()) + if (!mlir::isa(type)) + return absl::InvalidArgumentError(absl::StrCat( + "Values must be of tensor type, TensorFlow control type, or " + "TensorFlow token type. Found ", + mlir::debugString(type))); + + if (llvm::isa(inst)) { + // Skip tf_executor.NextIteration.Source as associated + // tf_executor.NextIteration.Sink will be used instead. + continue; + } else if (auto fetch = llvm::dyn_cast(inst)) { + TF_RETURN_IF_ERROR( + exporter.AddFetchNode(function, fetch, unique_output_names)); + } else if (auto island = + llvm::dyn_cast(inst)) { + Operation& inner_op = island.GetBody().front(); + auto op_name = GetTensorFlowOpName(inner_op.getName().getStringRef()); + if (llvm::isa(inner_op) && op_name.ok()) { + // If it is TF Control dialect specific op, look up custom operation + // in the module and first convert that, then add it to function + // definition library + // TODO(prakalps): If two functions have cyclic dependence, this will + // introduce an infinite loop. + TF_RETURN_IF_ERROR(convert_called_function(op_name.value().str())); + } + + if (IsLegacyCallInstruction(&inner_op)) { + TF_RETURN_IF_ERROR(convert_called_function( + inner_op.getAttrOfType("f") + .getLeafReference() + .getValue())); + } + + TF_RETURN_IF_ERROR(exporter.AddInstructionNode(&inner_op)); + } else { + TF_RETURN_IF_ERROR(exporter.AddInstructionNode(&inst)); + } + } + // Adds edges between the argument, operation and return nodes. + for (Operation& inst : graph_op.GetBody()) { + TF_RETURN_IF_ERROR(exporter.AddEdge(&inst)); + } + // Fixes the edges between the inserted nodes and special "_SOURCE" and + // "_SINK". + FixupSourceAndSinkEdges(graph.get()); + + FixupInputNamesFromEdges(graph.get()); + + TF_RETURN_IF_ERROR( + exporter.GetControlRetNodes(graph_op.GetFetch(), control_ret_nodes)); + + return graph; +} + +absl::Status Exporter::ConvertLibFunction( + const GraphExportConfig& configs, const Dialect* tf_dialect, + const SymbolTable& symbol_table, FuncOp function, + FunctionLibraryDefinition* flib_def, + llvm::SmallDenseSet& visited_functions) { + // Return early if the function has already been exported. + bool is_new_function = visited_functions.insert(function).second; + if (!is_new_function) return absl::OkStatus(); + + auto function_name = FindFunctionName(configs, function); + + // TODO(fengliuai): use a small flib_def to reduce overhead + absl::flat_hash_set control_ret_nodes; + TF_ASSIGN_OR_RETURN( + auto sub_graph, + Exporter::Convert(configs, tf_dialect, symbol_table, function, flib_def, + visited_functions, &control_ret_nodes)); + const auto control_ret = [&](const Node* n) -> std::optional { + return control_ret_nodes.contains(n) + ? std::make_optional(n->name()) + : std::nullopt; + }; + FunctionDef func_def; + TF_RETURN_IF_ERROR( + GraphToFunctionDef(*sub_graph, function_name, control_ret, &func_def)); + + // Checks for gradient attribute. If present converts the gradient function + // and populates the GradientDef. + auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); + if (auto attr = + function->getAttrOfType(grad_string)) { + auto grad_func = symbol_table.lookup(attr.getValue()); + TF_RETURN_IF_ERROR(ConvertLibFunction(configs, tf_dialect, symbol_table, + grad_func, flib_def, + visited_functions)); + GradientDef grad; + grad.set_function_name(function_name); + grad.set_gradient_func(grad_func.getName().str()); + TF_RETURN_IF_ERROR(flib_def->AddGradientDef(grad)); + } + + auto stateful_string = mlir::TF::TensorFlowDialect::GetStatefulAttrName(); + if (auto attr = function->getAttrOfType(stateful_string)) { + func_def.mutable_signature()->set_is_stateful(true); + } + + // Ignore the gradient and is_stateful attribute on the function as they have + // been handled above. Ignore the entry func attribute as it is an MLIR + // metadata attribute and is not required in the function definition. + absl::flat_hash_set attrs_to_ignore = { + grad_string.data(), stateful_string.data(), kEntryFuncAttr}; + llvm::SmallVector funcAttrs( + function->getDialectAttrs()); + TF_RETURN_IF_ERROR(ConvertAttributes(funcAttrs, attrs_to_ignore, + /*remove_ref_type=*/false, + func_def.mutable_attr())); + + for (int i = 0, e = function.getNumArguments(); i < e; ++i) { + if (auto resource_arg_unique_id_attr = + function.getArgAttrOfType( + i, kResourceArgUniqueIdAttr)) { + (*func_def.mutable_resource_arg_unique_id())[i] = + resource_arg_unique_id_attr.getInt(); + } + } + + return flib_def->AddFunctionDef(std::move(func_def)); +} + +absl::Status Exporter::Convert(mlir::ModuleOp module, + const GraphExportConfig& configs, + std::unique_ptr* graph, + FunctionLibraryDefinition* flib_def, + absl::flat_hash_set* control_ret_nodes) { + mlir::StringAttr entry_func_id = + mlir::StringAttr::get(module.getContext(), "main"); + std::optional entry_func; + FunctionLibraryDefinition temp_flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + llvm::SmallDenseSet visited_functions; + auto tf_dialect = module.getContext()->getLoadedDialect("tf"); + // Construct SymbolTable to enable cheap function lookups. The cost + // of constructing the table is offset by the number of queries. + SymbolTable symbol_table(module); + for (auto function : module.getOps()) { + if (function.isExternal()) + return absl::FailedPreconditionError("External functions not supported"); + + if (function.getName() == entry_func_id && + !configs.export_entry_func_to_flib) { + entry_func.emplace(function); + } else { + TF_RETURN_IF_ERROR(ConvertLibFunction(configs, tf_dialect, symbol_table, + function, &temp_flib_def, + visited_functions)); + } + } + + if (flib_def != nullptr) { + TF_RETURN_IF_ERROR(flib_def->AddLibrary(temp_flib_def)); + } + + if (!configs.export_entry_func_to_flib) { + if (!entry_func.has_value()) + return absl::FailedPreconditionError( + "entry function `main` must be present"); + + // Updates the graph and the function library definition. + TF_ASSIGN_OR_RETURN( + *graph, Exporter::Convert(configs, tf_dialect, symbol_table, + entry_func.value(), &temp_flib_def, + visited_functions, control_ret_nodes)); + // Add FunctionDefs and GradientDefs of MLIR functions to graph's function + // library. If duplicate FunctionDefs already exist (can happen if exporter + // had already added some FunctionDefs to the library to support legacy + // calls), they are ignored. + TF_RETURN_IF_ERROR( + graph->get()->mutable_flib_def()->AddLibrary(temp_flib_def)); + } else if (graph != nullptr) { + TF_RETURN_IF_ERROR( + graph->get()->mutable_flib_def()->AddLibrary(std::move(*flib_def))); + } + + return absl::OkStatus(); +} + +} // namespace + +absl::Status ConvertTfExecutorToGraph( + mlir::ModuleOp module, const GraphExportConfig& configs, + std::unique_ptr* graph, FunctionLibraryDefinition* flib_def, + absl::flat_hash_set* control_ret_nodes) { + mlir::StatusScopedDiagnosticHandler sh(module.getContext()); + if (failed(VerifyExportSuitable(module))) return sh.ConsumeStatus(); + return sh.Combine( + Exporter::Convert(module, configs, graph, flib_def, control_ret_nodes)); +} + +absl::Status ConvertMlirFunctionToFunctionLibraryDef( + FuncOp func, const GraphExportConfig& configs, FunctionDef* function_def) { + Dialect* tf_dialect = func.getContext()->getLoadedDialect("tf"); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + llvm::SmallDenseSet visited_functions; + // Construct SymbolTable to enable cheap function lookups. The cost + // of constructing the table is offset by the number of queries. Even + // though this only converts one function in theory, this function + // may have gradient associated which would result in a lookup. This + // could be made lazy if we find this to be broad. + SymbolTable symbol_table(func->getParentOfType()); + TF_RETURN_IF_ERROR(Exporter::ConvertLibFunction( + configs, tf_dialect, symbol_table, func, &flib_def, visited_functions)); + + auto name = FindFunctionName(configs, func); + const FunctionDef* func_def = flib_def.Find(name); + if (func_def != nullptr) { + *function_def = *func_def; + return absl::OkStatus(); + } + return absl::InvalidArgumentError( + absl::StrCat("Function '", name, + "' couldn't be found in the FunctionDefLibrary after " + "converting from MLIR")); +} + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h new file mode 100644 index 0000000000000000000000000000000000000000..f7eee606075b148ccd773404d09f4951552d7cad --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h @@ -0,0 +1,54 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_EXECUTOR_TO_GRAPH_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_EXECUTOR_TO_GRAPH_H_ + +#include "absl/base/attributes.h" +#include "absl/container/flat_hash_set.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { + +// Converts an MLIR module to TensorFlow graph and FunctionLibraryDefinition. +// The "main" function of the module is stored in the graph and the rest of +// functions are stored in the library. Control ret nodes are stored separately +// in `control_ret_nodes`. +absl::Status ConvertTfExecutorToGraph( + mlir::ModuleOp module, const GraphExportConfig& configs, + std::unique_ptr* graph, FunctionLibraryDefinition* flib_def, + absl::flat_hash_set* control_ret_nodes); + +// Converts an MLIR function and adds it to a FunctionLibraryDefinition. +absl::Status ConvertMlirFunctionToFunctionLibraryDef( + mlir::func::FuncOp func, const GraphExportConfig& configs, + FunctionDef* function_def); + +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_API_V2_TF_EXECUTOR_TO_GRAPH_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..202a5119581c1cc85391c62d9b484ee97d81b94e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph_test.cc @@ -0,0 +1,110 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" + +#include + +#include +#include + +#include +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "riegeli/bytes/fd_reader.h" // from @riegeli +#include "riegeli/bytes/read_all.h" // from @riegeli +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace { + +using mlir::DialectRegistry; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/api/v2/testdata/"); +} + +class TfExecutorToGraphTest : public ::testing::Test { + public: + TfExecutorToGraphTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + } + + absl::StatusOr> CreateMlirModule( + std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + return mlir::parseSourceFile(mlir_module_path, &context_); + } + + GraphDef CreateGraphDef(std::string graphdef_filename) { + std::string file_path = TestDataPath() + graphdef_filename; + std::string contents; + GraphDef graph_def; + auto status = riegeli::ReadAll(riegeli::FdReader(file_path), contents); + if (!status.ok()) { + return graph_def; + } + tsl::protobuf::TextFormat::ParseFromString(contents, &graph_def); + return graph_def; + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; +}; + +TEST_F(TfExecutorToGraphTest, ConvertMlirToGraphSucceeds) { + auto valid_executor_module = CreateMlirModule("valid_executor.mlir"); + GraphExportConfig confs; + absl::flat_hash_set control_ret_nodes; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), + FunctionDefLibrary()); + auto result_graph = std::make_unique(flib_def); + + TF_ASSERT_OK(ConvertTfExecutorToGraph(valid_executor_module.value().get(), + confs, &result_graph, &flib_def, + &control_ret_nodes)); + + GraphDef result_graphdef; + result_graph->ToGraphDef(&result_graphdef); + GraphDef expected_graphdef = CreateGraphDef("valid_graph.txt"); + EXPECT_EQ(result_graphdef.DebugString(), expected_graphdef.DebugString()); +} + +} // namespace +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..b24ec190a6e8abfd41123dcf4bb5233201910daa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/BUILD @@ -0,0 +1,429 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load( + "//tensorflow/core/platform:build_config.bzl", + "tf_proto_library", +) +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tensorflow/transforms:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/api/v1:__subpackages__", + "//tensorflow/compiler/mlir/tf2xla/api/v2:__subpackages__", + "//tensorflow/core/tfrt/saved_model:__pkg__", + ], +) + +cc_library( + name = "compilation_timer", + hdrs = ["compilation_timer.h"], + deps = [ + "//tensorflow/core/platform:profile_utils_cpu_utils", + ], +) + +tf_cc_test( + name = "compilation_timer_test", + srcs = ["compilation_timer_test.cc"], + deps = [ + ":compilation_timer", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "test_matchers", + testonly = True, + hdrs = ["test_matchers.h"], + deps = [ + "//tensorflow/compiler/mlir/tf2xla:compile_mlir_util", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "test_matchers_test", + srcs = ["test_matchers_test.cc"], + deps = [ + ":test_matchers", + "//tensorflow/compiler/mlir/tf2xla:compile_mlir_util", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:lib", + "//tensorflow/core/lib/monitoring:cell_reader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest_main", + "@xla//xla/hlo/builder:xla_computation", + "@xla//xla/service:hlo_proto_cc", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "mlir_pass_instrumentation", + srcs = ["mlir_pass_instrumentation.cc"], + hdrs = ["mlir_pass_instrumentation.h"], + deps = [ + "//tensorflow/core/platform:logging", + "@com_google_absl//absl/log", + "@llvm-project//mlir:Pass", + ], +) + +tf_cc_test( + name = "mlir_pass_instrumentation_test", + srcs = ["mlir_pass_instrumentation_test.cc"], + deps = [ + ":mlir_pass_instrumentation", + "//tensorflow/compiler/mlir/tf2xla/api/v1:compile_mlir_util_no_tf_dialect_passes", + "//tensorflow/core:test", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "legalize_tf_mlir", + srcs = ["legalize_tf_mlir.cc"], + hdrs = ["legalize_tf_mlir.h"], + deps = [ + ":compilation_timer", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:set_tpu_infeed_layout", + "//tensorflow/compiler/mlir/tf2xla:compile_mlir_util", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu:tpu_compile", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@stablehlo//:register", + "@tsl//tsl/platform:error_logging", + "@xla//xla:shape_util", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "legalize_tf_mlir_test", + srcs = ["legalize_tf_mlir_test.cc"], + deps = [ + ":legalize_tf_mlir", + ":test_matchers", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:test_main", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@stablehlo//:register", + "@xla//xla:shape_util", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/tsl/platform:errors", + ], +) + +cc_library( + name = "legalize_tf_to_hlo", + srcs = ["legalize_tf_to_hlo.cc"], + hdrs = ["legalize_tf_to_hlo.h"], + deps = [ + ":legalize_tf_mlir", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:set_tpu_infeed_layout", + "//tensorflow/compiler/mlir/tf2xla/api/v1:compile_tf_graph", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_tpu_backend_registration", + "//tensorflow/core:framework", + "//tensorflow/core/platform:status", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@stablehlo//:register", + "@xla//xla:shape_util", + "@xla//xla/client:compile_only_client", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "legalize_tf_to_hlo_test", + srcs = ["legalize_tf_to_hlo_test.cc"], + deps = [ + ":legalize_tf_to_hlo", + ":test_matchers", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/monitoring:cell_reader", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@stablehlo//:register", + "@xla//xla:shape_util", + "@xla//xla/client:client_library", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/stream_executor:platform", + "@xla//xla/stream_executor:platform_manager", + "@xla//xla/tsl/platform:errors", + ], +) + +cc_library( + name = "clustering_bridge_passes", + srcs = ["clustering_bridge_passes.cc"], + hdrs = ["clustering_bridge_passes.h"], + deps = [ + "//tensorflow/compiler/jit:flags_headers", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tensorflow/transforms/sparsecore:sparsecore_passes", + "//tensorflow/compiler/mlir/tf2xla/internal/passes:clustering_passes", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Transforms", + ], +) + +tf_cc_test( + name = "clustering_bridge_passes_test", + srcs = ["clustering_bridge_passes_test.cc"], + deps = [ + ":clustering_bridge_passes", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "logging_hooks", + srcs = ["logging_hooks.cc"], + hdrs = ["logging_hooks.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/core:framework", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Pass", + ], +) + +tf_cc_test( + name = "logging_hooks_test", + srcs = ["logging_hooks_test.cc"], + data = [ + "testdata/dead_const.mlir", + ], + deps = [ + ":logging_hooks", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Transforms", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:status", + ], +) + +cc_library( + name = "mlir_bridge_pass_util", + srcs = ["mlir_bridge_pass_util.cc"], + hdrs = ["mlir_bridge_pass_util.h"], + visibility = ["//tensorflow/compiler/tf2xla:__pkg__"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:tf2xla_defs", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/common_runtime:function_body", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:status", + ], + alwayslink = 1, +) + +tf_cc_test( + name = "mlir_bridge_pass_util_test", + srcs = ["mlir_bridge_pass_util_test.cc"], + deps = [ + ":mlir_bridge_pass_util", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "//tensorflow/cc:array_ops", + "//tensorflow/cc:function_ops", + "//tensorflow/cc:functional_ops", + "//tensorflow/cc:ops", + "//tensorflow/cc:scope", + "//tensorflow/cc:tpu_ops", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:tf2xla_defs", + "//tensorflow/core/protobuf:for_core_protos_cc", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core/platform:types", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/platform:enable_tf2_utils", + # "//tensorflow/core/platform:resource_loader", + "@xla//xla/tsl/lib/core:status_test_util", + ], +) + +tf_proto_library( + name = "reproducer_proto", + srcs = ["reproducer.proto"], + protodeps = [ + "//tensorflow/core:protos_all", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto", + ], + visibility = [ + "//learning/brain/mlir/bridge:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/api/v2:__pkg__", + ], +) + +cc_library( + name = "node_order", + srcs = ["node_order.cc"], + hdrs = ["node_order.h"], + deps = [ + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +tf_cc_test( + name = "node_order_test", + size = "small", + srcs = [ + "node_order_test.cc", + ], + deps = [ + ":node_order", + "//tensorflow/cc:cc_ops", + "//tensorflow/cc:cc_ops_internal", + "//tensorflow/cc:function_ops", + "//tensorflow/cc:sendrecv_ops", + "//tensorflow/core", + "//tensorflow/core:core_cpu", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:direct_session_internal", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:ops", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "@com_google_absl//absl/log:check", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "graph_to_tf_executor_util", + srcs = ["graph_to_tf_executor_util.cc"], + hdrs = ["graph_to_tf_executor_util.h"], + deps = [ + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:framework_types_hdr", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/common_runtime:function_body", + "//tensorflow/core/platform:enable_tf2_utils", + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + ], +) + +tf_cc_test( + name = "graph_to_tf_executor_util_test", + srcs = ["graph_to_tf_executor_util_test.cc"], + deps = [ + ":graph_to_tf_executor_util", + "//tensorflow/cc:array_ops", + "//tensorflow/cc:cc_ops", + "//tensorflow/cc:functional_ops", + "//tensorflow/cc:ops", + "//tensorflow/cc:scope", + "//tensorflow/cc:tpu_ops", + "//tensorflow/compiler/tf2xla/ops:xla_ops", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:tensor_testutil", + "//tensorflow/core/platform:enable_tf2_utils", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@xla//xla/tsl/lib/core:status_test_util", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.cc new file mode 100644 index 0000000000000000000000000000000000000000..6bf0f09b240eea61fcecd42de5cebfbedeac08ae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.cc @@ -0,0 +1,220 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h" + +#include + +#include "absl/log/log.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +using mlir::OpPassManager; +using mlir::func::FuncOp; + +// Adds replicated Bridge clustering pipeline passes to the given pass_manager. +// Does not run them. +void AddReplicatedBridgeClusteringPipelinePasses(OpPassManager& pm, + llvm::StringRef module_name) { + // The following ops must be preserved regardless of reachability. Ideally, + // all graphs should have control dependencies to enforce this but this is + // currently not the case (see b/177478741). + const llvm::SmallVector ops_to_preserve = { + "tf.TPUReplicateMetadata", "tf.TPUCompilationResult", + "tf.TPUReplicatedOutput"}; + bool strict_clusters = + tensorflow::GetMlirCommonFlags()->tf_mlir_enable_strict_clusters; + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass(ops_to_preserve)); + // It is assumed at this stage there are no V1 control flow ops as Graph + // functionalization is ran before import. Ops can be lifted out of + // tf_executor dialect islands/graphs. + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + // Guarantee all functions have one use, which enables more exact shape + // inference. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + // Run shape inference so that tf_executor/tf_device ops created later will + // likely to inherit more concrete types. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addNestedPass(mlir::TFTPU::CreateTPUPartitionedOpConversionPass()); + pm.addNestedPass( + mlir::TFTPU::CreateTPUReorderReplicateAndPartitionedInputsPass()); + pm.addNestedPass(mlir::TF::CreateDecomposeReduceDatasetPass()); + // Only one of EmbeddingSequencing and EmbeddingPipelining will actually + // run and the logic is in EmbeddingPipeliningPass. If the pipelining pass + // runs, embedding attributes are stripped and the sequencing pass will have + // no effect. If the pipelining pass doesn't run, embedding attributes are + // preserved and the sequencing rewrite will trigger. + pm.addPass(mlir::TFDevice::CreateEmbeddingPipeliningPass()); + pm.addPass(mlir::TFDevice::CreateEmbeddingSequencingPass()); + pm.addPass(tensorflow::tf2xla::internal::CreateTPUClusterFormationPass( + strict_clusters)); + // CreateEmbeddingPipeliningPass may have created more functions, but + // TPUClusterCleanup and OutsideCompiledToHostLaunch need every function to be + // only called from one cluster. Here, we choose to fix the all-funcs-one-use + // invariant right before it's needed, not after it's been broken. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + // Run TPU cluster cleanup attributes so ops with no outside compiled + // attribute have no host device attribute. + pm.addPass(mlir::TFTPU::CreateTPUClusterCleanupAttributesPass()); + pm.addNestedPass(mlir::TFDevice::CreateDeviceAttributeToLaunchPass()); + // Running canonicalizer before decomposing resource ops in cluster helps the + // latter pass to converge faster as it does not have to spend time folding + // away dead ops. + pm.addNestedPass(mlir::createCanonicalizerPass()); + // Place DecomposeResourceOpsPass before TFExecutorConstantSinking pass + // because DecomposeResourceOpsPass uses pattern rewriter which hoists + // changed constants out of tf_device.Launch. + pm.addPass(mlir::TFDevice::CreateDecomposeResourceOpsInClusterPass()); + // Encode this in its own scope so that func_pm is not mistakenly used + // later on. + { + OpPassManager& func_pm = pm.nest(); + func_pm.addPass(mlir::TFTPU::CreateTPUHostComputationExpansionPass()); + func_pm.addPass(mlir::TFTPU::CreateTPUUpdateEmbeddingEnqueueOpInputsPass()); + } + + // TODO(b/173622615): Once OutsideCompilation is represented by launch op and + // the remaining passes including Inliner support it, remove this + // LaunchToDeviceAttributePass. This LaunchToDeviceAttribute pass needs to + // come before TPUClusterCleanupAttributes pass or else the device attribute + // will be removed from launch causing an error. + pm.addNestedPass(mlir::TFDevice::CreateLaunchToDeviceAttributePass()); + + // Note that the region-based control-flow produced here still contains + // function call ops which get inlined by the subsequent inliner pass. + pm.addPass(mlir::TF::CreateTFFunctionalControlFlowToRegions()); + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass( + mlir::TF::CreateDropWhileShapeInvariantInDeviceClusterPass()); + // Run another shape inference pass because resource decomposition might have + // created new partial types. Also, after dropping `shape_invariant` attribute + // from While/WhileRegion ops within cluster would lead to more precise + // shapes. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::TFTPU::CreateTPUClusterCleanupAttributesPass()); + pm.addPass(mlir::TFDevice::CreateResourceOpLiftingPass()); + // Re-run the canonicalizer pass as some cleanup during resource op lifting + // pass opens up some opportunities for canonicalization of cluster ops. + // Specifically, we want to eliminate pass through results from the cluster + // op. + pm.addNestedPass(mlir::createCanonicalizerPass()); + + pm.addNestedPass(mlir::createCSEPass()); + if (tensorflow::GetMlirCommonFlags() + ->tf_mlir_enable_merge_control_flow_pass) { + pm.addPass(mlir::TFDevice::CreateMergeControlFlowPass()); + } + + pm.addPass( + tensorflow::tf2xla::internal::CreateMarkOpsForOutsideCompilationPass()); + pm.addPass(tensorflow::tf2xla::internal:: + CreateExtractHeadTailOutsideCompilationPass()); + pm.addPass( + tensorflow::tf2xla::internal::CreateExtractOutsideCompilationPass()); + pm.addNestedPass( + mlir::TFDevice::CreateVerifyNoOutsideCompilationMarkersPass()); + + pm.addNestedPass(mlir::TFDevice::CreateClusterConstantSinkingPass()); + pm.addPass(mlir::TF::CreateResourceDeviceInferencePass()); + pm.addNestedPass( + tensorflow::tf2xla::internal::CreateHoistBroadcastReadPass()); + pm.addNestedPass( + tensorflow::tf2xla::internal::CreateXlaBroadcastPass()); + pm.addPass(mlir::TFDevice::CreateClusterOutliningPass()); + pm.addPass(mlir::TFTPU::CreateTPUResourceReadForWritePass()); + pm.addPass(mlir::TFDevice::CreateMarkInputOutputAliasesPass()); + pm.addPass( + tensorflow::tf2xla::internal::CreateTPUShardingIdentificationPass()); + pm.addNestedPass( + mlir::TFTPU::CreateTPUResourceReadsWritesPartitioningPass()); + pm.addPass(mlir::TFDevice::CreateAnnotateParameterReplicationPass()); + pm.addNestedPass(mlir::TF::CreateRewriteTPUEmbeddingOpsPass()); + pm.addPass(mlir::TFTPU::CreateTPUAnnotateDynamicShapeInputsPass()); + pm.addNestedPass( + mlir::TF::CreateHoistReplicateInvariantResourceWritesPass()); + // Verifies clustering has conformed with the expected invariants + pm.addNestedPass( + tensorflow::tf2xla::internal::CreateVerifyClusteringPass()); +} + +void NoCanonicalization(OpPassManager& pm) {} + +// Same as above but for non-replicated Bridge. +void AddNonReplicatedBridgeClusteringPipelinePasses(OpPassManager& pm) { + // The following ops must be preserved regardless of reachability. Ideally, + // all graphs should have control dependencies to enforce this. + VLOG(2) << "Create TF XLA Bridge pipeline"; + pm.addPass(mlir::TFDevice::CreateXlaValidateInputsPass()); + pm.addNestedPass( + mlir::TF::CreateCanonicalizeCompileAndReplicateAttributesPass()); + const llvm::SmallVector ops_to_preserve = {}; + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass(ops_to_preserve)); + // It is assumed at this stage there are no V1 control flow ops as Graph + // functionalization is ran before import. Ops can be lifted out of + // tf_executor dialect islands/graphs. + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + // Guarantee all functions have one use, which enables more exact shape + // inference. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + // Encapsulate PartitionedCall ops within a cluster so that the composite + // resource ops can be decomposed. + pm.addPass(tensorflow::tf2xla::internal::CreateXlaClusterFormationPass()); + // Running canonicalizer before decomposing resource ops in cluster helps the + // latter pass to converge faster as it does not have to spend time folding + // away dead ops. + pm.addNestedPass(mlir::createCanonicalizerPass()); + // Decompose resource ops. + pm.addPass(mlir::TFDevice::CreateDecomposeResourceOpsInClusterPass()); + // Run another shape inference pass because resource decomposition might have + // created new partial types. Also, after dropping `shape_invariant` attribute + // from While/WhileRegion ops within cluster would lead to more precise + // shapes. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + // Inline all the function calls. Do not call canonicalizer to prevent it from + // moving the definition of any constant operand of ops within a cluster to + // its outside. This may cause the op to fail to verify after the cluster is + // outlined, as the constant operand is replaced by an argument. + pm.addPass(mlir::createInlinerPass({}, NoCanonicalization)); + // Lift resource operations out of device computation. This step needs to be + // done after inlining. + pm.addPass(mlir::TFDevice::CreateResourceOpLiftingPass()); + // Outline clusters into cluster functions. + pm.addPass(mlir::TFDevice::CreateClusterOutliningPass()); + // Verifies clustering has conformed with the expected invariants + pm.addNestedPass( + tensorflow::tf2xla::internal::CreateVerifyClusteringPass()); +} + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..b45cc662b39d0058a48ebb60a30ef278d9f8ce24 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h @@ -0,0 +1,39 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_CLUSTERING_BRIDGE_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_CLUSTERING_BRIDGE_PASSES_H_ + +#include "absl/base/attributes.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/PassManager.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Given the pass manager, add Bridge passes to cluster the replicated input +// graphs. +void AddReplicatedBridgeClusteringPipelinePasses( + mlir::OpPassManager& pm, llvm::StringRef module_name = llvm::StringRef()); + +// Same as above but for non replicated graphs. +void AddNonReplicatedBridgeClusteringPipelinePasses(mlir::OpPassManager& pm); + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_CLUSTERING_BRIDGE_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..09067b1f29eba30e58f95204558aa8f819eceda2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes_test.cc @@ -0,0 +1,43 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/clustering_bridge_passes.h" + +#include +#include "mlir/Pass/PassManager.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +using mlir::OpPassManager; + +TEST(ClusteringBridgePassesTest, AddsBridgePasses) { + OpPassManager pass_manager; + AddReplicatedBridgeClusteringPipelinePasses(pass_manager); + + EXPECT_EQ(pass_manager.size(), 45); +} + +TEST(ClusteringBridgePassesTest, AddsNonTPUBridgePasses) { + OpPassManager pass_manager; + AddNonReplicatedBridgeClusteringPipelinePasses(pass_manager); + + EXPECT_EQ(pass_manager.size(), 15); +} + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h new file mode 100644 index 0000000000000000000000000000000000000000..e58395f2602b71e71970ab1df408c42d9ae4d3ac --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h @@ -0,0 +1,43 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_COMPILATION_TIMER_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_COMPILATION_TIMER_H_ + +#include // NOLINT(build/c++11) + +#include "tensorflow/core/platform/profile_utils/cpu_utils.h" + +// Time the execution of kernels (in CPU cycles). Meant to be used as RAII. +struct CompilationTimer { + uint64_t start_cycles = + tensorflow::profile_utils::CpuUtils::GetCurrentClockCycle(); + + uint64_t ElapsedCycles() { + return tensorflow::profile_utils::CpuUtils::GetCurrentClockCycle() - + start_cycles; + } + + int64_t ElapsedCyclesInMilliseconds() { + std::chrono::duration duration = + tensorflow::profile_utils::CpuUtils::ConvertClockCycleToTime( + ElapsedCycles()); + + return std::chrono::duration_cast(duration) + .count(); + } +}; + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_COMPILATION_TIMER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..8da73a09b095973f9320c2b8078e0d81b2fda5ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/compilation_timer_test.cc @@ -0,0 +1,41 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h" + +#include + +#include +#include +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace { + +TEST(CompilationTimer, MeasuresElapsedTime) { + uint64_t timer_result_in_milliseconds; + + { + CompilationTimer timer; + + absl::SleepFor(absl::Milliseconds(100)); + + timer_result_in_milliseconds = timer.ElapsedCyclesInMilliseconds(); + } + + ASSERT_THAT(timer_result_in_milliseconds, testing::Ne(0)); +} + +} // namespace diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..26a044e62f16bae2d74b63b44c634cbec099bdb8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.cc @@ -0,0 +1,332 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h" + +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/common_runtime/function_body.h" +#include "tensorflow/core/common_runtime/function_def_utils.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/platform/enable_tf2_utils.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +namespace { +// Internal encapsulation of state for the MLIR bridge graph analyzer. Steps +// through the nodes in the graph and reachable functions, tracking whether +// each feature of interest is found. +// +// Tracks the presence of each feature of interest in the corresponding streamz +// metric. Note that the graph traversal does not terminate early so as to +// capture all of these features. +class MlirBridgeGraphAnalyzer { + public: + explicit MlirBridgeGraphAnalyzer(bool single_core_inference_mode) + : single_core_inference_mode_(single_core_inference_mode) {} + ~MlirBridgeGraphAnalyzer() = default; + // Not copyable or movable. + MlirBridgeGraphAnalyzer(const MlirBridgeGraphAnalyzer&) = delete; + MlirBridgeGraphAnalyzer& operator=(const MlirBridgeGraphAnalyzer&) = delete; + + // Analyzes whether the graph has features not guaranteed to be supported by + // the MLIR-based TF XLA bridge. + bool HasUnsupportedFeatures(const Graph& graph, + const FunctionLibraryDefinition* function_library, + std::optional config_proto, + tensorflow::TF2XLABridgeVersion bridge_version) { + // Non-ok status is considered as "unsupported" since this means something + // is wrong or unexpected with the graph itself. + invalid_graph_ = + invalid_graph_ || !AnalyzeGraphAndReachableFunctions( + graph, function_library, config_proto) + .ok(); + + // We conservatively consider the graph to be unsupported if it's not + // *known* to be TF2. That is, graphs that have kNotTracked construction + // context are considered unsupported, even though they might in fact be + // TF2 models. + auto construction_context = graph.GetConstructionContextInternal(); + bool is_tf2 = construction_context == ConstructionContext::kEagerRuntime; + auto is_tf2_execution_enabled = tensorflow::tf2_execution_enabled(); + auto has_unsupported_features = false; + auto is_v1_compat = bridge_version == TF2XLABridgeVersion::kV1Compat; + auto is_nominal_bridge = bridge_version == TF2XLABridgeVersion::kNominal; + auto is_tfrt_bridge = bridge_version == TF2XLABridgeVersion::kTFRTNominal; + is_eager_compliant_ = is_tf2_execution_enabled || is_tf2 || + is_nominal_bridge || is_tfrt_bridge; + + is_eager_compliant_ |= (is_v1_compat && contains_partitioned_call_); + + has_unsupported_features = contains_ref_type_ || invalid_graph_; + + // For non single core inference mode, checking conditions: + if (!single_core_inference_mode_) { + has_unsupported_features |= + !is_eager_compliant_ || uses_v1_control_flow_ || + HasTpuReplicatedCoreUnsupportedFeature(is_nominal_bridge, + is_v1_compat, is_tfrt_bridge); + } + + PrintGraphUnsupportedFeatures(is_tf2, is_tf2_execution_enabled, + is_v1_compat, is_tfrt_bridge, + is_nominal_bridge, has_unsupported_features); + + // Determine whether or not the graph contains unsupported features. + return has_unsupported_features; + } + + private: + static constexpr char kPartitionedCall[] = "TPUPartitionedCall"; + + bool HasTPUReplicatedCoreAttr(const Node& node) { + constexpr absl::string_view kTPUReplicatedCore = "TPU_REPLICATED_CORE"; + const std::string& device = node.requested_device(); + if (!device.empty()) { + DeviceNameUtils::ParsedName name; + if (DeviceNameUtils::ParseFullName(device, &name)) { + // The TPU_REPLICATED_CORE attrs is not relevant for single TPU core + // inference. + // TODO(b/201091475): this can be generalized to check + // num_cores_per_replica != 1, rather than being special cased for + // single core inference. + if (name.type == kTPUReplicatedCore && !single_core_inference_mode_) { + return true; + } + } + } + return false; + } + + bool HasTpuReplicatedCoreUnsupportedFeature(bool is_nominal_bridge, + bool is_v1_compat, + bool is_tfrt_bridge) { + if (!has_tpu_replicated_core_) { + return false; + } + return has_infeed_dequeue_tuple_with_tpu_replicated_core_; + } + + void PrintGraphUnsupportedFeatures(bool is_tf2, bool is_tf2_execution_enabled, + bool is_v1_compat, bool is_tfrt_bridge, + bool is_nominal_bridge, + bool has_unsupported_features) { + if (!has_unsupported_features) { + VLOG(1) << "Graph doesn't have unsupported features"; + return; + } + + LOG(INFO) + << "Graph has unsupported features: " << (is_tf2 ? "" : "not is_tf2, ") + << (is_tf2_execution_enabled ? "" : "not tf2_execution, ") + << (is_nominal_bridge ? "" : "not nominal bridge, ") + << (is_tfrt_bridge ? "" : "not tfrt bridge, ") + << (is_v1_compat && contains_partitioned_call_ + ? "contains partitioned calls at v1 compat bridge call site, " + : "") + << (contains_ref_type_ ? "contains ref variables, " : "") + << (invalid_graph_ ? "Invalid graph, " : "") + << (uses_v1_control_flow_ ? "uses control flow v1 " : "") + << ((has_tpu_replicated_core_ && + has_infeed_dequeue_tuple_with_tpu_replicated_core_) + ? "InfeedDequeueTuple op with TPU_REPLICATED_CORE attr, " + : ""); + } + + // Traverses each node in the graph and gathers information about each of the + // features. Specifically, sets the relevant class variable to true when a + // feature is found. + void AnalyzeGraphNodes(const Graph& graph) { + constexpr absl::string_view kIdentityOp = "Identity"; + constexpr absl::string_view kIdentityNOp = "IdentityN"; + constexpr absl::string_view kCastOp = "Cast"; + constexpr absl::string_view kInfeedDequeueTuple = "InfeedDequeueTuple"; + constexpr absl::string_view kOutsideCompilationAttr = + "_xla_outside_compilation"; + constexpr absl::string_view kAllowSoftPlacementAttr = + "allow_soft_placement"; + constexpr absl::string_view kManualControlDepsAttr = + "_has_manual_control_dependencies"; + + auto has_ref_type = [](const DataTypeVector& types) { + for (const DataType& dtype : types) + if (IsRefType(dtype)) return true; + return false; + }; + + for (const Node* node : graph.nodes()) { + contains_ref_type_ = + (contains_ref_type_ || has_ref_type(node->input_types()) || + has_ref_type(node->output_types())); + contains_partitioned_call_ = (contains_partitioned_call_ || + node->type_string() == kPartitionedCall); + uses_v1_control_flow_ = (uses_v1_control_flow_ || node->IsControlFlow()); + uses_outside_compilation_ = + (uses_outside_compilation_ || + node->attrs().Find(kOutsideCompilationAttr) != nullptr); + has_manual_control_deps_ = (has_manual_control_deps_ || + node->attrs().Find(kManualControlDepsAttr)); + + auto soft_placement_attr = node->attrs().Find(kAllowSoftPlacementAttr); + if (soft_placement_attr != nullptr) { + uses_outside_compilation_ = + (uses_outside_compilation_ || soft_placement_attr->b()); + } + + // TODO(b/187611527): Add support for the ops with explicit device + // assignment on the TPU_REPLICATED_CORE. + if (node->type_string() == kIdentityOp || + node->type_string() == kCastOp || + node->type_string() == kIdentityNOp) { + if (HasTPUReplicatedCoreAttr(*node)) { + has_tpu_replicated_core_ = true; + VLOG(2) << node->type_string() + << " node has TPU_REPLICATED_CORE attribute."; + } + } + if (node->type_string() == kInfeedDequeueTuple && + HasTPUReplicatedCoreAttr(*node)) { + has_infeed_dequeue_tuple_with_tpu_replicated_core_ = true; + } + } + } + + // Analyze all functions from the flib_def if there are any that belong to + // the inference graph. + void AnalyzeInferenceGraphs(const FunctionLibraryDefinition& flib_def) { + if (contains_partitioned_call_) return; + + for (const std::string& func_name : flib_def.ListFunctionNames()) { + const FunctionDef* func_def = flib_def.Find(func_name); + for (const NodeDef& node_def : func_def->node_def()) { + contains_partitioned_call_ = node_def.op() == kPartitionedCall; + if (contains_partitioned_call_) return; + } + } + } + + // Checks any reachable functions from `graph_def` in `flib_def` + // for unsupported features in the MLIR-based bridge. + // + // Returns failure in the event that the FunctionDef fails to convert to + // FunctionBody. Otherwise returns success. + absl::Status AnalyzeReachableFunctions( + const GraphDef& graph_def, const FunctionLibraryDefinition& flib_def) { + // Check the inputs and outputs of a function for reference variables. + auto signature_contains_ref_type = [](const OpDef& signature) { + for (const auto& args : {signature.input_arg(), signature.output_arg()}) { + for (const auto& arg : args) { + if (IsRefType(arg.type())) return true; + } + } + return false; + }; + + for (const std::string& func_name : + flib_def.ReachableDefinitions(graph_def).ListFunctionNames()) { + const FunctionDef* func_def = flib_def.Find(func_name); + if (func_def->has_signature()) { + contains_ref_type_ = contains_ref_type_ || + signature_contains_ref_type(func_def->signature()); + } + // Check the function body. + std::unique_ptr func_body; + TF_RETURN_IF_ERROR(FunctionDefToBodyHelper( + *func_def, AttrSlice(&func_def->attr()), &flib_def, &func_body)); + AnalyzeGraphNodes(*func_body->graph); + } + return absl::OkStatus(); + } + + // Checks the inputted graph for any features which aren't supported in the + // MLIR-based bridge, stepping through each node in the graph as well as any + // reachable functions (inputs, outputs, and function body). + // + // Note that this analysis does not terminate early because we care about + // collecting all of these metrics. + // + // Returns failure in the event that the FunctionDef fails to convert to + // FunctionBody. Otherwise returns success. + absl::Status AnalyzeGraphAndReachableFunctions( + const Graph& graph, const FunctionLibraryDefinition* function_library, + std::optional config_proto) { + // First, check whether soft placement is enabled. This means that auto + // outside compilation may be used. + uses_outside_compilation_ = + uses_outside_compilation_ || + (config_proto.has_value() && config_proto->allow_soft_placement()); + + // Analyze each node in this graph. + AnalyzeGraphNodes(graph); + + // Then check any associated functions in the graph + // FunctionLibraryDefinition. + GraphDef graph_def; + graph.ToGraphDef(&graph_def); + TF_RETURN_IF_ERROR(AnalyzeReachableFunctions(graph_def, graph.flib_def())); + // Analyze whether there is an inference graph, including non reachable + // from the `graph` itself. This happens when there is a sequence of + // TPUPartitionedCall()->main()->PartitionedCall() and only second part + // of the graph is processed by the MLIR bridge. + AnalyzeInferenceGraphs(graph.flib_def()); + + // Check any associated function in the graph defined in a separate + // FunctionLibraryDefinition. + if (function_library != nullptr) { + TF_RETURN_IF_ERROR( + AnalyzeReachableFunctions(graph_def, *function_library)); + AnalyzeInferenceGraphs(*function_library); + } + + return absl::OkStatus(); + } + + bool contains_partitioned_call_ = false; + bool contains_ref_type_ = false; + bool invalid_graph_ = false; + bool uses_outside_compilation_ = false; + bool uses_v1_control_flow_ = false; + bool has_manual_control_deps_ = false; + bool single_core_inference_mode_ = false; + bool is_eager_compliant_ = false; + bool has_tpu_replicated_core_ = false; + bool has_infeed_dequeue_tuple_with_tpu_replicated_core_ = false; +}; + +} // namespace + +bool GraphHasUnsupportedFeaturesInMlirBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library, + std::optional config_proto, TF2XLABridgeVersion bridge_version, + bool single_core_inference_mode) { + return MlirBridgeGraphAnalyzer(single_core_inference_mode) + .HasUnsupportedFeatures(graph, function_library, config_proto, + bridge_version); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h new file mode 100644 index 0000000000000000000000000000000000000000..6a629d5054a3d6caba445cfc9a8b357e37399e8e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h @@ -0,0 +1,65 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_GRAPH_TO_TF_EXECUTOR_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_GRAPH_TO_TF_EXECUTOR_UTIL_H_ + +#include + +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace tensorflow { + +// These are used for grouping the recorded stats appropriately. Specifically, +// we're considering different entrypoints to the bridge as having potentially +// interesting differences at least in the domain of accepted graphs so we want +// to separately track graph features based on these unique entrypoints. One key +// example of this distinction is for TFRT which uses the "nominal" TPU bridge +// pipeline, but may potentially allow graphs with v1 control flow. This +// separate grouping will allow us to dig into these differences granularly. +enum class TF2XLABridgeVersion { + kNominal = 0, + kV1Compat, + kTFRTNominal, + kNotBridgeUseCase, +}; + +// Analyzes whether the graph has features not guaranteed to be supported by the +// MLIR-based TF XLA bridge for phase 1. If MLIR bridge phase 1 is not used, +// then MLIR bridge phase 2 will not be used. The optional `function_library` +// can be provided if it contains function definitions not including in the +// `graph` FunctionLibraryDefinition. +// +// Conservatively, during the initial rollout, we are not supporting graphs for +// which any of the following are true: +// +// - Not known to be TF2 +// - Contains one or more reference variables +// - Contains one or more TPUPartitionedCall ops (which is a proxy for +// inference), but the graph is not v1 compat +// - Uses V1 control flow +// - Graph is invalid or otherwise encounters error during traversal +// If `single_core_inference_mode` is true, we skip some of check conditions +// because they are not applicable. +// TODO(b/241702857): remove single_core_inference_mode +bool GraphHasUnsupportedFeaturesInMlirBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library, + std::optional config_proto, TF2XLABridgeVersion bridge_version, + bool single_core_inference_mode); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_GRAPH_TO_TF_EXECUTOR_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..7a8ca740577cebc3683856583c8dd8bf5eb57b6a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util_test.cc @@ -0,0 +1,736 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/graph_to_tf_executor_util.h" + +#include +#include +#include +#include + +#include +#include "absl/log/check.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "tensorflow/cc/framework/ops.h" +#include "tensorflow/cc/framework/scope.h" +#include "tensorflow/cc/ops/array_ops.h" +#include "tensorflow/cc/ops/control_flow_ops.h" +#include "tensorflow/cc/ops/functional_ops.h" +#include "tensorflow/cc/ops/tpu_functional_ops.h" +#include "tensorflow/cc/ops/tpu_replication_ops.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/platform/enable_tf2_utils.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace tensorflow { + +namespace { + +REGISTER_OP("OneRefOutput").Output("y: Ref(float)"); + +FunctionDef XTimesTwo() { + const Tensor kTwo = test::AsScalar(2); + return FunctionDefHelper::Define( + // Name + "XTimesTwo", + // Args + {"x: T"}, + // Return values + {"y: T"}, + // Attr def + {"T: {float, double, int32, int64}"}, + // Nodes + { + {{"two"}, "Const", {}, {{"value", kTwo}, {"dtype", DT_INT64}}}, + {{"scale"}, "Cast", {"two"}, {{"SrcT", DT_INT64}, {"DstT", "$T"}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", "$T"}}}, + }); +} + +FunctionDef XTimesTwoFloat() { + const Tensor kTwo = test::AsScalar(2); + return FunctionDefHelper::Define( + // Name + "XTimesTwoFloat", + // Args + {"x: float"}, + // Return values + {"y: float"}, + // Attr def + {}, + // Nodes + { + {{"two"}, "Const", {}, {{"value", kTwo}, {"dtype", DT_INT64}}}, + {{"scale"}, + "Cast", + {"two"}, + {{"SrcT", DT_INT64}, {"DstT", DT_FLOAT}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", DT_FLOAT}}}, + }); +} + +FunctionDef XTimesTwoFloatRef() { + const Tensor kTwo = test::AsScalar(2); + return FunctionDefHelper::Define( + // Name + "XTimesTwoFloatRef", + // Args + {"x: float"}, + // Return values + {"y: float"}, + // Attr def + {}, + // Nodes + { + {{"two"}, "Const", {}, {{"value", kTwo}, {"dtype", DT_INT64_REF}}}, + {{"scale"}, + "Cast", + {"two"}, + {{"SrcT", DT_INT64_REF}, {"DstT", DT_FLOAT}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", DT_FLOAT}}}, + }); +} + +Node* FromNodeDef(absl::string_view name, absl::string_view node_type, + int num_inputs, DataType dt, Graph& graph) { + auto builder = NodeDefBuilder(name, node_type); + for (int i = 0; i < num_inputs; ++i) { + builder = builder.Input(absl::StrCat("node_", i), i, dt); + } + + NodeDef node_def; + CHECK_OK(builder.Finalize(&node_def)); + + absl::Status s; + Node* node = graph.AddNode(node_def, &s); + CHECK_OK(s); + return node; +} + +TEST(SupportedGraphTest, SupportedGraphReturnsFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + tensorflow::set_tf2_execution(true); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(InvalidGraphTest, InvalidFuncBodyReturnsTrue) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwo(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwo"); + ops::PartitionedCall f(root.WithOpName("f"), {x}, {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + // The call to XTimesTwo is invalid (missing an attribute), so we expect the + // graph to be unsupported. + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(RefVarTest, RefVariablesReturnsTrue) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output cond_a = ops::Placeholder(root.WithOpName("cond_a"), DT_BOOL); + Output cond_b = ops::Placeholder(root.WithOpName("cond_b"), DT_BOOL); + + // Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT); + tensorflow::set_tf2_execution(true); + const std::vector shape_array{2, 2}; + auto shape = TensorShape(); + TF_ASSERT_OK(TensorShapeUtils::MakeShape(shape_array, &shape)); + Output value = Output( + FromNodeDef("value", "OneRefOutput", 0, DT_FLOAT_REF, *root.graph())); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(RefVarTest, NoRefVariablesCalleeFuncReturnsFalse) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::PartitionedCall f(root.WithOpName("f"), {x}, {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(RefVarTest, RefVariablesInCalleeFunctionReturnsTrue) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloatRef(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloatRef"); + ops::PartitionedCall f(root.WithOpName("f"), {x}, {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(RefVarTest, RefVariablesInExternalCalleeFunctionReturnsTrue) { + tensorflow::set_tf2_execution(true); + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloatRef(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloatRef"); + ops::PartitionedCall f(root.WithOpName("f"), {x}, {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/&flib_def, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, ContainsInferenceNodeEagerRuntimeReturnsTrue) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, ContainsInferenceNodeTFRTBridgeReturnsTrue) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kTFRTNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, ContainsInferenceNodeDirectSessionReturnsFalse) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(ControlFlowTest, ContainsV1ControlFlowReturnsTrue) { + tensorflow::set_tf2_execution(true); + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output cond_a = ops::Placeholder(root.WithOpName("cond_a"), DT_BOOL); + Output cond_b = ops::Placeholder(root.WithOpName("cond_b"), DT_BOOL); + + Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT); + + ops::Switch switch_a(root.WithOpName("switch_a"), value, cond_a); + ops::Switch switch_b(root.WithOpName("switch_b"), value, cond_b); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(ControlFlowTest, TFRTContainsV1ControlFlowReturnsTrue) { + tensorflow::set_tf2_execution(true); + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output cond_a = ops::Placeholder(root.WithOpName("cond_a"), DT_BOOL); + Output cond_b = ops::Placeholder(root.WithOpName("cond_b"), DT_BOOL); + + Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT); + + ops::Switch switch_a(root.WithOpName("switch_a"), value, cond_a); + ops::Switch switch_b(root.WithOpName("switch_b"), value, cond_b); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kTFRTNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF1ReturnsTrue) { + tensorflow::set_tf2_execution(false); + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionFalseV1CompatBridgeReturnTrue) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(false); + + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionTrueV1CompatBridgeReturnFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(true); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionFalseTfrtNominalBridgeReturnFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(false); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kTFRTNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionTrueTfrtNominalBridgeReturnFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + auto depth = tensorflow::ops::Placeholder(root.WithOpName("depth"), DT_INT32); + auto on = tensorflow::ops::Placeholder(root.WithOpName("on"), DT_UINT8); + auto off = tensorflow::ops::Placeholder(root.WithOpName("off"), DT_UINT8); + (void)tensorflow::ops::OneHot(root.WithOpName("output"), input, depth, on, + off); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(true); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kTFRTNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionFalseNominalBridgeReturnsFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(false); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(TFVersionTest, TF2ExecutionTrueNominalBridgeReturnsFalse) { + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = tensorflow::ops::Placeholder(root.WithOpName("input"), DT_UINT8); + + Graph graph(OpRegistry::Global()); + TF_ASSERT_OK(root.ToGraph(&graph)); + tensorflow::set_tf2_execution(true); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(UnsupportedOpTest, + InfeedDequeueTupleWithTPUReplicatedCoreAttrNotSupported) { + tensorflow::set_tf2_execution(true); + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + // Build InfeedDequeueTuple node with TPU_REPLICATED_CORE Attr + auto builder = NodeDefBuilder("InfeedDequeueTuple", "InfeedDequeueTuple"); + builder.Attr("dtypes", DT_FLOAT); + builder.Attr("shapes", 1); + NodeDef node_def; + CHECK_OK(builder.Finalize(&node_def)); + absl::Status s; + Node* node_InfeedDequeueTuple = (*root.graph()).AddNode(node_def, &s); + node_InfeedDequeueTuple->set_requested_device( + "/device:TPU_REPLICATED_CORE:0"); + CHECK_OK(s); + ASSERT_NE(node_InfeedDequeueTuple, nullptr); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/true)); +} + +TEST(ManualControlDependencyTest, + TPUReplicatedCoreWithManualControlDependencyReturnsFalse) { + tensorflow::set_tf2_execution(true); + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + auto metadata = tensorflow::ops::TPUReplicateMetadata(root, 2); + metadata.operation.node()->AddAttr("_has_manual_control_dependencies", true); + + Graph graph(OpRegistry::Global()); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/true)); +} + +TEST(InferenceTest, + ContainsInferenceNodeTPUReplicatedCoreDirectSessionReturnsFalse) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, + ContainsInferenceNodeTPUReplicatedCoreEagerRuntimeReturnsTrue) { + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kNominal, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, TF2ExecutionFalseV1CompatBridgeReturnFalse) { + tensorflow::set_tf2_execution(false); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +TEST(InferenceTest, V1CompatBridgeVariableRefReturnTrue) { + tensorflow::set_tf2_execution(false); + FunctionDefLibrary flib; + *flib.add_function() = XTimesTwoFloat(); + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + auto input = + tensorflow::ops::Placeholder(root.WithOpName("node_0"), DT_FLOAT); + auto node = FromNodeDef("Identity", "Identity", 1, DT_FLOAT, *root.graph()); + ASSERT_NE(node, nullptr); + node->set_requested_device("/device:TPU_REPLICATED_CORE:0"); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + Output cond_a = ops::Placeholder(root.WithOpName("cond_a"), DT_BOOL); + Output cond_b = ops::Placeholder(root.WithOpName("cond_b"), DT_BOOL); + + tensorflow::set_tf2_execution(true); + const std::vector shape_array{2, 2}; + auto shape = TensorShape(); + TF_ASSERT_OK(TensorShapeUtils::MakeShape(shape_array, &shape)); + Output value = Output( + FromNodeDef("value", "OneRefOutput", 0, DT_FLOAT_REF, *root.graph())); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(GraphHasUnsupportedFeaturesInMlirBridge( + graph, /*function_library=*/nullptr, config, + /*bridge_version=*/tensorflow::TF2XLABridgeVersion::kV1Compat, + /*single_core_inference_mode=*/false)); +} + +} // namespace + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/hlo_post_processing/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/hlo_post_processing/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..e8bd2723ee163b462bbe36f3d1bb88accb2db73e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/hlo_post_processing/BUILD @@ -0,0 +1,7 @@ +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tf2xla/internal:__subpackages__", + ], + licenses = ["notice"], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..708e799f657dfb64754c41b382fd5c77517f7fc1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/BUILD @@ -0,0 +1,62 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//visibility:private", + ], +) + +gentbl_cc_library( + name = "inference_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"inference_passes.h.inc": [ + "-gen-pass-decls", + "-name=TF2XLA", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "inference_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "inference_metrics_pass", + srcs = ["inference_metrics_pass.cc"], + hdrs = ["inference_passes.h"], + visibility = ["//visibility:public"], + deps = [ + ":inference_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:lib", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +tf_cc_test( + name = "inference_metrics_pass_test", + srcs = ["inference_metrics_pass_test.cc"], + deps = [ + "inference_metrics_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:test_utils", + "//tensorflow/core/lib/monitoring:cell_reader", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:statusor", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..46088fad91b1db439a2ec5f9914c267ec337ea8e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass.cc @@ -0,0 +1,73 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/lib/monitoring/counter.h" + +namespace mlir { +namespace tf2xla { +namespace internal { + +auto* has_tpu_partitioned_call_streamz = + tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/internal/inference/tpu_partitioned_call", + "Whether the model has TPUPartitionedCallOp.", + "has_tpu_partitioned_call"); + +namespace { + +#define GEN_PASS_DEF_INFERENCEMETRICSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h.inc" + +class InferenceMetricsPass + : public impl::InferenceMetricsPassBase { + public: + void runOnOperation() override; +}; + +void InferenceMetricsPass::runOnOperation() { + bool has_tpu_partitioned_call = false; + ModuleOp module = getOperation(); + + for (auto func_op : module.getOps()) { + func_op->walk( + [&](TF::TPUPartitionedCallOp op) { has_tpu_partitioned_call = true; }); + + if (has_tpu_partitioned_call) break; + } + + std::string has_tpu_partitioned_call_str = + has_tpu_partitioned_call ? "true" : "false"; + has_tpu_partitioned_call_streamz->GetCell(has_tpu_partitioned_call_str) + ->IncrementBy(1); +} + +} // namespace + +std::unique_ptr> +CreateInferenceMetricsPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6f796e2fb28ad2e0b0487fe94fa51671bd01cfa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_metrics_pass_test.cc @@ -0,0 +1,105 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" + +namespace mlir { +namespace tf2xla { +namespace internal { +namespace { + +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::mlir::hlo::test::GetMlirModuleFromString; +using ::tensorflow::monitoring::testing::CellReader; + +static constexpr char kHasTpuPartitionedCallStreamzName[] = + "/tensorflow/core/tf2xla/internal/inference/tpu_partitioned_call"; + +class InferenceMetricsPassTest : public ::testing::Test { + protected: + void CreateModule(const char* module_string) { + TF_ASSERT_OK_AND_ASSIGN(module_, + GetMlirModuleFromString(module_string, &context_)); + + pm_ = std::make_unique(&context_); + pm_->addPass(CreateInferenceMetricsPass()); + } + mlir::LogicalResult Run() { return pm_->run(module_.get()); } + + private: + MLIRContext context_; + OwningOpRef module_; + std::unique_ptr pm_; +}; + +TEST_F(InferenceMetricsPassTest, RecordsTrueForTPUPartitionedCallOp) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @tpu_partitioned_call_func(%arg0: tensor) -> (tensor) { + func.return %arg0 : tensor + } + + func.func @main(%arg0: tensor<20xi32>, %arg1: tensor) -> tensor<*xi32> { + %2 = "tf.TPUPartitionedCall"(%arg0, %arg1) {f = @tpu_partitioned_call_func} : (tensor<20xi32>, tensor) -> tensor<*xi32> + func.return %2 : tensor<*xi32> + } + })"; + + CellReader error(kHasTpuPartitionedCallStreamzName); + CreateModule(kMlirModuleStr); + + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + + EXPECT_EQ(error.Delta("true"), 1); + EXPECT_EQ(error.Delta("false"), 0); +} + +TEST_F(InferenceMetricsPassTest, RecordsFalseForNonTPUPartitionedCallOp) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.BadValue"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + CellReader error(kHasTpuPartitionedCallStreamzName); + CreateModule(kMlirModuleStr); + + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + EXPECT_EQ(error.Delta("false"), 1); + EXPECT_EQ(error.Delta("true"), 0); +} + +} // namespace +} // namespace internal +} // namespace tf2xla +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..f08f7e8bfc640837120b93ac3fd65bcb14446df1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h @@ -0,0 +1,39 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_INFERENCE_INFERENCE_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_INFERENCE_INFERENCE_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace mlir { +namespace tf2xla { +namespace internal { + +std::unique_ptr> CreateInferenceMetricsPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_INFERENCEMETRICSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.h.inc" + +} // namespace internal +} // namespace tf2xla +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_INFERENCE_INFERENCE_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..4185de9353945fc35621e843bfa006048af230d1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/inference/inference_passes.td @@ -0,0 +1,25 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +// Internal TF2XLA Bridge Passes used during interence only. +def InferenceMetricsPass : Pass<"tf2xla-inference-metrics-pass", "ModuleOp"> { + let summary = "A pass to go over a Module and collect various metrics"; + let description = [{ + Collects metrics about a Module during inference, such as which ops are used. + }]; + let constructor = "internal::CreateInferenceMetricsPass()"; +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.cc new file mode 100644 index 0000000000000000000000000000000000000000..bbc7dfa313e81fd21b6ece93ea41e955130faf0e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.cc @@ -0,0 +1,98 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "stablehlo/dialect/Register.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/compilation_timer.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/profile_utils/cpu_utils.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tensorflow/core/tpu/tpu_compile.h" +#include "tsl/platform/error_logging.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Name of component for error logging. This name is fixed and required to +// enable logging. +constexpr char kBridgeComponent[] = "TFXLABridge"; + +using tpu::ShardingAndIndex; + +absl::Status CompileFromMlirToXlaHlo( + bool lower_to_xla_hlo, mlir::ModuleOp mlir_module_op, + const tpu::TPUCompileMetadataProto& metadata, llvm::StringRef device_type, + const XlaShapeLayoutHelpers::ShapeDeterminationFns& shape_determination_fns, + bool use_tuple_args, XlaCompiler::CompilationResult* compilation_result, + std::vector>& custom_legalization_passes, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes) { + LOG_FIRST_N(INFO, 1) + << "Compiling MLIR computation to XLA HLO using MLIR tf2xla bridge in " + "the op by op fallback mode. This is Phase 2 of the TF2XLA Bridge. " + "Old (non-MLIR) bridge may be used in case of unsupported feature " + "or compilation failure from the MLIR bridge (full fallback mode)."; + + llvm::SmallVector tensor_or_resource_shapes; + tensor_or_resource_shapes.reserve(arg_shapes.size()); + for (const auto& arg_shape : arg_shapes) + tensor_or_resource_shapes.push_back({arg_shape}); + + TF_RETURN_IF_ERROR(CompileMlirToXlaHlo( + mlir_module_op, tensor_or_resource_shapes, device_type, use_tuple_args, + /*enable_op_fallback=*/true, /*use_return_tuple=*/true, + /*use_resource_updates_for_aliases=*/false, shape_determination_fns, + compilation_result, custom_legalization_passes, metadata.module_name(), + lower_to_xla_hlo)); + + // Compute how arguments are shared across different cores. + auto sharding_result = + tpu::GetShardingInfo(metadata, arg_shapes, shape_determination_fns, + arg_core_mapping, per_core_arg_shapes); + if (!sharding_result.ok()) { + return sharding_result; + } + return absl::OkStatus(); +} + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h new file mode 100644 index 0000000000000000000000000000000000000000..260c2c9772ed8196a366ee96c1eda734c9469f60 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h @@ -0,0 +1,51 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_MLIR_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_MLIR_H_ + +#include +#include + +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Runs all the MLIR Bridge passes on the given MLIR module. +// If compile_to_xla_hlo is true then those passes include all the Legalization +// to XLA HLO which is returned in the compilation_result. +absl::Status CompileFromMlirToXlaHlo( + bool lower_to_xla_hlo, mlir::ModuleOp mlir_module_op, + const tpu::TPUCompileMetadataProto& metadata, llvm::StringRef device_type, + const XlaShapeLayoutHelpers::ShapeDeterminationFns& shape_determination_fns, + bool use_tuple_args, XlaCompiler::CompilationResult* compilation_result, + std::vector>& custom_legalization_passes, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes); + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_MLIR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..52f82a94354b99c073ca0b29b6e05e4735d9d6dc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir_test.cc @@ -0,0 +1,96 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h" + +#include +#include + +#include +#include +#include "absl/status/statusor.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "stablehlo/dialect/Register.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +using tpu::ShardingAndIndex; +using tpu::TPUCompileMetadataProto; + +static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + +absl::StatusOr CompileMlirModule(bool compile_to_xla_hlo, + const char* module_str) { + XlaCompilationResult compilation_result; + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::mhlo::registerAllMhloDialects(registry); + mlir::stablehlo::registerAllDialects(registry); + mlir::MLIRContext context(registry); + mlir::OwningOpRef mlir_module; + TF_RETURN_IF_ERROR( + tensorflow::DeserializeMlirModule(module_str, &context, &mlir_module)); + + std::vector arg_shapes; + TPUCompileMetadataProto metadata_proto; + bool use_tuple_args = true; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + std::vector> custom_legalization_passes; + + TF_RETURN_IF_ERROR(CompileFromMlirToXlaHlo( + compile_to_xla_hlo, mlir_module.get(), metadata_proto, + /*device_type=*/"XLA_TPU_JIT", + /*shape_determination_fns=*/{}, use_tuple_args, &compilation_result, + custom_legalization_passes, arg_shapes, &arg_core_mapping, + &per_core_arg_shapes)); + return compilation_result; +} + +TEST(CompileFromMlir, ReturnsLoweredModule) { + auto result = CompileMlirModule(/*compile_to_xla_hlo=*/true, kMlirModuleStr); + + ASSERT_THAT(result, IsOkOrFiltered()); + EXPECT_THAT(result, ComputationProtoContains("opcode: \"constant\"")); +} + +} // namespace + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.cc new file mode 100644 index 0000000000000000000000000000000000000000..2f5004668570c8d4ba34a202950ba5a1918c742d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.cc @@ -0,0 +1,151 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Interfaces/DataLayoutInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "stablehlo/dialect/Register.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_tf_graph.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_mlir.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/compile_only_client.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/metrics.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +using metrics::IncrementTfMlirBridgeSecondPhaseCounter; +using metrics::MlirBridgeSecondPhaseMetric; +using tpu::MlirToHloArgs; + +absl::Status CheckAndIncrementCounter(absl::Status status, + MlirBridgeSecondPhaseMetric metric) { + if (!status.ok()) { + IncrementTfMlirBridgeSecondPhaseCounter(metric); + return status; + } + return absl::OkStatus(); +} + +absl::StatusOr LegalizeTfToHlo( + mlir::ModuleOp mlir_module_op, const tpu::TPUCompileMetadataProto& metadata, + bool use_tuple_args, llvm::StringRef device_type, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + std::vector>& custom_legalization_passes, + xla::CompileOnlyClient* client, XlaCompilationResult* compilation_result) { + if (!mlir::SetTPUInfeedLayout(mlir_module_op)) + return absl::AbortedError("Failed to set layouts attribute"); + + absl::Status mlir_compilation = internal::CompileFromMlirToXlaHlo( + /*lower_to_xla_hlo=*/false, mlir_module_op, metadata, device_type, + shape_determination_fns, use_tuple_args, compilation_result, + custom_legalization_passes, arg_shapes, arg_core_mapping, + per_core_arg_shapes); + + TF_RETURN_IF_ERROR(CheckAndIncrementCounter( + mlir_compilation, MlirBridgeSecondPhaseMetric::kMlirCombinedMlirFailure)); + + IncrementTfMlirBridgeSecondPhaseCounter( + MlirBridgeSecondPhaseMetric::kMlirCombinedMlirSuccess); + + MlirToHloArgs mlir_to_hlo_args; + mlir_to_hlo_args.mlir_module_op = mlir_module_op; + absl::Status old_bridge_status = v1::CompileTensorflowGraphToHlo( + mlir_to_hlo_args, metadata, use_tuple_args, shape_determination_fns, + arg_shapes, tsl::DeviceType(device_type.str()), arg_core_mapping, + per_core_arg_shapes, client, compilation_result); + + TF_RETURN_IF_ERROR(CheckAndIncrementCounter( + old_bridge_status, MlirBridgeSecondPhaseMetric::kMlirCombinedOldFailure)); + + IncrementTfMlirBridgeSecondPhaseCounter( + MlirBridgeSecondPhaseMetric::kMlirCombinedOldSuccess); + + return *compilation_result; +} +} // namespace + +absl::StatusOr LegalizeTfToHlo( + const tpu::MlirToHloArgs& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + llvm::StringRef device_type, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + std::vector>& custom_legalization_passes, + xla::CompileOnlyClient* client, XlaCompilationResult* compilation_result) { + LOG_FIRST_N(INFO, 1) << "Compiling MLIR computation to XLA HLO using the " + "Combined MLIR Tf2Xla Bridge."; + + if (computation.mlir_module_op.has_value()) { + return LegalizeTfToHlo(computation.mlir_module_op.value(), metadata, + use_tuple_args, device_type, shape_determination_fns, + arg_shapes, arg_core_mapping, per_core_arg_shapes, + custom_legalization_passes, client, + compilation_result); + } + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::mhlo::registerAllMhloDialects(registry); + mlir::stablehlo::registerAllDialects(registry); + mlir::MLIRContext context(registry); + mlir::OwningOpRef mlir_module; + absl::Status deserialization_status = tensorflow::DeserializeMlirModule( + computation.mlir_module, &context, &mlir_module); + + TF_RETURN_IF_ERROR(CheckAndIncrementCounter( + deserialization_status, + MlirBridgeSecondPhaseMetric::kMlirCombinedMlirFailure)); + + return LegalizeTfToHlo(mlir_module.get(), metadata, use_tuple_args, + device_type, shape_determination_fns, arg_shapes, + arg_core_mapping, per_core_arg_shapes, + custom_legalization_passes, client, + compilation_result); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h new file mode 100644 index 0000000000000000000000000000000000000000..da9adea52988872007943c3cdec6f2f289f2a7c5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h @@ -0,0 +1,49 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_TO_HLO_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_TO_HLO_H_ + +#include "absl/status/statusor.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/compile_only_client.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Legalize the given MLIR module to XLA HLO using a combination of the MLIR +// Bridge and XlaBuilder +absl::StatusOr LegalizeTfToHlo( + const tpu::MlirToHloArgs& computation, + const tpu::TPUCompileMetadataProto& metadata, bool use_tuple_args, + llvm::StringRef device_type, + XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns, + const std::vector& arg_shapes, + std::vector* arg_core_mapping, + std::vector>* per_core_arg_shapes, + std::vector>& custom_legalization_passes, + xla::CompileOnlyClient* client, XlaCompilationResult* compilation_result); + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LEGALIZE_TF_TO_HLO_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..8fa7bf938fcd6956a3d33c8364e58493a85598a3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo_test.cc @@ -0,0 +1,186 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/legalize_tf_to_hlo.h" + +#include +#include +#include + +#include +#include +#include "absl/status/statusor.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "stablehlo/dialect/Register.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/client_library.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +using ::tensorflow::monitoring::testing::CellReader; +using tpu::MlirToHloArgs; +using tpu::ShardingAndIndex; +using tpu::TPUCompileMetadataProto; + +static constexpr char kMlirLegalizeCount[] = + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_count"; +static constexpr char kMlirLegalizeErrors[] = + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_pass_count"; +static constexpr char kBridgeStatusCounter[] = + "/tensorflow/core/tf2xla/api/v2/phase2_compilation_status"; +constexpr char kMlirCombinedMlirSuccess[] = "kMlirCombinedMlirSuccess"; +constexpr char kMlirCombinedOldSuccess[] = "kMlirCombinedOldSuccess"; +constexpr char kMlirCombinedOldFailure[] = "kMlirCombinedOldFailure"; + +static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0 : tensor<1xf32>) -> tensor<1xf32> { + %0 = "tf.Acos"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> + func.return %0 : tensor<1xf32> + } +})"; + +static constexpr char kBadMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.DoesntExist"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + +absl::StatusOr CompileMlirModule( + const char* module_str, bool compile_serialized = true) { + MlirToHloArgs mlir_to_hlo_args; + mlir_to_hlo_args.rollout_state = + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_UNSPECIFIED; + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::mhlo::registerAllMhloDialects(registry); + mlir::stablehlo::registerAllDialects(registry); + mlir::MLIRContext context(registry); + mlir::OwningOpRef mlir_module; + + if (compile_serialized) { + mlir_to_hlo_args.mlir_module = module_str; + } else { + TF_RETURN_IF_ERROR( + tensorflow::DeserializeMlirModule(module_str, &context, &mlir_module)); + mlir_to_hlo_args.mlir_module_op = mlir_module.get(); + } + + se::Platform* platform = + se::PlatformManager::PlatformWithName("Host").value(); + auto client = + xla::ClientLibrary::GetOrCreateCompileOnlyClient(platform).value(); + + std::vector arg_shapes = {{1}}; + TPUCompileMetadataProto metadata_proto; + auto arg = metadata_proto.add_args(); + arg->set_dtype(DataType::DT_FLOAT); + arg->set_kind(TPUCompileMetadataProto::Arg::PARAMETER); + metadata_proto.add_retvals(); + bool use_tuple_args = true; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + std::vector> custom_legalization_passes; + auto compilation_result = std::make_unique(); + + return LegalizeTfToHlo(mlir_to_hlo_args, metadata_proto, use_tuple_args, + /*device_type=*/"XLA_TPU_JIT", + /*shape_determination_fns=*/{}, arg_shapes, + &arg_core_mapping, &per_core_arg_shapes, + custom_legalization_passes, client, + compilation_result.get()); +} + +using LegalizeWithCombinedBridge = ::testing::TestWithParam; + +TEST_P(LegalizeWithCombinedBridge, DoesNotUseMlirLowering) { + const bool compile_serialized = GetParam(); + + CellReader mlir_bridge_legalize_count(kMlirLegalizeCount); + CellReader counts(kBridgeStatusCounter); + + auto result = CompileMlirModule(kMlirModuleStr, compile_serialized); + + ASSERT_THAT(result, IsOkOrFiltered()); + EXPECT_EQ(mlir_bridge_legalize_count.Delta("tf.Acos"), 0); + EXPECT_THAT(result, + IncrementedOrFiltered(counts.Delta(kMlirCombinedMlirSuccess), 1)); + EXPECT_THAT(result, + IncrementedOrFiltered(counts.Delta(kMlirCombinedOldSuccess), 1)); +} + +TEST_P(LegalizeWithCombinedBridge, + CorrectlyCountsMlirBridgePassingAndGraphBridgeFailing) { + const bool compile_serialized = GetParam(); + + CellReader legalize_failure_count(kMlirLegalizeErrors); + CellReader counts(kBridgeStatusCounter); + + auto result = CompileMlirModule(kBadMlirModuleStr, compile_serialized); + + ASSERT_FALSE(result.ok()); + // Never failed to legalize because it was never attempted + EXPECT_EQ(legalize_failure_count.Read("tf.DoesntExist", "Unknown"), 0); + EXPECT_THAT(result, + IncrementedOrFiltered(counts.Delta(kMlirCombinedMlirSuccess), 1)); + EXPECT_THAT(result, + IncrementedOrFiltered(counts.Delta(kMlirCombinedOldFailure), 1)); +} + +TEST_P(LegalizeWithCombinedBridge, RecordsDynamicOps) { + const bool compile_serialized = GetParam(); + + static constexpr char kDynamismFunctionCounterStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/dynamism_function_counter"; + constexpr char kNotDynamicFunctionName[] = "kNotDynamicFunction"; + CellReader dynamic_function_op_count( + kDynamismFunctionCounterStreamzName); + + auto result = CompileMlirModule(kMlirModuleStr, compile_serialized); + + ASSERT_TRUE(result.ok()); + EXPECT_EQ(dynamic_function_op_count.Delta(kNotDynamicFunctionName), 1); +} + +INSTANTIATE_TEST_SUITE_P(LegalizeWithCombinedBridge, LegalizeWithCombinedBridge, + ::testing::Bool()); + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.cc new file mode 100644 index 0000000000000000000000000000000000000000..0e175b9e8ab6fe0eb2f9cf73c302d7f75b361d91 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.cc @@ -0,0 +1,52 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +using mlir::PassManager; + +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(PassManager& pm, const std::string& dump_group_name, + llvm::StringRef module_name) { + // Print the whole module after each pass, which requires disabling + // multi-threading as well. + pm.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique<::tensorflow::DataDumperLoggerConfig>( + [module_name, dump_group_name](const std::string& pass_tag_name, + mlir::Operation* op) { + return DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), dump_group_name, pass_tag_name); + }, + /*pass_prefix=*/"", + /*print_module_scope=*/true)); + pm.enableTiming(); +} + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h new file mode 100644 index 0000000000000000000000000000000000000000..4baee18047ac6843672d2cd4fe959f28abdd1e16 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h @@ -0,0 +1,38 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LOGGING_HOOKS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LOGGING_HOOKS_H_ + +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/Pass/PassManager.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(mlir::PassManager& pm, + const std::string& dump_group_name, + llvm::StringRef module_name = llvm::StringRef()); + +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_LOGGING_HOOKS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..72bfcbc9db14e2550f23d046acc6207b39cfdae6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/logging_hooks_test.cc @@ -0,0 +1,112 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/logging_hooks.h" + +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/file_statistics.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +using mlir::DialectRegistry; +using mlir::LogicalResult; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::OwningOpRef; +using mlir::PassManager; +using mlir::func::FuncOp; + +std::string TestDataPath() { + return tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tf2xla/internal/testdata/"); +} + +class LoggingHooksTest : public ::testing::Test { + public: + LoggingHooksTest() { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + env_ = Env::Default(); + + test_group_name_ = "TestGroup"; + test_dir_ = testing::TmpDir(); + setenv(/*name=*/"TF_DUMP_GRAPH_PREFIX", /*value=*/test_dir_.c_str(), + /*overwrite=*/1); + } + + absl::Status CreateMlirModule(std::string mlir_module_filename) { + std::string mlir_module_path = TestDataPath() + mlir_module_filename; + mlir_module_ = + mlir::parseSourceFile(mlir_module_path, &context_); + if (!mlir_module_) { + return absl::Status( + absl::StatusCode::kNotFound, + absl::StrCat("Could not find MLIR module at ", mlir_module_path)); + } + return absl::OkStatus(); + } + + DialectRegistry registry_; + MLIRContext context_; + OwningOpRef mlir_module_; + Env* env_; + std::string test_dir_; + std::string test_group_name_; +}; + +TEST_F(LoggingHooksTest, DumpsPassData) { + std::vector files; + TF_ASSERT_OK(env_->GetChildren(test_dir_, &files)); + EXPECT_THAT(files, ::testing::IsEmpty()); + + TF_ASSERT_OK(CreateMlirModule("dead_const.mlir")); + PassManager pass_manager(&context_); + pass_manager.addNestedPass(mlir::createCanonicalizerPass()); + + EnablePassIRPrinting(pass_manager, test_group_name_); + + LogicalResult pass_status = pass_manager.run(mlir_module_.get()); + EXPECT_TRUE(pass_status.succeeded()); + + TF_ASSERT_OK(env_->GetChildren(test_dir_, &files)); + EXPECT_THAT(files, ::testing::SizeIs(2)); +} + +}; // namespace +}; // namespace internal +}; // namespace tf2xla +}; // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..f86a6c74a71fb707ee00db5b8b09d539c91f84f9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.cc @@ -0,0 +1,246 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.h" + +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/common_runtime/function_body.h" +#include "tensorflow/core/common_runtime/function_def_utils.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { + +using ::mlir::failure; +using ::mlir::LogicalResult; +using ::mlir::success; + +namespace { +constexpr absl::string_view kPartitionedCall = "TPUPartitionedCall"; + +LogicalResult HasAttr( + const Graph& graph, const FunctionLibraryDefinition* function_library, + const std::function& predicate) { + if (predicate(graph)) { + return success(); + } + + // Check if any reachable functions from the graph has the target attribute. + GraphDef graph_def; + graph.ToGraphDef(&graph_def); + if (!function_library) return failure(); + for (const std::string& func_name : + function_library->ReachableDefinitions(graph_def).ListFunctionNames()) { + const FunctionDef* func_def = function_library->Find(func_name); + std::unique_ptr func_body; + absl::Status status = FunctionDefToBodyHelper( + *func_def, AttrSlice(&func_def->attr()), function_library, &func_body); + // This is not expected to happen in practice + if (!status.ok()) { + LOG(ERROR) << "Failed to parse " << func_name << ": " + << absl::StatusMessageAsCStr(status) << "; this can " + << "happen if the function contains unregistered ops."; + return failure(); + } + if (predicate(*func_body->graph)) { + return success(); + } + } + return failure(); +} + +// Check if the `graph` has parameter server jobs and resource variable +// arguments that are on parameter servers +bool HasPsWithResourceVariable(const Graph& graph) { + // Check parameter serverjobs and resource variable arguments that are + // on parameter servers. + const std::string jobType = "ps"; + const std::string nodeType = "_Arg"; + const std::string attrKey = "T"; + for (const Node* node : graph.nodes()) { + if (node->type_string() == nodeType) { + auto device_name = node->assigned_device_name(); + DeviceNameUtils::ParsedName device; + if (DeviceNameUtils::ParseFullName(device_name, &device) && + device.has_job && device.job == jobType) { + for (const auto& attr : node->attrs()) { + auto attr_key = attr.first; + auto attr_value = attr.second; + if (attr_key == attrKey && + attr_value.value_case() == AttrValue::kType && + attr_value.type() == DT_RESOURCE) { + return true; + break; + } + } + } + } + } + return false; +} + +bool IsNonReplicatedGraph(const Graph& graph, + const FunctionLibraryDefinition* function_library) { + auto predicate = [](const Graph& graph) { + const std::string kStatefulPartitionedCallOp = "StatefulPartitionedCall"; + for (const Node* node : graph.nodes()) { + auto node_op = node->type_string(); + if (node_op == kStatefulPartitionedCallOp) { + // Functions called by StatefulfulPartitionedCall ops with + // _XlaMustCompile=true are compiled by XLA. + auto attr = node->attrs().FindByString(std::string(kMustCompileAttr)); + if (attr != nullptr && attr->b() == true) { + return true; + } + } + } + return false; + }; + return HasAttr(graph, function_library, predicate).succeeded(); +} + +bool IsReplicatedGraph(const Graph& graph, + const FunctionLibraryDefinition* function_library) { + auto predicate = [](const Graph& graph) { + for (const Node* node : graph.nodes()) { + // _tpu_replicate is used in replicated TPU graphs. It will be converted + // to_replication_info and _xla_compile_device_type in phase 1 pipelines. + if (node->attrs().FindByString(std::string(kTpuReplicateAttr))) { + return true; + } + } + return false; + }; + return HasAttr(graph, function_library, predicate).succeeded(); +} + +bool IsReplicatedGraph(mlir::ModuleOp module) { + auto walk_result = module.walk([&](mlir::Operation* op) { + // TODO(b/223677572): Once the scope for new compilation and replication + // markers is expanded beyond bridge we can remove this check for + // `kTPUReplicateAttr`, we will then always have a `kCompileDeviceTypeAttr` + // in such cases (see above). + // TODO(b/229028654): Remove string conversion once we have C++17. + const llvm::StringRef tpu_replicate_attr_name(kTpuReplicateAttr.data(), + kTpuReplicateAttr.size()); + auto replicate_attr = + op->getAttrOfType(tpu_replicate_attr_name); + if (replicate_attr) return mlir::WalkResult::interrupt(); + return mlir::WalkResult::advance(); + }); + return walk_result.wasInterrupted(); +} + +// Traverses each node in the graph and check if any of them is +// TPUPartitionedCall. If so, return true. Otherwise, return false. +bool DoesGraphContainTPUPartitionedCall(const Graph& graph) { + for (const Node* node : graph.nodes()) { + if (node->type_string() == kPartitionedCall) return true; + } + return false; +} + +// Checks any reachable functions from `graph_def` in `flib_def` +// for inference graphs. +bool DoReachableFuncsContainTPUPartitionedCall( + const GraphDef& graph_def, const FunctionLibraryDefinition& flib_def) { + for (const std::string& func_name : + flib_def.ReachableDefinitions(graph_def).ListFunctionNames()) { + const FunctionDef* func_def = flib_def.Find(func_name); + std::unique_ptr func_body; + if (!FunctionDefToBodyHelper(*func_def, AttrSlice(&func_def->attr()), + &flib_def, &func_body) + .ok()) + return false; + if (DoesGraphContainTPUPartitionedCall(*func_body->graph)) return true; + } + return false; +} + +// Iterate all functions from the flib_def if there are any that belong to +// the inference graph. +bool AreFunctionsFromFlibDefInference( + const FunctionLibraryDefinition& flib_def) { + for (const std::string& func_name : flib_def.ListFunctionNames()) { + const FunctionDef* func_def = flib_def.Find(func_name); + for (const NodeDef& node_def : func_def->node_def()) { + if (node_def.op() == kPartitionedCall) return true; + } + } + return false; +} + +} // namespace + +bool IsSupportedByNonReplicatedBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library) { + return IsNonReplicatedGraph(graph, function_library) && + HasPsWithResourceVariable(graph); +} + +bool IsSupportedByReplicatedBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library) { + return IsReplicatedGraph(graph, function_library); +} + +bool IsSupportedByReplicatedBridge(mlir::ModuleOp module) { + return IsReplicatedGraph(module); +} + +bool HasTPUPartitionedCallOpInModule(mlir::ModuleOp module) { + bool has_tpu_partitioned_call = false; + for (auto func_op : module.getOps()) { + func_op->walk([&](mlir::TF::TPUPartitionedCallOp op) { + has_tpu_partitioned_call = true; + }); + if (has_tpu_partitioned_call) break; + } + return has_tpu_partitioned_call; +} + +bool IsInferenceGraph(const Graph& graph, + const FunctionLibraryDefinition* function_library) { + if (DoesGraphContainTPUPartitionedCall(graph)) return true; + GraphDef graph_def; + graph.ToGraphDef(&graph_def); + if (DoReachableFuncsContainTPUPartitionedCall(graph_def, graph.flib_def())) + return true; + if (AreFunctionsFromFlibDefInference(graph.flib_def())) return true; + if (function_library == nullptr) return false; + if (DoReachableFuncsContainTPUPartitionedCall(graph_def, *function_library)) + return true; + if (AreFunctionsFromFlibDefInference(*function_library)) return true; + return false; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.h new file mode 100644 index 0000000000000000000000000000000000000000..0f600b9e23de679b6e9cdac813a910c4013d19e3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.h @@ -0,0 +1,54 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_BRIDGE_PASS_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_BRIDGE_PASS_UTIL_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/framework/function.h" + +namespace tensorflow { + +// Checks if a graph or reachable functions in the library have any +// StatefulPartitionedOps with _XlaMustCompile=true. The function library will +// be skipped if nullptr is provided. +bool IsSupportedByNonReplicatedBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library); + +// Checks if a graph or reachable functions in the library have any ops with +// _tpu_replicate or _xla_compile_device_type=TPU. The function library will be +// skipped if nullptr is provided. + +bool IsSupportedByReplicatedBridge( + const Graph& graph, const FunctionLibraryDefinition* function_library); + +// Check if an MLIR module has any ops with _tpu_replicate or +// _xla_compile_device_type=TPU. +bool IsSupportedByReplicatedBridge(mlir::ModuleOp module); + +// Check if an MLIR module contains TPUPartitionedCall op. If so, we define +// such graph as an inference graph. Otherwise, it is non inference graph. +bool HasTPUPartitionedCallOpInModule(mlir::ModuleOp module); + +// Check if a graph contains TPUPartitionedCall op, including its reachable +// functions. The function library is used to store the functions that are +// defined in a TensorFlow program +bool IsInferenceGraph(const Graph& graph, + const FunctionLibraryDefinition* function_library); +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_BRIDGE_PASS_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..cf4ff52bdd1a2fb7a4b3f379bbba0a3f29dbe6cf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util_test.cc @@ -0,0 +1,316 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/mlir_bridge_pass_util.h" + +#include +#include +#include + +#include +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/cc/framework/ops.h" +#include "tensorflow/cc/framework/scope.h" +#include "tensorflow/cc/ops/array_ops.h" +#include "tensorflow/cc/ops/function_ops.h" +#include "tensorflow/cc/ops/tpu_functional_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/function_testlib.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/platform/enable_tf2_utils.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace tensorflow { + +namespace { + +// Produce a valid graph with a resource-type input. +FunctionDef PassThroughResource() { + return FunctionDefHelper::Define( + /*function_name=*/"PassThroughResource", + /*arg_def=*/{"in: resource"}, + /*ret_def=*/{"out: resource"}, + /*attr_def=*/{}, + /*node_def=*/ + {{{"out"}, "Identity", {"in"}, {{"T", DataType::DT_RESOURCE}}}}); +} + +TEST(IsSupportedByNonReplicatedBridge, NonReplicatedGraph) { + const FunctionDef& fd = PassThroughResource(); + FunctionDefLibrary flib; + *flib.add_function() = fd; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + tensorflow::set_tf2_execution(true); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output a = ops::_Arg(root.WithOpName("A"), DT_RESOURCE, 0); + std::vector inputs({NodeBuilder::NodeOut(a.node())}); + + Node* call; + NameAttrList f_name_attr; + f_name_attr.set_name(fd.signature().name()); + TF_ASSERT_OK( + NodeBuilder("B", "StatefulPartitionedCall", &root.graph()->flib_def()) + .Input(inputs) + .Attr("Tin", {DT_RESOURCE}) + .Attr("Tout", {DT_RESOURCE}) + .Attr("f", f_name_attr) + .Finalize(root.graph(), &call)); + call->AddAttr(std::string(kMustCompileAttr), true); + + TF_ASSERT_OK(root.ToGraph(&graph)); + + // Required for passing the PS server parameter check. + for (Node* node : graph.nodes()) { + node->set_assigned_device_name("/job:ps/replica:0/task:0/device:GPU:0"); + } + + EXPECT_TRUE( + IsSupportedByNonReplicatedBridge(graph, /*function_library=*/nullptr)); +} + +TEST(IsSupportedByReplicatedBridge, ReplicatedGraph) { + const FunctionDef& fd = test::function::XTimesTwo(); + FunctionDefLibrary flib; + *flib.add_function() = fd; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kEagerRuntime); + tensorflow::set_tf2_execution(true); + + ConfigProto config = ConfigProto(); + Scope root = Scope::NewRootScope().ExitOnError(); + + Output a = ops::_Arg(root.WithOpName("A"), DT_FLOAT, 0); + std::vector inputs({NodeBuilder::NodeOut(a.node())}); + + Node* call; + NameAttrList f_name_attr; + f_name_attr.set_name(fd.signature().name()); + TF_ASSERT_OK( + NodeBuilder("B", "StatefulPartitionedCall", &root.graph()->flib_def()) + .Input(inputs) + .Attr("Tin", {DT_FLOAT}) + .Attr("Tout", {DT_FLOAT}) + .Attr("f", f_name_attr) + .Finalize(root.graph(), &call)); + call->AddAttr(std::string(kTpuReplicateAttr), "cluster"); + + TF_ASSERT_OK(root.ToGraph(&graph)); + + EXPECT_TRUE( + IsSupportedByReplicatedBridge(graph, /*function_library=*/nullptr)); +} + +TEST(IsSupportedByReplicatedBridge, ReplicatedModule) { + const char* const code = R"mlir( +func.func @entry_func_1(%arg0: tensor) -> tensor attributes {tf.entry_function = {}} { + %0 = "tf.Identity"(%arg0) {_tpu_replicate = "cluster"} : (tensor) -> (tensor) + func.return %0 : tensor +} +)mlir"; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + EXPECT_TRUE(IsSupportedByReplicatedBridge(*module)); +} + +TEST(HasTPUPartitionedCallOpInModule, HasTPUPartitionedCallModule) { + const char* const code = R"mlir( +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() { + %outputs_0 = "tf.TPUOrdinalSelector"() {device = ""} : () -> tensor + "tf.TPUPartitionedCall"(%outputs_0) {f = @reachable_func} : (tensor) -> () + func.return + } + func.func @reachable_func() { + func.return + } +} +)mlir"; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + EXPECT_TRUE(HasTPUPartitionedCallOpInModule(*module)); +} + +TEST(HasTPUPartitionedCallOpInModule, HasNotTPUPartitionedCallModule) { + const char* const code = R"mlir( +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() { + "tf.StatefulPartitionedCall"() {config = "", config_proto = "", executor_type = "", f = @reachable_func} : () -> () + func.return + } + func.func @reachable_func() { + func.return + } +} +)mlir"; + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module = + mlir::parseSourceString(code, &context); + ASSERT_TRUE(module); + EXPECT_FALSE(HasTPUPartitionedCallOpInModule(*module)); +} + +TEST(IsInferenceGraph, GraphContrainsTPUPartitionedCall) { + FunctionDef fd = FunctionDefHelper::Define( + // Name + "XTimesTwoFloat", + // Args + {"x: float"}, + // Return values + {"y: float"}, + // Attr def + {}, + // Nodes + { + {{"two"}, + "Const", + {}, + {{"value", test::AsScalar(2)}, {"dtype", DT_INT64}}}, + {{"scale"}, + "Cast", + {"two"}, + {{"SrcT", DT_INT64}, {"DstT", DT_FLOAT}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", DT_FLOAT}}}, + }); + + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = fd; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + ops::TPUPartitionedCall f(root.WithOpName("f"), {x}, /*device_ordinal=*/0, + {DT_FLOAT}, f_name_attr); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(IsInferenceGraph(graph, /*function_library=*/nullptr)); +} + +TEST(IsInferenceGraph, GraphDoesNotContrainTPUPartitionedCall) { + FunctionDef fd = FunctionDefHelper::Define( + // Name + "XTimesTwoFloat", + // Args + {"x: float"}, + // Return values + {"y: float"}, + // Attr def + {}, + // Nodes + { + {{"two"}, + "Const", + {}, + {{"value", test::AsScalar(2)}, {"dtype", DT_INT64}}}, + {{"scale"}, + "Cast", + {"two"}, + {{"SrcT", DT_INT64}, {"DstT", DT_FLOAT}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", DT_FLOAT}}}, + }); + + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = fd; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_FALSE(IsInferenceGraph(graph, /*function_library=*/nullptr)); +} + +TEST(IsInferenceGraph, FlibDefIsNotNullptrAndContainsTPUPartitionedCall) { + FunctionDef fd = FunctionDefHelper::Define( + // Name + "XTimesTwoFloat", + // Args + {"x: float"}, + // Return values + {"y: float"}, + // Attr def + {}, + // Nodes + { + {{"two"}, + "Const", + {}, + {{"value", test::AsScalar(2)}, {"dtype", DT_INT64}}}, + {{"scale"}, + "Cast", + {"two"}, + {{"SrcT", DT_INT64}, {"DstT", DT_FLOAT}}}, + {{"y"}, "Mul", {"x", "scale"}, {{"T", DT_FLOAT}}}, + {{"tpu_op"}, "TPUPartitionedCall", {}, {{"Tout", DT_FLOAT}}}, + }); + + tensorflow::set_tf2_execution(true); + FunctionDefLibrary flib; + *flib.add_function() = fd; + FunctionLibraryDefinition flib_def(OpRegistry::Global(), flib); + Graph graph(flib_def); + graph.SetConstructionContext(ConstructionContext::kDirectSession); + + Scope root = Scope::NewRootScope().ExitOnError(); + + Output x = ops::Placeholder(root.WithOpName("x"), DT_FLOAT); + NameAttrList f_name_attr; + f_name_attr.set_name("XTimesTwoFloat"); + + TF_ASSERT_OK(root.ToGraph(&graph)); + EXPECT_TRUE(IsInferenceGraph(graph, /*function_library=*/&flib_def)); +} + +} // namespace + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.cc new file mode 100644 index 0000000000000000000000000000000000000000..4321c47cb3a3a67d62336706a23cb3c54b462377 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.cc @@ -0,0 +1,67 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "tensorflow/core/platform/logging.h" + +namespace mlir { + +class MlirPassInstrumentationRegistry { + public: + static MlirPassInstrumentationRegistry& Instance() { + static MlirPassInstrumentationRegistry* r = + new MlirPassInstrumentationRegistry; + return *r; + } + std::unordered_map()>> + instrumentors_; +}; + +void RegisterPassInstrumentor( + const std::string& name, + std::function()> creator) { + MlirPassInstrumentationRegistry& r = + MlirPassInstrumentationRegistry::Instance(); + auto result = r.instrumentors_.emplace(name, creator); + if (!result.second) { + VLOG(1) << "Duplicate MLIR pass instrumentor registration"; + } +} + +std::vector()>> +GetPassInstrumentors() { + MlirPassInstrumentationRegistry& r = + MlirPassInstrumentationRegistry::Instance(); + std::vector()>> result; + result.reserve(r.instrumentors_.size()); + + std::transform(r.instrumentors_.begin(), r.instrumentors_.end(), + std::back_inserter(result), [](auto v) { return v.second; }); + + return result; +} + +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h new file mode 100644 index 0000000000000000000000000000000000000000..ce78eac6f23ddc3008d791ac2d408ace1b426cac --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h @@ -0,0 +1,36 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_PASS_INSTRUMENTATION_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_PASS_INSTRUMENTATION_H_ + +#include +#include +#include +#include + +#include "mlir/Pass/PassInstrumentation.h" // from @llvm-project + +namespace mlir { + +void RegisterPassInstrumentor( + const std::string& name, + std::function()> creator); +std::vector()>> +GetPassInstrumentors(); + +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_MLIR_PASS_INSTRUMENTATION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..b158b52285ec144aef67216d16772bfde12869a2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation_test.cc @@ -0,0 +1,112 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/mlir_pass_instrumentation.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace mlir { +namespace { +static const char* kTestInstrumentationName = "test-intrumentatron"; +static const char* kTestInstrumentationSearch = "tf.Identity"; + +struct StringStream : public llvm::raw_ostream { + StringStream() { SetUnbuffered(); } + ~StringStream() override = default; + uint64_t current_pos() const override { return 0; } + + void write_impl(const char* ptr, size_t size) override { + ss.write(ptr, size); + } + std::stringstream ss; +}; + +class TestPassInstrumentation : public ::testing::Test { + public: + void SetPassThatChangedIdentity(absl::string_view pass_name) { + pass_that_changed_identity_ = pass_name; + } + absl::string_view GetPassThatChangedIdentity() { + return pass_that_changed_identity_; + } + + private: + std::string pass_that_changed_identity_; + friend class TestInstrumentor; +}; + +class TestInstrumentor : public PassInstrumentation { + public: + explicit TestInstrumentor(TestPassInstrumentation* test) : test_(test) {} + + private: + void runBeforePass(Pass* pass, Operation* op) override { + StringStream stream; + op->print(stream, mlir::OpPrintingFlags().useLocalScope()); + ops_seen_by_pass_[pass] = stream.ss.str(); + } + void runAfterPass(Pass* pass, Operation* op) override { + StringStream stream; + op->print(stream, mlir::OpPrintingFlags().useLocalScope()); + if (!absl::StrContains(stream.ss.str(), kTestInstrumentationSearch) && + absl::StrContains(ops_seen_by_pass_[pass], + kTestInstrumentationSearch)) { + test_->SetPassThatChangedIdentity(pass->getName().str()); + } + } + + private: + TestPassInstrumentation* test_; + std::unordered_map ops_seen_by_pass_; +}; + +TEST_F(TestPassInstrumentation, CreatedCalledAndSetsPassName) { + RegisterPassInstrumentor(kTestInstrumentationName, [&]() { + return std::make_unique(this); + }); + constexpr char legalization[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor>) -> tensor> { + %0 = "tf.Identity"(%arg0) : (tensor>) -> tensor> + func.return %0 : tensor> + } + })"; + SetPassThatChangedIdentity(""); + std::vector<::tensorflow::TensorShape> arg_shapes = {{1}}; + auto compilation_result = tensorflow::XlaCompilationResult(); + + TF_EXPECT_OK(tensorflow::CompileSerializedMlirToXlaHlo( + legalization, arg_shapes, /*device_type=*/"XLA_TPU_JIT", + /*use_tuple_args=*/true, /*enable_op_fallback=*/false, + /*shape_determination_fns=*/{}, &compilation_result) + .status()); + + EXPECT_FALSE(GetPassThatChangedIdentity().empty()); +} + +} // namespace +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.cc new file mode 100644 index 0000000000000000000000000000000000000000..8dcaead2c45e10cbd8086c2e7b3be6f474e20b50 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.cc @@ -0,0 +1,122 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/node_order.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" + +namespace tensorflow { + +void TopologicalOrdering( + const Graph& g, const std::function& emit, + const std::function& get_grouping_key) { + std::unordered_map group_key_string_to_integer; + absl::flat_hash_map node_to_group; + absl::flat_hash_map remaining_incoming_nodes; + absl::flat_hash_map node_to_position; + using Ready = std::vector; + std::vector group_members_that_are_ready; + std::set groups_that_are_ready; + + // Visit all nodes once, for initialization. It doesn't matter whether we use + // BFS or DFS. + int i = 0; + DFS( + g, [](Node*) {}, + [&](Node* n) { + // Find which group this node belongs to. + std::string group_key_string = get_grouping_key(n); + auto entry = group_key_string_to_integer.try_emplace( + group_key_string, group_key_string_to_integer.size()); + int group_key = entry.first->second; + node_to_position[n] = i++; + node_to_group[n] = group_key; + if (entry.second) { + group_members_that_are_ready.push_back({}); + } + + // Count the incoming nodes and store. Also remember nodes ("sources") + // that don't have any inputs. + auto in_nodes = n->in_nodes(); + int num_incoming = std::distance(in_nodes.begin(), in_nodes.end()); + remaining_incoming_nodes[n] = num_incoming; + if (num_incoming == 0) { + // NO_CDC: This array is max(group_key) + 1. + group_members_that_are_ready[group_key].push_back(n); + groups_that_are_ready.emplace(group_key); + } + }, + [](const Node* n1, const Node* n2) { return n1->name() < n2->name(); }); + + assert(group_key_string_to_integer.size() == + group_members_that_are_ready.size()); + + int num_nodes = remaining_incoming_nodes.size(); + + // We emit one node per step, thus we just run this as often as we have nodes. + int current_group = 0; + for (int i = 0; i < num_nodes; i++) { + if (groups_that_are_ready.find(current_group) == + groups_that_are_ready.end()) { + current_group = *groups_that_are_ready.begin(); + } + + // NO_CDC: This array is max(group_key) + 1. + int size = group_members_that_are_ready[current_group].size(); + assert(size); + // NO_CDC: This array is max(group_key) + 1. + Node* node = group_members_that_are_ready[current_group][--size]; + // NO_CDC: This array is max(group_key) + 1. + group_members_that_are_ready[current_group].pop_back(); + if (size == 0) { + groups_that_are_ready.erase(current_group); + } + + // Emit the operation and make its results available. + emit(node); + + auto out_nodes = node->out_nodes(); + std::vector nodes_sorted(out_nodes.begin(), out_nodes.end()); + std::sort(nodes_sorted.begin(), nodes_sorted.end(), [&](Node* a, Node* b) { + return node_to_position[a] < node_to_position[b]; + }); + + for (Node* out : nodes_sorted) { + remaining_incoming_nodes[out]--; + if (remaining_incoming_nodes[out] == 0) { + int group_key = node_to_group[out]; + // NO_CDC: This array is max(group_key) + 1. + if (group_members_that_are_ready[group_key].empty()) { + groups_that_are_ready.emplace(group_key); + } + // NO_CDC: This array is max(group_key) + 1. + group_members_that_are_ready[group_key].push_back(out); + } + } + } +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.h new file mode 100644 index 0000000000000000000000000000000000000000..c54446ba4ac451e18c0f66f9839ded748cfc8ec5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order.h @@ -0,0 +1,51 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_NODE_ORDER_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_NODE_ORDER_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/lib/gtl/array_slice.h" + +namespace tensorflow { + +struct GroupByDevice { + std::string operator()(const Node* node) const { + return node->requested_device(); + } +}; + +// Performs a topological ordering of nodes. +// This has the property that any child node of a parent node p is emitted +// before p. A grouping function is used to break ties if multiple child nodes +// (of possibly different parents) are ready to be emitted at some point, which +// is when we prefer to stay in the current group. Remaining ties are broken by +// node name. +// The "emit" function is used for outputing the result, and is called once +// for each node. +// This algorithm is O(n * k * log k), with k the largest node degree. +void TopologicalOrdering( + const Graph& g, const std::function& emit, + const std::function& get_grouping_key); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_NODE_ORDER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..e1090bbbbcca8582038b23565416826602194b8c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/node_order_test.cc @@ -0,0 +1,294 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/node_order.h" + +#include +#include +#include + +#include "xla/tsl/lib/core/status_test_util.h" +#include "tensorflow/core/common_runtime/graph_def_builder_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/graph_def_builder.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace { + +REGISTER_OP("TestParams").Output("o: float"); +REGISTER_OP("TestInput").Output("a: float").Output("b: float"); +REGISTER_OP("TestMul").Input("a: float").Input("b: float").Output("o: float"); +REGISTER_OP("TestUnary").Input("a: float").Output("o: float"); +REGISTER_OP("TestTwoOutputs").Output("a: float").Output("b: float"); +REGISTER_OP("TestBinary") + .Input("a: float") + .Input("b: float") + .Output("o: float"); + +// Compares that the order of nodes in 'inputs' respects the +// pair orders described in 'ordered_pairs'. +bool ExpectBefore( + const std::vector>& ordered_pairs, + const std::vector& inputs, std::string* error) { + for (const std::pair& pair : ordered_pairs) { + const std::string& before_node = pair.first; + const std::string& after_node = pair.second; + bool seen_before = false; + bool seen_both = false; + for (const Node* node : inputs) { + if (!seen_before && after_node == node->name()) { + *error = std::string("Saw ") + after_node + std::string(" before ") + + before_node; + return false; + } + + if (before_node == node->name()) { + seen_before = true; + } else if (after_node == node->name()) { + seen_both = seen_before; + break; + } + } + if (!seen_both) { + *error = std::string("didn't see either ") + before_node + + std::string(" or ") + after_node; + return false; + } + } + + return true; +} + +TEST(AlgorithmTest, TopologicalOrdering) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + Node* n1 = SourceOp("TestParams", b.opts().WithName("n1")); + Node* n2 = + SourceOp("TestParams", b.opts().WithName("n2").WithControlInput(n1)); + Node* n3 = + SourceOp("TestParams", b.opts().WithName("n3").WithControlInput(n2)); + Node* n4 = BinaryOp("TestMul", n1, {n3, 0}, b.opts().WithName("n4")); + Node* n5 = BinaryOp("TestMul", n1, {n3, 0}, + b.opts().WithName("n5").WithControlInput(n1)); + Node* n6 = BinaryOp("TestMul", n2, {n3, 0}, b.opts().WithName("n6")); + n3->set_requested_device("a"); + n4->set_requested_device("a"); + n5->set_requested_device("b"); + n6->set_requested_device("b"); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order; + + TopologicalOrdering(g, [&](Node* n) { order.push_back(n); }, GroupByDevice()); + + std::vector> desired_order = { + {"n1", "n2"}, // because of control dependency + {"n2", "n3"}, // because of control dependency + {"n3", "n4"}, // because of NodeScorerDevice + {"n1", "n4"}, // data dependency + {"n1", "n5"}, // data dependency + {"n2", "n6"}, // data dependency + {"n3", "n4"}, // data dependency + {"n3", "n5"}, // data dependency + {"n3", "n6"}, // data dependency + }; + std::string error; + EXPECT_TRUE(ExpectBefore(desired_order, order, &error)) << error; +} + +TEST(AlgorithmTest, TopologicalOrderingOnShallowTree) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + Node* n1 = SourceOp("TestParams", b.opts().WithName("n1").WithDevice("a")); + Node* n2 = + SourceOp("TestParams", + b.opts().WithName("n2").WithDevice("b").WithControlInput(n1)); + Node* n3 = + SourceOp("TestParams", + b.opts().WithName("n3").WithDevice("c").WithControlInput(n2)); + Node* n4 = + SourceOp("TestParams", + b.opts().WithName("n4").WithDevice("a").WithControlInput(n1)); + Node* n5 = + SourceOp("TestParams", + b.opts().WithName("n5").WithDevice("b").WithControlInput(n2)); + Node* n6 = + SourceOp("TestParams", + b.opts().WithName("n6").WithDevice("c").WithControlInput(n3)); + Node* n7 = + SourceOp("TestParams", + b.opts().WithName("n7").WithDevice("a").WithControlInput(n4)); + Node* n8 = + SourceOp("TestParams", + b.opts().WithName("n8").WithDevice("b").WithControlInput(n5)); + Node* n9 = + SourceOp("TestParams", + b.opts().WithName("n9").WithDevice("c").WithControlInput(n6)); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order; + + TopologicalOrdering(g, [&](Node* n) { order.push_back(n); }, GroupByDevice()); + + std::vector desired_order = { + g.source_node(), n1, n4, n7, n2, n5, n8, n3, n6, n9, g.sink_node()}; + for (int i = 0; i < desired_order.size(); i++) { + desired_order[i] = g.FindNodeId(desired_order[i]->id()); + } + EXPECT_EQ(order, desired_order); +} + +TEST(AlgorithmTest, TopologicalOrderingGivesTheSameResultIfCalledTwice) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + SourceOp("TestParams", b.opts().WithName("n1")); + SourceOp("TestParams", b.opts().WithName("n2")); + SourceOp("TestParams", b.opts().WithName("n3")); + SourceOp("TestParams", b.opts().WithName("n4")); + SourceOp("TestParams", b.opts().WithName("n5")); + SourceOp("TestParams", b.opts().WithName("n6")); + SourceOp("TestParams", b.opts().WithName("n7")); + SourceOp("TestParams", b.opts().WithName("n8")); + SourceOp("TestParams", b.opts().WithName("n9")); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order1; + std::vector order2; + + TopologicalOrdering( + g, [&](Node* n) { order1.push_back(n); }, + [&](const Node* node) { return std::string("same"); }); + + TopologicalOrdering( + g, [&](Node* n) { order2.push_back(n); }, + [&](const Node* node) { return std::string("same"); }); + + EXPECT_EQ(order1, order2); +} + +TEST(AlgorithmTest, TopologicalOrderingOnChain) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + Node* n1 = SourceOp("TestParams", b.opts().WithName("n1")); + Node* n2 = UnaryOp("TestUnary", n1, b.opts().WithName("n2")); + Node* n3 = UnaryOp("TestUnary", n2, b.opts().WithName("n3")); + Node* n4 = UnaryOp("TestUnary", n3, b.opts().WithName("n4")); + Node* n5 = UnaryOp("TestUnary", n4, b.opts().WithName("n5")); + Node* n6 = UnaryOp("TestUnary", n5, b.opts().WithName("n6")); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order; + TopologicalOrdering(g, [&](Node* n) { order.push_back(n); }, GroupByDevice()); + + std::vector desired_order = {g.source_node(), n1, n2, n3, n4, n5, n6, + g.sink_node()}; + for (int i = 0; i < desired_order.size(); i++) { + desired_order[i] = g.FindNodeId(desired_order[i]->id()); + } + EXPECT_EQ(order, desired_order); +} + +TEST(AlgorithmTest, TopologicalOrderingOnMultipleOutputs) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + Node* n1 = SourceOp("TestTwoOutputs", b.opts().WithName("n1")); + UnaryOp("TestUnary", {n1, 0}, b.opts().WithName("n2")); + UnaryOp("TestUnary", {n1, 1}, b.opts().WithName("n3")); + UnaryOp("TestUnary", {n1, 0}, b.opts().WithName("n4")); + UnaryOp("TestUnary", {n1, 1}, b.opts().WithName("n5")); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order; + TopologicalOrdering(g, [&](Node* n) { order.push_back(n); }, GroupByDevice()); + + std::vector> desired_order = { + {"n1", "n2"}, + {"n1", "n3"}, + {"n1", "n4"}, + {"n1", "n5"}, + }; + std::string error; + EXPECT_TRUE(ExpectBefore(desired_order, order, &error)) << error; +} + +TEST(AlgorithmTest, TopologicalOrderingSameAsReversePostOrder) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + Node* n = SourceOp("TestTwoOutputs", b.opts().WithName("n")); + Node* n0 = UnaryOp("TestUnary", {n, 0}, b.opts().WithName("n2")); + Node* n1 = UnaryOp("TestUnary", {n, 1}, b.opts().WithName("n1")); + UnaryOp("TestUnary", n0, b.opts().WithName("n1a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n8a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n2a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n7a")); + UnaryOp("TestUnary", n1, b.opts().WithName("n1b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n8b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n2b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n7b")); + UnaryOp("TestUnary", n0, b.opts().WithName("n3a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n6a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n4a")); + UnaryOp("TestUnary", n0, b.opts().WithName("n5a")); + UnaryOp("TestUnary", n1, b.opts().WithName("n3b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n6b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n4b")); + UnaryOp("TestUnary", n1, b.opts().WithName("n5b")); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + std::vector order; + TopologicalOrdering(g, [&](Node* n) { order.push_back(n); }, GroupByDevice()); + + std::vector desired_order; + GetReversePostOrder(g, &desired_order, [](const Node* n1, const Node* n2) { + return n1->name() < n2->name(); + }); + + EXPECT_EQ(desired_order, order); +} + +TEST(AlgorithmTest, TopologicalOrderingWithEachDeviceUsedOnce) { + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + using namespace ::tensorflow::ops; // NOLINT + SourceOp("TestParams", b.opts().WithName("n1").WithDevice("a")); + SourceOp("TestParams", b.opts().WithName("n2").WithDevice("b")); + SourceOp("TestParams", b.opts().WithName("n3").WithDevice("c")); + SourceOp("TestParams", b.opts().WithName("n4").WithDevice("d")); + + Graph g(OpRegistry::Global()); + TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g)); + + int count = 0; + TopologicalOrdering(g, [&](Node* n) { count++; }, GroupByDevice()); + EXPECT_EQ(count, 6); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..ac6bf6177a25f947419ca8970901da37f850f1e7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/BUILD @@ -0,0 +1,573 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow:tensorflow.default.bzl", "filegroup", "get_compatible_with_portable") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//learning/brain/tfrt/tpu/compiler/mlir:__pkg__", + "//tensorflow/compiler/mlir:__subpackages__", + "//tensorflow/compiler/mlir/tf2xla/api:__subpackages__", + "//tensorflow/compiler/mlir/tf2xla/internal:__subpackages__", + ], + licenses = ["notice"], +) + +cc_library( + name = "clustering_passes", + hdrs = [ + "clustering_passes.h", + ], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":extract_head_tail_outside_compilation", + ":extract_outside_compilation", + ":hoist_broadcast_read", + ":mark_ops_for_outside_compilation", + ":tpu_cluster_formation", + ":tpu_sharding_identification_pass", + ":tpu_validate_inputs", + ":tpu_validate_session_inputs", + ":verify_clustering_pass", + ":xla_broadcast", + ":xla_cluster_formation", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "verify_clustering_pass", + srcs = [ + "verify_clustering_pass.cc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tf2xla/internal/utils:dialect_detection_utils", + "//tensorflow/core:framework", + "//tensorflow/core/transforms/toposort:Pass", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +gentbl_cc_library( + name = "clustering_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"clustering_passes.h.inc": [ + "-gen-pass-decls", + "-name=TFXLABridgeClustering", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "clustering_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +tf_cc_test( + name = "verify_clustering_pass_test", + srcs = ["verify_clustering_pass_test.cc"], + deps = [ + ":clustering_passes", + "//tensorflow/compiler/mlir/tf2xla/transforms:test_utils", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:statusor", + ], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "@llvm-project//mlir:run_lit.sh", + test_file_exts = [ + "mlir", + ], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-opt", + "@llvm-project//llvm:FileCheck", + ], +) + +cc_library( + name = "tpu_cluster_formation", + srcs = ["tpu_cluster_formation.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/core:framework", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "extract_outside_compilation", + srcs = ["extract_outside_compilation.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:shape_inference_pass", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "extract_head_tail_outside_compilation", + srcs = ["extract_head_tail_outside_compilation.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "mlir_to_graph_passes", + hdrs = [ + "mlir_to_graph_passes.h", + ], + textual_hdrs = [ + "mlir_to_graph_passes.h.inc", + ], + deps = [ + ":verify_input_dialect_to_executor_pass", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + ], +) + +gentbl_cc_library( + name = "mlir_to_graph_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"mlir_to_graph_passes.h.inc": [ + "-gen-pass-decls", + "-name=TFXLABridgeMlirToGraph", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "mlir_to_graph_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "verify_input_dialect_to_executor_pass", + srcs = [ + "verify_input_dialect_to_executor_pass.cc", + ], + deps = [ + ":mlir_to_graph_passes_inc_gen", + "//tensorflow/compiler/mlir/tf2xla/internal/utils:dialect_detection_utils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "xla_cluster_formation", + srcs = ["xla_cluster_formation.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:call_graph_util", + "//tensorflow/compiler/mlir/tensorflow:cluster_util", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core/platform:types", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "mark_ops_for_outside_compilation", + srcs = ["mark_ops_for_outside_compilation.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:lower_tf_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalize_tf", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "tpu_sharding_identification_pass", + srcs = ["tpu_sharding_identification_pass.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_traits", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:lower_tf_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalize_tf", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:sharding_builder", + ], +) + +cc_library( + name = "hoist_broadcast_read", + srcs = ["hoist_broadcast_read.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:lower_tf_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalize_tf", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "xla_broadcast", + srcs = ["xla_broadcast.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow:xla_rewrite_util", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:lower_tf_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow/transforms:verify_no_outside_compilation_markers_pass", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalize_tf", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/ir/types:Dialect", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "lowering_passes", + hdrs = [ + "lowering_passes.h", + ], + textual_hdrs = [ + "lowering_passes.h.inc", + ], + deps = [ + ":input_metrics_lowering_pass", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +gentbl_cc_library( + name = "lowering_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"lowering_passes.h.inc": [ + "-gen-pass-decls", + "-name=TFXLABridgeLowering", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "lowering_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "input_metrics_lowering_pass", + srcs = [ + "input_lowering_metrics_pass.cc", + ], + deps = [ + ":lowering_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tf2xla/transforms:legalization_op_config", + "//tensorflow/core:lib", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +tf_cc_test( + name = "input_metrics_lowering_pass_test", + srcs = ["input_lowering_metrics_pass_test.cc"], + deps = [ + ":lowering_passes", + "//tensorflow/compiler/mlir/tf2xla/transforms:test_utils", + "//tensorflow/core/lib/monitoring:cell_reader", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "tpu_validate_inputs_utils", + srcs = ["tpu_validate_inputs_utils.cc"], + hdrs = ["tpu_validate_inputs_utils.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tpu_validate_session_inputs", + srcs = ["tpu_validate_session_inputs.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + ":tpu_validate_inputs_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "tpu_validate_inputs", + srcs = ["tpu_validate_inputs.cc"], + textual_hdrs = [ + "clustering_passes.h.inc", + ], + deps = [ + ":clustering_passes_inc_gen", + ":tpu_validate_inputs_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:attribute_utils", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:string_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "//tensorflow/core:framework", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:sharding_builder", + ], +) + +tf_cc_test( + name = "tpu_validate_inputs_utils_test", + srcs = ["tpu_validate_inputs_utils_test.cc"], + deps = [ + ":tpu_validate_inputs_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tf2xla/transforms:test_utils", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..0dd85b89c2423bbe845334f108bcdd9392db360e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h @@ -0,0 +1,93 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_CLUSTERING_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_CLUSTERING_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Verifies that all MLIR Ops have the expected attributes. +std::unique_ptr> +CreateVerifyClusteringPass(); + +// Creates a pass that forms clusters from operations of the same +// `_replication_info` attribute. +std::unique_ptr> +CreateTPUClusterFormationPass(bool strict_clusters = false); + +// Creates a pass that extracts outside compilation (Host ops inside device +// cluster) at head/tail of Device cluster to run before/after XLA computation. +std::unique_ptr> +CreateExtractHeadTailOutsideCompilationPass(); + +// Creates a pass that extract outside compilation (Host ops inside cevice +// cluster) ops to a separate parallel_execute region to run on CPU. +std::unique_ptr> +CreateExtractOutsideCompilationPass(); + +// Create a pass that encapsulates StatefulPartitionedCallOp within a cluster. +std::unique_ptr> +CreateXlaClusterFormationPass(); + +// Creates a pass that marks unsupported ops in device cluster for outside +// compilation. +std::unique_ptr> +CreateMarkOpsForOutsideCompilationPass(); + +// Creates a pass that hoists reads out of a replicate that are on a variable +// whose value is broacast to all replicas. +std::unique_ptr> +CreateHoistBroadcastReadPass(); + +// Creates a pass that moves broadcasts from TF host ops to XLA code, encoded as +// XlaAllReduces. This enables use of the device network for broadcasts, which +// is faster. +std::unique_ptr> +CreateXlaBroadcastPass(); + +// Creates a pass that identifies XLASharding ops in launch op for TPU +// computation. +std::unique_ptr> +CreateTPUShardingIdentificationPass(); + +// Creates a pass that validates the inputs to a TPU computation. +std::unique_ptr> +CreateTPUValidateSessionInputsPass(); + +std::unique_ptr> +CreateTPUValidateInputsPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_MARKOPSFOROUTSIDECOMPILATIONPASS +#define GEN_PASS_DECL_TPUCLUSTERFORMATIONPASS +#define GEN_PASS_DECL_TPUEXTRACTHEADTAILOUTSIDECOMPILATIONPASS +#define GEN_PASS_DECL_TPUEXTRACTOUTSIDECOMPILATIONPASS +#define GEN_PASS_DECL_TPUSHARDINGIDENTIFICATIONPASS +#define GEN_PASS_DECL_TPUVALIDATEINPUTSPASS +#define GEN_PASS_DECL_TPUVALIDATESESSIONINPUTSPASS +#define GEN_PASS_DECL_VERIFYCLUSTERINGPASS +#define GEN_PASS_DECL_XLACLUSTERFORMATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_CLUSTERING_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..9c72719b304d5b087bf400f81f65d42ead4499d9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.td @@ -0,0 +1,466 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/Pass/PassBase.td" + +def VerifyClusteringPass : Pass<"verify-clustering-pass", "mlir::func::FuncOp"> { + + let summary = "Verify that the Bridge output is correct and errors if verification fails."; + + let description = [{ + Verifies whether clustering has resulted in the expected invariants. These + include verifying that clusters have been created and have been outside + compiled, the result is device agnostic and in TF functional dialect & + that the device attribute exists. + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateVerifyClusteringPass()"; +} + +def TPUClusterFormationPass : Pass<"tf-tpu-cluster-formation", "ModuleOp"> { + let summary = "Forms clusters from operations assigned to the same TPU computation"; + + let description = [{ + TPU computations from the frontend are composed of a `tf.TPUReplicateMetadata` + op, a subgraph of ops (TensorFlow Dialect) each with a matching + `_replication_info` attribute relative to the associated + `tf.TPUReplicateMetadata` op, and optionally `tf.TPUReplicatedInput` and + `tf.TPUReplicatedOutput` ops feeding in inputs and outputs to and from a + replicated TPU computation. The number of times a TPU computation is + replicated is defined in the `tf.TPUReplicateMetadata` op (`num_replicas` + attribute) and operand and result sizes of `tf.TPUReplicatedInput` and + `tf.TPUReplicatedOutput` respectively must match, excluding packed tensors. + It is also assumed ops of the same TPU computation do not have ops outside + of the TPU computation that are both inputs and outputs to the same TPU + computation. Furthermore, we assume that every node has either none or both + of `_replication_info` and `_xla_compile_device_type` attributes defined. + + This pass takes the TPU computation subgraph, moves them into a + `tf_device.cluster`, and copies over attributes from the associated + `tf.TPUReplicateMetadata` op to the newly created `tf_device.cluster`. If the + computation is replicated (`num_replicas` > 1), the `num_replicas` attribute is + not copied over but instead the `tf_device.cluster` is further wrapped with a + `tf_device.replicate`, and associated `tf.TPUReplicatedInput` and + `tf.TPUReplicatedOutput` ops are replaced as the `tf_device.replicate` operands + and results. Otherwise, the single operands and results of the associated + `tf.TPUReplicatedInput` and `tf.TPUReplicatedOutput` ops are simply forwarded to + the `tf_device.cluster`. + + For example, the following non replicated computation: + + ```mlir + func @tpu_computation(%arg0: tensor) -> tensor { + // Metadata op for cluster `cluster` with 1 replica, 1 core per replica and + // with topology ``. + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", num_relicas = 1, num_cores_per_replica = 1, topology = "", device_assignment = [], padding_map = []} : () -> () + %replicated_input = "tf.TPUReplicatedInput"(%arg0) : (tensor) -> tensor + %identity = "tf.Identity"(%replicated_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster"} : (tensor) -> tensor + %replicated_output = "tf.TPUReplicatedOutput(%identity) : (tensor) -> tensor + return %replicated_output : tensor + } + ``` + + will be transformed into: + + ```mlir + func @tpu_computation(%arg0: tensor) -> tensor { + %cluster = "tf_device.cluster"() ( { + %identity = "tf.Identity"(%arg0) : (tensor) -> tensor + tf_device.return %identity : tensor + }) {_xla_compile_device_type = "TPU", _replication_info = "cluster", num_cores_per_replica = 1, topology = "topology", device_assignment = [], padding_map = []} : () -> (tensor) + return %cluster : tensor + } + ``` + + The following replicated computation: + + ```mlir + func @tpu_computation(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + "tf.TPUReplicateMetadata"() {_xla_compile_device_type = "TPU", _replication_info = "cluster", num_relicas = 2, num_cores_per_replica = 1, topology = "topology", device_assignment = [], padding_map = []} : () -> () + %replicated_input = "tf.TPUReplicatedInput"(%arg0, %arg1) : (tensor, tensor) -> tensor + %identity = "tf.Identity"(%replicated_input) {_xla_compile_device_type = "TPU", _replication_info = "cluster"} : (tensor) -> tensor + %replicated_output:2 = "tf.TPUReplicatedOutput(%identity) : (tensor) -> (tensor, tensor) + return %replicated_output#0, %replicated_output#1 : tensor, tensor + } + ``` + + will be transformed into: + + ```mlir + func @tpu_computation(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %replicate:2 = tf_device.replicate([%arg0, %arg1] as %replicated_input) {n = 2 : i32} { + %cluster = "tf_device.cluster"() ( { + %identity = "tf.Identity"(%replicated_input) : (tensor) -> tensor + tf_device.return %identity : tensor + }) {_xla_compile_device_type = "TPU", _replication_info = "cluster", num_cores_per_replica = 1, topology = "topology", device_assignment = [], padding_map = []} : () -> (tensor) + tf_device.return %cluster : tensor + } + return %replicate#0, %replicate#1 : tensor, tensor + } + ``` + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateTPUClusterFormationPass()"; +} + +def ExtractHeadTailOutsideCompilationPass : Pass<"tf-extract-head-tail-outside-compilation", "ModuleOp"> { + let summary = "Extracts head or tail outside compilation to separate host launches before/after device cluster."; + + let description = [{ + This pass extracts a CPU computation cluster with `_xla_outside_compilation` + annotation from the head or tail of a Device cluster. + + For example: + + ```mlir + %cluster = "tf_device.cluster"() ( { + %a = "tf.A"(%arg0) {_xla_outside_compilation = "cluster1"} : (tensor) -> tensor + %b = "tf.B"(%a) : (tensor) -> tensor + %c = "tf.C"(%b) {_xla_outside_compilation = "cluster1"} : (tensor) -> tensor + tf_device.return %c : tensor + }) {num_cores_per_replica = 1, step_marker_location = "", padding_map = [], topology = "", device_assignment = []} : () -> tensor + return %cluster : tensor + ``` + + becomes: + + ```mlir + %0 = "tf_device.launch"() ( { + %3 = "tf.A"(%arg0) : (tensor) -> tensor + tf_device.return %3 : tensor + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> tensor + %1 = "tf_device.cluster"() ( { + %3 = "tf.B"(%0) : (tensor) -> tensor + tf_device.return %3 : tensor + }) {device_assignment = [], num_cores_per_replica = 1 : i64, padding_map = [], step_marker_location = "", topology = ""} : () -> tensor + %2 = "tf_device.launch"() ( { + %3 = "tf.C"(%1) : (tensor) -> tensor + tf_device.return %3 : tensor + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> tensor + return %2 : tensor + + ``` + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateExtractHeadTailOutsideCompilationPass()"; +} + +def ExtractOutsideCompilationPass : Pass<"tf-extract-outside-compilation", "ModuleOp"> { + let summary = "Extracts device outside compilation computation to a separate tf_device.parallel_execute region."; + + let description = [{ + This pass extracts a CPU computation cluster with `_xla_outside_compilation` + annotation, which denotes ops that should be run on CPU/host, from a device cluster. + Each outside compilation cluster is moved to + a tf_device.parallel_execute region. The device cluster is also moved to a + tf_device.parallel_execute region. Communication ops between device and host are + added to pass inputs/outputs to/from the outside compiled region. + + For example, the following tf_device.cluster with an op marked for `xla_outside_compilation`: + + ```mlir + func @outside_compilation() -> tensor { + %0 = "tf_device.cluster"() ( { + %1 = "tf.Const"() {_xla_outside_compilation = "0", value = dense<1.0> : tensor} : () -> (tensor) + %2 = "tf.Identity"(%1) {_xla_outside_compilation = "0"} : (tensor) -> (tensor) + %3 = "tf.AddV2"(%1, %2) : (tensor, tensor) -> (tensor) + tf_device.return %3 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor + } + ``` + + will become a tf_device.parallel_execute op with a CPU/host region and + a tf_device.cluster with communication ops to send data to/from device/host: + + ```mlir + func @outside_compilation() -> tensor { + %0 = "tf_device.parallel_execute"() ( { + "tf_device.launch"() ( { + %1 = "tf._XlaCompileMlirPlaceholderProgramKey"() : () -> tensor<3x!tf_type.string> + %2 = "tf._XlaRecvAtHost"(%1) {device_ordinal = 0 : i64, key = "host_compute_channel_0_0_args"} : (tensor<3x!tf_type.string>) -> tensor + %3 = "tf.Identity"(%2) : (tensor) -> tensor + "tf._XlaSendFromHost"(%3, %1) {device_ordinal = 0 : i64, key = "host_compute_channel_0_0_retvals"} : (tensor, tensor<3x!tf_type.string>) -> () + tf_device.return + }) {device = "/job:worker/replica:0/task:0/device:CPU:0"} : () -> () + tf_device.return + }, { + %1 = "tf_device.cluster"() ( { + %2 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + %3 = "tf._XlaHostComputeMlir"(%2) {recv_key = "host_compute_channel_0_0_retvals", send_key = "host_compute_channel_0_0_args", tpu_core = 0 : i64} : (tensor) -> tensor + %4 = "tf.AddV2"(%2, %3) : (tensor, tensor) -> tensor + tf_device.return %4 : tensor + }) {device_assignment = [], num_cores_per_replica = 1 : i64, topology = ""} : () -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + return %0 : tensor + } + ``` + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateExtractOutsideCompilationPass()"; +} + +def XlaClusterFormationPass : Pass<"tf-xla-cluster-formation", "ModuleOp"> { + let summary = "Encapsulate partitioned calls within a Cluster op"; + let description = [{ + This pass clusters `tf.PartitionedCall` and `tf.StatefulPartitionedCall` + with `_xla_compile_device_type` attribute into a `tf_device.cluster`. + Notice this pass will only rewrite the outermost call if there are nested + calls to avoid nested `tf.XlaLaunch` operations from being created later. + + For example, the following code + + ```mlir + func.func @main() -> tensor { + %0 = "tf.StatefulPartitionedCall"() {_xla_compile_device_type = "CPU", f = @stateful_pcall_func} : () -> (tensor) + func.return %0 : tensor + } + + func.func @stateful_pcall_func() -> tensor { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + func.return %0 : tensor + } + ``` + + will be transformed into, + + ```mlir + func.func @main() -> tensor { + %0 = "tf_device.cluster"() ({ + %1 = "tf.StatefulPartitionedCall"() {_xla_compile_device_type = "CPU", f = @stateful_pcall_func} : () -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + func.return %0 : tensor + } + + func.func @stateful_pcall_func() -> tensor { + %0 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + func.return %0 : tensor + } + + ``` + }]; + let constructor = "tensorflow::tf2xla::internal::CreateXlaClusterFormationPass()"; + let dependentDialects = ["mlir::tf_device::TensorFlowDeviceDialect"]; +} + +def MarkOpsForOutsideCompilationPass : Pass<"tf-mark-ops-for-outside-compilation", "ModuleOp"> { + let summary = "Marks ops in device cluster for outside compilation if they are unsupported on device."; + + let description = [{ + This pass marks unsupported ops in a device cluster with + `_xla_outside_compilation` attribute so the operations will run on the host + instead of the device. Unsupported ops are ops that can not be code + generated to run on the device for the cluster including: + + 1. String operations on TPUs. + 2. Operations that don't have a kernel defined for the device. + + This pass is conservative in that it will mark all ops for outside compilation + that can not be compiled for the device. Exceptions for this are added for ops + that will be rewritten or decomposed before compiling on device. + + + For example, tf_device.cluster op with an unsupported op, tf.UnsupportedOp: + + ```mlir + func @unsupported_op() -> tensor { + %0 = "tf_device.cluster"() ( { + %1 = "tf.UnsupportedOp"() : () -> tensor + %2 = "tf.Identity"(%1) : (tensor) -> tensor + tf_device.return %2 : tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor + } + ``` + + will mark tf.UnsupportedOp with `_xla_outside_compilation` attribute: + + ```mlir + func @unsupported_op() -> tensor { + %0 = "tf_device.cluster"() ( { + %1 = "tf.UnsupportedOp"() {_xla_outside_compilation = "auto0"} : () -> tensor + %2 = "tf.Identity"(%1) : (tensor) -> tensor + tf_device.return %2 : tensor + }) {allow_soft_placement = true, device_assignment = [], num_cores_per_replica = 1 : i64, topology = ""} : () -> tensor + return %0 : tensor + } + ``` + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateMarkOpsForOutsideCompilationPass()"; +} + +def HoistBroadcastReadPass : Pass<"tf-hoist-broadcast-read", "mlir::func::FuncOp"> { + let summary = "Hoist reads out of a replicate that are on a resource that is broacast to all replicas."; + + let description = [{ + Some `ReadVariableOp`s that are within a `tf_device.replicate` read the same + value across all replicas. These reads can be hoisted out of the + `tf_device.replicate` so there's one read for all replicas, and each replica + depends on the result of the read. This transform enables the + xla-broadcast-pass to optimize the broadcast value. + + For example, the following: + + ```mlir + tf_device.replicate {n = 2 : i32} { + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + "tf.OpA"(%0) : (tensor) -> () + } + ``` + + will be transformed into: + + ``mlir + %0 = "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>>) -> tensor + tf_device.replicate {n = 2 : i32} { + "tf.OpA"(%0) : (tensor) -> () + } + ``` + + We must ensure that there is a single underlying resource that not + distributed across replicas. There is a single underlying resource when the + resource device type is CPU, so we cautiously only apply in this case. + + To be cautious we never hoist a read that comes after a write to the same + resource. + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateHoistBroadcastReadPass()"; +} + +def XlaBroadcastPass : Pass<"tf-xla-broadcast", "mlir::func::FuncOp"> { + let summary = "Moves a broadcast from host into XLA, encoded as XlaAllReduce"; + + let description = [{ + This pass moves brodcasts from host TF ops into XLA. This enables use of + the inter-device network, which is faster than the inter-host network. + Broadcasts in XLA are encoded as XlaAllReduce. An all_reduce with all 0 + inputs except for one real input produces the same result as a broadcast. + + For example, the following: + + ```mlir + func.func @main(%arg0: tensor) -> () { + tf_device.replicate {n = 2 : i32} { + "tf_device.cluster"() ({ + "tf.OpA"(%arg0) : (tensor) -> () + tf_device.return + }) : () -> () + } + func.return + } + ``` + + will be transformed into + + ```mlir + func.func @main(%arg0: tensor) { + %elem = "tf.Const"() <{value = dense<0.000000e+00> : tensor}> : () -> tensor + %shape = "tf.Const"() <{value = dense<> : tensor<0xi64>}> : () -> tensor<0xi64> + %zero = "tf.Fill"(%shape, %elem) : (tensor<0xi64>, tensor) -> tensor + tf_device.replicate([%arg0, %zero] as %arg1: tensor) {n = 2 : i32} { + %arg1_id = "tf_device.launch"() <{device = "TPU_REPLICATED_HOST_0"}> ({ + %1 = "tf.Identity"(%arg1) : (tensor) -> tensor + tf_device.return %1 : tensor + }) : () -> tensor + "tf_device.cluster"() ({ + %group = "tf.Const"() <{value = dense<[[0, 1]]> : tensor<1x2xi32>}> : () -> tensor<1x2xi32> + %arg1_reduced = "tf.XlaAllReduce"(%arg1_id, %group) <{mode = "CrossReplica", reduce_op = "Add"}> : (tensor, tensor<1x2xi32>) -> tensor + "tf.OpA"(%arg1_reduced) : (tensor) -> () + tf_device.return + }) : () -> () + tf_device.return + } + return + } + ``` + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateXlaBroadcastPass()"; + let dependentDialects = ["mlir::tf_device::TensorFlowDeviceDialect"]; +} + +def TPUShardingIdentificationPass : Pass<"tf-tpu-sharding-identification", "ModuleOp"> { + let summary = "Identifies and handles inputs/outputs of TPU computation that is " + "sharded across logical cores."; + let constructor = "tensorflow::tf2xla::internal::CreateTPUShardingIdentificationPass()"; + let description = [{ + Bubbles up sharding configuration from `cluster_func` regions into + the attributes of `cluster_func`. This is done by parsing the + `XlaSharding` / `TPUPartitionedOutput` / `TPUPartitionedInput` ops inside + `cluster_func`. + + For example, given the following `cluster_func` wrapping `func`: + + ```mlir + func @test(%arg0: tensor<*xi32>) { + "tf_device.cluster_func"(%arg0) { + func = @func, + step_marker_location = ""} : (tensor<*xi32>) -> tensor<*xi32> + return + } + + func @func(%arg0: tensor<*xi32>) -> tensor<*xi32> { + %0 = "tf.XlaSharding"(%arg0) {_XlaSharding = "\01\02\03", + sharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + return %1 : tensor<*xi32> + } + ``` + + Now, cluster_func receives the following `*_sharding_configuration` + attributes, and `func` receives the mhlo.sharding attribute: + + ```mlir + func @test(%arg0: tensor<*xi32>) { + %0 = "tf_device.cluster_func"(%arg0) { + func = @func, + input_sharding_configuration = ["\01\02\03"], + output_sharding_configuration = ["\08\01\1A\01\01\22\01\00"], + step_marker_location = ""} : (tensor<*xi32>) -> tensor<*xi32> + return + } + func @func(%arg0: tensor<*xi32> {mhlo.sharding = "\01\02\03"}) -> + (tensor<*xi32> {mhlo.sharding = "\08\01\1A\01\01\22\01\00"}) { + %0 = "tf.XlaSharding"(%arg0) {_XlaSharding = "\01\02\03", sharding = "\01\02\03"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> tensor<*xi32> + return %1 : tensor<*xi32> + } + ``` + }]; +} + +def TPUValidateSessionInputsPass : Pass<"tf-tpu-validate-session-inputs", "mlir::ModuleOp"> { + let summary = "Validates inputs to the TPU TF/XLA bridge in session api"; + + let description = [{ + This pass checks that the IR has valid input to TPU TF/XLA bridge in session api. + It checks the relations of multiple ops. Properties of single ops are + checked by the 'verify' method of ops. + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateTPUValidateSessionInputsPass()"; +} + +def TPUValidateInputsPass : Pass<"tf-tpu-validate-inputs", "mlir::ModuleOp"> { + let summary = "Validates inputs to the TPU TF/XLA bridge"; + + let description = [{ + This pass checks that the IR has valid input to TPU TF/XLA bridge. + It checks the relations of multiple ops. Properties of single ops are + checked by the 'verify' method of ops. + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateTPUValidateInputsPass()"; +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_head_tail_outside_compilation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_head_tail_outside_compilation.cc new file mode 100644 index 0000000000000000000000000000000000000000..251eb77a8c5625333950777bbe42d1d834f54e86 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_head_tail_outside_compilation.cc @@ -0,0 +1,492 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// This pass extracts a CPU computation cluster with `_xla_outside_compilation` +// annotation from the head or tail of a TPU cluster. + +namespace { + +using mlir::Block; +using mlir::BlockArgument; +using mlir::BoolAttr; +using mlir::ModuleOp; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OperationPass; +using mlir::Region; +using mlir::StringAttr; +using mlir::Type; +using mlir::Value; +using mlir::WalkResult; +using mlir::func::FuncOp; + +constexpr char kXlaMapOutsideCompilationAttr[] = "_xla_map_outside_compilation"; +constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; + +// Return true if `op` has attributes that say it can be outside compiled by +// this pass. This pass ignores _xla_map_outside_compilation, which will only be +// handled by extract_outside_compilation pass. +bool HasOutsideCompilationAttribute(Operation* op) { + return op->getAttrOfType(kXlaOutsideCompilationAttr) != nullptr && + !op->hasAttrOfType(kXlaMapOutsideCompilationAttr); +} + +// Finds op that created a given value. If the value is a BlockArgument, this +// returns the owner of the Block. +Operation* GetOpOfValue(Value value) { + if (auto block_arg = mlir::dyn_cast(value)) + return block_arg.getOwner()->getParentOp(); + + return value.getDefiningOp(); +} + +// Checks if `op` is nested in `block`. +bool OpInBlock(Operation* op, Block* block) { + Block* op_block = op->getBlock(); + while (op_block) { + if (op_block == block) return true; + if (auto* parent_op = op_block->getParentOp()) { + op_block = parent_op->getBlock(); + } else { + break; + } + } + return false; +} + +// Wraps block in a Launch. External uses of ops in the block will be return +// values of the Launch and remapped to the Launch results. If `before` is set +// to true, the Launch is created before `op`. Otherwise the Launch is created +// after `op`. +mlir::tf_device::LaunchOp CreateLaunchForBlock(OpBuilder* builder, + Operation* op, bool before, + Block* launch_block, + llvm::StringRef host_device) { + // Find results and result types of ops in block that needs to returned. + llvm::SmallVector launch_results; + llvm::SmallVector launch_result_types; + for (Operation& head_outside_compiled_op : *launch_block) { + for (Value result : head_outside_compiled_op.getResults()) { + bool has_external_uses = false; + for (Operation* user : result.getUsers()) { + if (OpInBlock(user, launch_block)) continue; + has_external_uses = true; + break; + } + if (has_external_uses) { + launch_results.push_back(result); + launch_result_types.push_back(result.getType()); + } + } + } + + before ? builder->setInsertionPoint(op) : builder->setInsertionPointAfter(op); + auto launch = mlir::tf_device::LaunchOp::create( + *builder, op->getLoc(), builder->getStringAttr(host_device), + launch_result_types); + launch.getBody().push_back(launch_block); + + builder->setInsertionPointToEnd(&launch.GetBody()); + mlir::tf_device::ReturnOp::create(*builder, op->getLoc(), launch_results); + + return launch; +} + +// Checks if an operation is a supported TPU embedding op. +bool IsEmbeddingOp(Operation* op) { + return llvm::isa(op); +} + +// Returns a set of ops that are outside compiled and can be extracted to before +// the TPU computation. These ops are either connected to the inputs of the TPU +// computation or other ops that can be extracted, and have no operands from +// other ops in the TPU computation that cannot be extracted. +llvm::SmallVector FindOutsideCompiledOpsAtHead( + const mlir::TF::SideEffectAnalysis& side_effect_analysis, + mlir::tf_device::ClusterOp cluster) { + const auto& analysis = side_effect_analysis.GetAnalysisForFunc( + cluster->getParentOfType()); + Region* cluster_region = &cluster.getBody(); + llvm::SmallSetVector head_outside_compiled_ops; + + auto cluster_ops = cluster.GetBody().without_terminator(); + for (Operation& cluster_op : cluster_ops) { + if (!HasOutsideCompilationAttribute(&cluster_op)) continue; + // An outside compiled op can be extracted if its operands are not from + // other ops in the cluster that cannot be extracted. + + // Check if the side effecting op right before this side effecting op, if + // it is side effecting, can be head extracted. Because of op ordering due + // to side effects, if this is not true, this op cannot be head extracted. + // TODO(lyandy): Remove special handling of embedding ops. Currently the IR + // is in a topological sort order and depending on that ordering, embedding + // ops may prevent other ops from being head extracted. + auto predecessors = analysis.DirectControlPredecessors(&cluster_op); + if (!predecessors.empty() && !IsEmbeddingOp(&cluster_op)) { + bool skip = false; + for (Operation* predecessor : llvm::reverse(predecessors)) { + if (IsEmbeddingOp(predecessor)) continue; + skip = !head_outside_compiled_ops.contains(predecessor); + break; + } + if (skip) continue; + } + + auto walk_result = cluster_op.walk([&](Operation* op) { + for (Value operand : op->getOperands()) { + Operation* operand_op = GetOpOfValue(operand); + if (head_outside_compiled_ops.count(operand_op) || + operand_op == &cluster_op) + continue; + + if (operand_op->getParentRegion() == cluster_region) + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + + if (!walk_result.wasInterrupted()) + head_outside_compiled_ops.insert(&cluster_op); + } + + return head_outside_compiled_ops.takeVector(); +} + +// Moves head outside compiled ops into its own `tf_device.LaunchOp` +// computation before the cluster. +void CreateHeadComputation(OpBuilder* builder, + mlir::tf_device::ClusterOp cluster, + llvm::ArrayRef head_outside_compiled_ops, + llvm::StringRef host_device) { + Block* launch_block = new Block; + for (Operation* head_outside_compiled_op : head_outside_compiled_ops) { + head_outside_compiled_op->removeAttr(kXlaOutsideCompilationAttr); + head_outside_compiled_op->moveBefore(launch_block, launch_block->end()); + } + + mlir::tf_device::LaunchOp launch = CreateLaunchForBlock( + builder, cluster, /*before=*/true, launch_block, host_device); + + for (auto result : llvm::zip(launch.GetBody().getTerminator()->getOperands(), + launch.getResults())) + replaceAllUsesInRegionWith(std::get<0>(result), std::get<1>(result), + cluster.getBody()); +} + +// Extracts and move outside compiled ops that have no dependencies in the +// cluster to before the cluster. +mlir::LogicalResult LiftHeadOutsideCompiledOps( + OpBuilder* builder, + const mlir::TF::SideEffectAnalysis& side_effect_analysis, + const mlir::TF::RuntimeDevices& devices, mlir::tf_device::ClusterOp cluster, + std::string* host_device, bool* cluster_updated) { + llvm::SmallVector head_outside_compiled_ops = + FindOutsideCompiledOpsAtHead(side_effect_analysis, cluster); + if (head_outside_compiled_ops.empty()) return mlir::success(); + if (failed(tensorflow::GetHostDeviceOutsideComputation(devices, cluster, + host_device))) + return mlir::failure(); + + CreateHeadComputation(builder, cluster, head_outside_compiled_ops, + *host_device); + + *cluster_updated = true; + return mlir::success(); +} + +// Fills `tail_outside_compiled_ops` with ops that are outside compiled and +// can be extracted to after the TPU computation, and `cluster_results` with new +// results of the cluster. These ops are either connected to the output of the +// TPU computation or other ops that can be extracted, and have no results used +// by other ops in the TPU computation that cannot be extracted. +void FindOutsideCompiledOpsAtTailAndClusterResults( + const mlir::TF::SideEffectAnalysis& side_effect_analysis, + mlir::tf_device::ClusterOp cluster, + llvm::SmallVectorImpl* tail_outside_compiled_ops, + llvm::SmallVectorImpl* cluster_results) { + const auto& analysis = side_effect_analysis.GetAnalysisForFunc( + cluster->getParentOfType()); + Region* cluster_region = &cluster.getBody(); + llvm::SmallSetVector tail_outside_compiled_ops_set; + Operation* terminator = cluster.GetBody().getTerminator(); + llvm::SmallSetVector cluster_results_set; + cluster_results_set.insert(terminator->getOperands().begin(), + terminator->getOperands().end()); + + auto cluster_ops = llvm::reverse(cluster.GetBody().without_terminator()); + for (Operation& cluster_op : cluster_ops) { + if (!HasOutsideCompilationAttribute(&cluster_op)) continue; + + // Check if the side effecting op right after this side effecting op, if + // it is side effecting, can be tail extracted. Because of op ordering due + // to side effects, if this is not true, this op cannot be tail extracted. + // TODO(lyandy): Remove special handling of embedding ops. Currently the IR + // is in a topological sort order and depending on that ordering, embedding + // ops may prevent other ops from being tail extracted. + auto successors = analysis.DirectControlSuccessors( + &cluster_op, [&terminator](Operation* op) { return op != terminator; }); + if (!successors.empty() && !IsEmbeddingOp(&cluster_op)) { + bool skip = false; + for (Operation* successor : successors) { + if (IsEmbeddingOp(successor)) continue; + skip = !tail_outside_compiled_ops_set.contains(successor); + break; + } + if (skip) continue; + } + + llvm::SmallVector results_to_forward; + bool can_be_extracted = + llvm::all_of(cluster_op.getUsers(), [&](Operation* op) { + return op == terminator || tail_outside_compiled_ops_set.count(op); + }); + if (!can_be_extracted) continue; + + // Collect operands of cluster op that are generated within the cluster. + // These values should be returned by the cluster. + cluster_op.walk([&](Operation* op) { + for (Value operand : op->getOperands()) { + Operation* operand_op = GetOpOfValue(operand); + if (operand_op->getParentRegion() == cluster_region) + cluster_results_set.insert(operand); + } + }); + + // Remove results of op to be extracted as there are no uses in the cluster. + for (Value result : cluster_op.getResults()) + cluster_results_set.remove(result); + // Insert all ops including nested ops for checking outputs/side effects. + cluster_op.walk( + [&](Operation* op) { tail_outside_compiled_ops_set.insert(op); }); + + // Only add top level ops to output vector. + tail_outside_compiled_ops->push_back(&cluster_op); + } + + *cluster_results = cluster_results_set.takeVector(); +} + +// Moves tail outside compiled ops into its own `tf_device.LaunchOp` +// computation after the cluster. +void CreateTailComputation(OpBuilder* builder, + mlir::tf_device::ClusterOp cluster, + llvm::ArrayRef tail_outside_compiled_ops, + llvm::StringRef host_device) { + Block* launch_block = new Block; + for (Operation* tail_outside_compiled_op : tail_outside_compiled_ops) { + tail_outside_compiled_op->removeAttr(kXlaOutsideCompilationAttr); + tail_outside_compiled_op->moveBefore(launch_block, launch_block->begin()); + } + + mlir::tf_device::LaunchOp launch = CreateLaunchForBlock( + builder, cluster, /*before=*/false, launch_block, host_device); + + auto operand_not_in_launch = [&](mlir::OpOperand& operand) { + return !launch.getOperation()->isProperAncestor(operand.getOwner()); + }; + for (auto result : llvm::zip(launch.GetBody().getTerminator()->getOperands(), + launch.getResults())) + std::get<0>(result).replaceUsesWithIf(std::get<1>(result), + operand_not_in_launch); +} + +// Updates cluster with updated cluster results after extracting tail outside +// compiled ops. +mlir::tf_device::ClusterOp UpdateClusterResults( + OpBuilder* builder, mlir::tf_device::ClusterOp cluster, + llvm::ArrayRef new_cluster_results) { + Operation* old_terminator = cluster.GetBody().getTerminator(); + builder->setInsertionPoint(old_terminator); + mlir::tf_device::ReturnOp::create(*builder, old_terminator->getLoc(), + new_cluster_results); + old_terminator->erase(); + + builder->setInsertionPoint(cluster); + llvm::SmallVector new_cluster_result_types; + new_cluster_result_types.reserve(new_cluster_results.size()); + for (const auto& new_cluster_result : new_cluster_results) + new_cluster_result_types.push_back(new_cluster_result.getType()); + + auto new_cluster = mlir::tf_device::ClusterOp::create( + *builder, cluster.getLoc(), new_cluster_result_types, + /*operands=*/llvm::ArrayRef{}, cluster->getAttrs()); + new_cluster.getBody().takeBody(cluster.getBody()); + + auto operand_not_in_cluster = [&](mlir::OpOperand& operand) { + return !new_cluster.getOperation()->isProperAncestor(operand.getOwner()); + }; + for (auto result : + llvm::zip(new_cluster.GetBody().getTerminator()->getOperands(), + new_cluster.getResults())) + std::get<0>(result).replaceUsesWithIf(std::get<1>(result), + operand_not_in_cluster); + + cluster.erase(); + return new_cluster; +} + +// Extracts and move outside compiled ops that do not create dependencies in the +// cluster to after the cluster. +mlir::LogicalResult LiftTailOutsideCompiledOps( + OpBuilder* builder, + const mlir::TF::SideEffectAnalysis& side_effect_analysis, + const mlir::TF::RuntimeDevices& devices, std::string host_device, + mlir::tf_device::ClusterOp* cluster, bool* cluster_updated) { + llvm::SmallVector tail_outside_compiled_ops; + llvm::SmallVector cluster_results; + FindOutsideCompiledOpsAtTailAndClusterResults(side_effect_analysis, *cluster, + &tail_outside_compiled_ops, + &cluster_results); + if (tail_outside_compiled_ops.empty()) return mlir::success(); + + if (host_device.empty()) + if (failed(tensorflow::GetHostDeviceOutsideComputation(devices, *cluster, + &host_device))) + return mlir::failure(); + + // Forward all results of cluster first. These results will be remapped once + // a new cluster is formed. + cluster->replaceAllUsesWith( + cluster->GetBody().getTerminator()->getOperands()); + + CreateTailComputation(builder, *cluster, tail_outside_compiled_ops, + host_device); + + *cluster = UpdateClusterResults(builder, *cluster, cluster_results); + + *cluster_updated = true; + return mlir::success(); +} + +// Removes aliased outputs in cluster from ops outside of cluster. +void RemoveClusterAliasedOutputs(OpBuilder* builder, + mlir::tf_device::ClusterOp cluster) { + llvm::SmallVector used_old_cluster_results; + llvm::SmallVector new_cluster_results; + llvm::SmallVector new_cluster_result_types; + Operation* cluster_terminator = cluster.GetBody().getTerminator(); + for (auto result : + llvm::zip(cluster_terminator->getOperands(), cluster.getResults())) { + Value cluster_terminator_operand = std::get<0>(result); + if (cluster_terminator_operand.getDefiningOp() && + cluster.getOperation()->isProperAncestor( + cluster_terminator_operand.getDefiningOp())) { + new_cluster_results.push_back(cluster_terminator_operand); + new_cluster_result_types.push_back(cluster_terminator_operand.getType()); + used_old_cluster_results.push_back(std::get<1>(result)); + } else { + std::get<1>(result).replaceAllUsesWith(cluster_terminator_operand); + } + } + + if (new_cluster_results.size() == cluster.getNumResults()) return; + + builder->setInsertionPoint(cluster); + auto new_cluster = mlir::tf_device::ClusterOp::create( + *builder, cluster.getLoc(), new_cluster_result_types, + /*operands=*/llvm::ArrayRef{}, cluster->getAttrs()); + new_cluster.getBody().takeBody(cluster.getBody()); + new_cluster.GetBody().getTerminator()->setOperands(new_cluster_results); + + for (auto result : + llvm::zip(used_old_cluster_results, new_cluster.getResults())) + std::get<0>(result).replaceAllUsesWith(std::get<1>(result)); + + cluster.erase(); +} + +#define GEN_PASS_DEF_EXTRACTHEADTAILOUTSIDECOMPILATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +struct ExtractHeadTailOutsideCompilationPass + : public impl::ExtractHeadTailOutsideCompilationPassBase< + ExtractHeadTailOutsideCompilationPass> { + void runOnOperation() override; +}; + +void ExtractHeadTailOutsideCompilationPass::runOnOperation() { + auto& side_effect_analysis = getAnalysis(); + // Get runtime devices information from the closest parent module. + auto module = getOperation(); + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(module, &devices))) + return signalPassFailure(); + + OpBuilder builder(&getContext()); + llvm::SmallVector clusters; + module.walk( + [&](mlir::tf_device::ClusterOp cluster) { clusters.push_back(cluster); }); + + for (mlir::tf_device::ClusterOp cluster : clusters) { + std::string host_device; + bool cluster_updated = false; + if (failed(LiftHeadOutsideCompiledOps(&builder, side_effect_analysis, + devices, cluster, &host_device, + &cluster_updated)) || + failed(LiftTailOutsideCompiledOps(&builder, side_effect_analysis, + devices, host_device, &cluster, + &cluster_updated))) + return signalPassFailure(); + if (cluster_updated) RemoveClusterAliasedOutputs(&builder, cluster); + } +} + +} // anonymous namespace + +std::unique_ptr> +CreateExtractHeadTailOutsideCompilationPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_outside_compilation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_outside_compilation.cc new file mode 100644 index 0000000000000000000000000000000000000000..3e19298049f3abb5b776847af32059937f9b706c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/extract_outside_compilation.cc @@ -0,0 +1,1593 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/TypeRange.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/shape_inference.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using llvm::ArrayRef; +using llvm::SmallVector; +using mlir::Block; +using mlir::BlockArgument; +using mlir::DenseIntElementsAttr; +using mlir::IRMapping; +using mlir::Location; +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OperationPass; +using mlir::OpOperand; +using mlir::OpResult; +using mlir::OwningOpRef; +using mlir::RankedTensorType; +using mlir::StringAttr; +using mlir::StringRef; +using mlir::SymbolTable; +using mlir::Type; +using mlir::TypeRange; +using mlir::Value; +using mlir::ValueRange; +using mlir::WalkResult; +using mlir::func::FuncOp; +using mlir::func::ReturnOp; + +constexpr char kDeviceAttr[] = "device"; +constexpr char kHostFunctionAttr[] = "host_func"; +constexpr char kXlaMapOutsideCompilationAttr[] = "_xla_map_outside_compilation"; +constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; + +#define GEN_PASS_DEF_EXTRACTOUTSIDECOMPILATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +struct ExtractOutsideCompilation + : public impl::ExtractOutsideCompilationPassBase< + ExtractOutsideCompilation> { + void runOnOperation() override; +}; + +// Build a function containing `ops` with `inputs` and `outputs` using +// `builder`. The `ops` are cloned and modified to use the function arguments +// as inputs. +FuncOp BuildFunction(llvm::ArrayRef ops, + llvm::ArrayRef inputs, + llvm::ArrayRef outputs, OpBuilder* builder) { + llvm::SmallVector operand_types; + operand_types.reserve(inputs.size()); + for (Value v : inputs) operand_types.emplace_back(v.getType()); + llvm::SmallVector output_types; + output_types.reserve(outputs.size()); + for (Value v : outputs) output_types.emplace_back(v.getType()); + + auto func_type = builder->getFunctionType(operand_types, output_types); + + FuncOp outlined_func = + FuncOp::create(ops.front()->getLoc(), kHostFunctionAttr, func_type); + + // Create function body. + Block* outlined_func_block = outlined_func.addEntryBlock(); + + // Clone the operations and remap the inputs to use the function arguments. + IRMapping mapping; + mapping.map(inputs, outlined_func.getArguments()); + builder->setInsertionPoint(outlined_func_block, outlined_func_block->begin()); + for (Operation* op : ops) { + builder->clone(*op, mapping); + } + + // Set the returned values to use cloned ops results using mapping. + llvm::SmallVector results_after_mapping; + for (Value result : outputs) { + results_after_mapping.push_back(mapping.lookupOrDefault(result)); + } + + ReturnOp::create(*builder, ops.front()->getLoc(), results_after_mapping); + return outlined_func; +} + +// Encapsulates `func` in a module and serializes that module. +// `serialized_func_module` is set to the serialized module. +void EncapsulateFuncAndSerialize(FuncOp func, + std::string* serialized_func_module) { + // Create a new module to hold func and all referenced functions. + OwningOpRef module_for_func = + ModuleOp::create(mlir::UnknownLoc::get(func.getContext())); + SymbolTable symbol_table(module_for_func.get()); + + symbol_table.insert(func); + *serialized_func_module = + tensorflow::SerializeMlirModule(module_for_func.get()); +} + +// Returns whether `op` or ops nested in `op` are outside compiled. +bool HasOutsideCompilationNested(Operation* op) { + return op + ->walk([&](Operation* walked_op) { + if (op == walked_op) return WalkResult::advance(); + if (walked_op->hasAttr(kXlaOutsideCompilationAttr)) { + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }) + .wasInterrupted(); +} + +// Returns whether `op` or any ancestors of `op` are outside compiled. +bool HasOutsideCompilationAncestor(Operation* op) { + while (op) { + if (op->hasAttr(kXlaOutsideCompilationAttr)) { + return true; + } + op = op->getParentOp(); + } + return false; +} + +// Returns whether any ancestors of `op` are outside compiled. +bool HasOutsideCompilationAncestorExclusive(Operation* op) { + Operation* parent_op = op->getParentOp(); + if (!parent_op) return false; + return HasOutsideCompilationAncestor(parent_op); +} + +Operation* ApplyXlaHostTransferAttr(Operation* op, OpBuilder& builder) { + op->setAttr("_xla_has_host_transfer", builder.getBoolAttr(true)); + return op; +} + +// Creates a tf._XlaSendFromHost or tf._XlaSendFromHostV2 op. If device ordinal +// is present, a tf._XlaSendFromHostV2 op is created instead. +Operation* CreateSendFromHostOp(OpBuilder& builder, Location loc, + ValueRange inputs, Value compilation_key, + Value device_ordinal, + StringAttr device_type_attr, + llvm::StringRef communication_key) { + if (device_ordinal) + return ApplyXlaHostTransferAttr( + mlir::TF::_XlaSendFromHostV2Op::create( + builder, loc, inputs, + /*dynamic_key=*/compilation_key, device_ordinal, + builder.getStringAttr(communication_key), device_type_attr), + builder); + + return ApplyXlaHostTransferAttr( + mlir::TF::_XlaSendFromHostOp::create( + builder, loc, inputs, + /*dynamic_key=*/compilation_key, + builder.getStringAttr(communication_key), + /*device_ordinal=*/builder.getI64IntegerAttr(0), device_type_attr), + builder); +} + +// Creates a tf._XlaRecvAtHost or tf._XlaRecvAtHostV2 op. If device ordinal is +// present, a tf._XlaRecvAtHostV2 op is created instead. +Operation* CreateRecvAtHostOp(OpBuilder& builder, Location loc, + TypeRange output_types, Value compilation_key, + Value device_ordinal, StringAttr device_type_attr, + llvm::StringRef communication_key) { + if (device_ordinal) + return ApplyXlaHostTransferAttr( + mlir::TF::_XlaRecvAtHostV2Op::create( + builder, loc, output_types, /*dynamic_key=*/compilation_key, + device_ordinal, builder.getStringAttr(communication_key), + device_type_attr), + builder); + + return ApplyXlaHostTransferAttr( + mlir::TF::_XlaRecvAtHostOp::create( + builder, loc, output_types, /*dynamic_key=*/compilation_key, + builder.getStringAttr(communication_key), + /*device_ordinal=*/builder.getI64IntegerAttr(0), device_type_attr), + builder); +} + +// Clones an IfRegionOp 'if_region' and attributes and creates then/else regions +// with yield op and an empty block. +mlir::TF::IfRegionOp CloneEmptyIfWithPredicate(mlir::TF::IfRegionOp if_region, + OpBuilder& builder) { + // Mark op as stateful due to side-effecting communication ops added later. + auto host_side_if = mlir::TF::IfRegionOp::create( + builder, if_region.getLoc(), llvm::SmallVector{}, + if_region.getCond(), + /*is_stateless=*/false, if_region.get_thenFuncNameAttr(), + if_region.get_elseFuncNameAttr()); + + // Create empty then branch region. + auto& then_branch = host_side_if.getThenBranch(); + then_branch.push_back(new Block); + builder.setInsertionPointToEnd(&then_branch.front()); + mlir::TF::YieldOp::create(builder, if_region.getLoc(), + /*operands=*/ArrayRef{}); + + // Create empty else branch region. + auto& else_branch = host_side_if.getElseBranch(); + else_branch.push_back(new Block); + builder.setInsertionPointToEnd(&else_branch.front()); + mlir::TF::YieldOp::create(builder, if_region.getLoc(), + /*operands=*/ArrayRef{}); + return host_side_if; +} +// Creates a WhileRegionOp cond and body regions with yield op and +// an empty body. +mlir::TF::WhileRegionOp CloneEmptyWhile(uint64_t parallel_iterations, + Location loc, OpBuilder& builder) { + // Mark op as stateful due to side-effecting communication ops added later. + auto host_side_while = mlir::TF::WhileRegionOp::create( + builder, loc, /*output=*/ArrayRef{}, /*input=*/ArrayRef{}, + parallel_iterations, /*is_stateless=*/false, /*shape_invariant=*/false); + + // Create empty else branch region. + auto& body = host_side_while.getBody(); + body.push_back(new Block); + builder.setInsertionPointToEnd(&body.front()); + mlir::TF::YieldOp::create(builder, loc, /*operands=*/ArrayRef{}); + return host_side_while; +} + +// Creates a compilation key as placeholder. A placeholder compilation cache key +// is created because it is a required input to _XlaRecvAtHost and +// _XlaSendFromHost but the _XlaCompileMlir has not yet been created for device +// cluster that contains the outside compiled ops. This placeholder should be +// replaced by the TPU cluster _XlaCompileMlir in a subsequent pass. +mlir::TF::_XlaCompileMlirPlaceholderProgramKeyOp +CreateCompilationKeyPlaceholder(Location loc, OpBuilder& builder) { + auto result_type = + RankedTensorType::get({3}, builder.getType()); + return mlir::TF::_XlaCompileMlirPlaceholderProgramKeyOp::create( + builder, loc, /*program=*/result_type, llvm::ArrayRef{}); +} + +// Creates a `tf_device.launch` to wrap cluster ops. +mlir::tf_device::LaunchOp CreateLaunchOpForOutsideCluster( + OpBuilder& builder, Operation* loc_op, llvm::StringRef host_device, + llvm::SmallVector& return_value_from_host) { + llvm::SmallVector host_result_types; + host_result_types.reserve(return_value_from_host.size()); + for (Value old_result : return_value_from_host) + host_result_types.push_back(old_result.getType()); + + // An empty string placeholder is used for the device as that will be later + // populated with the device of the associated Device op. + // For TPU case, it is TPUReplicateMetadata op. + auto launch_op = mlir::tf_device::LaunchOp::create( + builder, loc_op->getLoc(), builder.getStringAttr(host_device), + /*result_types=*/host_result_types); + + launch_op.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&launch_op.GetBody()); + mlir::tf_device::ReturnOp::create(builder, loc_op->getLoc(), + return_value_from_host); + return launch_op; +} + +// Returns true if `op` has non-static shaped outputs. +bool HasDynamicOutputs(Operation* op) { + for (Value v : op->getResults()) { + if (mlir::TF::CanBeRefined(v.getType())) return true; + } + return false; +} + +// Returns true if any op in `cluster_ops` has outputs consumed by ops not +// `cluster_ops` with a non-static shape. +bool HasDynamicOutputs(const llvm::SmallSetVector& cluster_ops) { + for (Operation* op : cluster_ops) { + for (const OpOperand& use : op->getUses()) { + if (cluster_ops.count(use.getOwner())) { + continue; + } + if (mlir::TF::CanBeRefined(use.get().getType())) return true; + } + } + return false; +} + +bool HasDynamicExternalValues(Operation* op) { + return op + ->walk([](Operation* walked_op) { + for (Value v : walked_op->getOperands()) { + if (mlir::TF::CanBeRefined(v.getType())) { + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }) + .wasInterrupted(); +} + +// Returns operands of `cluster_ops` that need to be +// communicated from device->host. This is for the case when all operands have a +// static shape. +llvm::SmallSetVector GetStaticExternalOperands( + mlir::tf_device::ClusterOp device_cluster, + const llvm::SmallSetVector& cluster_ops) { + llvm::SmallSetVector external_values; + for (Operation* op : cluster_ops) { + op->walk([&](Operation* walked_op) { + if (llvm::isa(walked_op)) + return WalkResult::advance(); + for (Value v : walked_op->getOperands()) { + if (!tensorflow::TypeValidForXLA(v.getType())) continue; + if (auto* defining_op = v.getDefiningOp()) { + if (!op->isAncestor(defining_op) && + device_cluster->isAncestor(defining_op) && + !HasOutsideCompilationAncestor(defining_op) && + // Ignore operands that have already been received by a previously + // created cluster. + !llvm::isa(defining_op)) { + external_values.insert(v); + } + continue; + } + auto block_arg = mlir::cast(v); + if (block_arg.getParentRegion() == op->getParentRegion()) + external_values.insert(v); + } + return WalkResult::advance(); + }); + } + return external_values; +} + +// Returns every operand of `cluster_ops` that does not come from an op in +// `cluster_ops`. +llvm::SmallSetVector GetAllExternalOperands( + const llvm::SmallSetVector& cluster_ops) { + llvm::SmallSetVector external_values; + for (Operation* op : cluster_ops) { + op->walk([&](Operation* walked_op) { + for (Value v : walked_op->getOperands()) { + if (!tensorflow::TypeValidForXLA(v.getType())) continue; + Operation* defining_op = v.getDefiningOp(); + if (!defining_op || !cluster_ops.count(defining_op)) { + external_values.insert(v); + } + } + }); + } + return external_values; +} + +// Returns a SmallSetVector containing all of the operands that need to be +// communicated from device->host. +llvm::SmallSetVector GetExternalOperands( + mlir::tf_device::ClusterOp device_cluster, + const llvm::SmallSetVector& cluster_ops) { + // If there are any dynamic outputs, get all of the operands which are defined + // external to `cluster_ops`. + bool has_dynamic_outputs = HasDynamicOutputs(cluster_ops); + if (has_dynamic_outputs) { + return GetAllExternalOperands(cluster_ops); + } else { + return GetStaticExternalOperands(device_cluster, cluster_ops); + } +} + +// Gets all outputs that need to be communicated from host->device. +void GetExternalOutputs(const llvm::SmallSetVector& cluster_ops, + llvm::SmallSetVector& external_outputs, + llvm::SmallVector& host_outputs) { + bool has_dynamic_outputs = HasDynamicOutputs(cluster_ops); + llvm::SmallVector tmp_host_outputs; + for (Operation* op : cluster_ops) { + llvm::SmallDenseSet user_set; + for (Operation* user : op->getUsers()) { + // We skip any operations that are in the same outside compilation + // cluster that will be moved to the host at the same time since both + // defining op and user op will be moved to host. + if (cluster_ops.count(user)) { + continue; + } + // This is pessimistic and in some cases will add extra communication. + if (!HasOutsideCompilationAncestor(user) || has_dynamic_outputs || + HasDynamicOutputs(user)) { + if (!user_set.insert(user).second) continue; + for (Value v : user->getOperands()) { + if (tensorflow::TypeValidForXLA(v.getType()) && + v.getDefiningOp() == op && + !llvm::isa(user)) + external_outputs.insert(v); + if (v.getDefiningOp() == op && + llvm::isa(user)) + tmp_host_outputs.push_back(v); + } + } + } + } + + // Value in `tmp_host_outputs` may contain user in non return op, which has + // been in `external_outputs`. We need exclude those Value. + for (auto val : tmp_host_outputs) { + if (!external_outputs.contains(val)) host_outputs.push_back(val); + } +} + +// Output `shard_type`, which is the type of each shard, given `full_type`. If +// the full shape is (num_cores_per_replica * a, b, c), then the shard shape is +// (a, b, c). `context_op` is used for error reporting, in case of errors. +LogicalResult GetShardShapedType(Operation* context_op, + int num_cores_per_replica, Type full_type, + Type& shard_type) { + RankedTensorType ranked_type = mlir::dyn_cast(full_type); + if (!ranked_type) + return context_op->emitOpError() + << "A map_outside_compilation op's input and output types must be " + "ranked tensors."; + ArrayRef in_shape = ranked_type.getShape(); + if (in_shape.empty() || in_shape[0] < 0) { + return context_op->emitOpError() + << "A map_outside_compilation op's input and output shapes must " + "have rank at least one and the first dimension must be known."; + } + int64_t split_size = in_shape[0] / num_cores_per_replica; + if (in_shape[0] % num_cores_per_replica != 0) { + return context_op->emitOpError() + << "A map_outside_compilation op's input and output shapes must be " + "divisible by num_cores_per_replica=" + << num_cores_per_replica; + } + llvm::SmallVector shape; + shape.push_back(split_size); + for (int i = 1; i < in_shape.size(); ++i) { + shape.push_back(in_shape[i]); + } + shard_type = RankedTensorType::Builder(ranked_type).setShape(shape); + return mlir::success(); +} + +// Output `sharding`, which is the sharding of `val`. `context_op` is used for +// error reporting, in case of errors. +// TODO(b/255350483): Explicitly pass the sharding to map_outside_compilation, +// so it does not need to be retrieved from a Value. +LogicalResult GetShardingOfValue(Operation* context_op, Value val, + std::string& sharding) { + Operation* op = val.getDefiningOp(); + // val should always have a defining op because cluster inputs always have + // defining ops. + assert(op); + StringAttr sharding_attr = op->getAttrOfType("_XlaSharding"); + if (!sharding_attr) + return context_op->emitOpError() + << "A map_outside_compilation op's input should have an explicit " + "sharding. There is no _XlaSharding attribute on the input op."; + sharding = sharding_attr.str(); + return mlir::success(); +} + +// Create an `_XlaHostComputeMlir` for the map_outside_compilation case. Inputs +// are converted from split sharding to MANUAL sharding and outputs are +// converted from MANUAL sharding to split sharding. Set `common_split_sharding` +// if it has not yet been set. Output `full_outputs`, which is the outputs of +// the `_XlaHostComputeMlir` and add the +// `_XlaHostComputeMlir` to `host_compute_out_ops`. +LogicalResult CreateHostComputeMap( + Operation* original_op, OpBuilder& builder, Location loc, + ArrayRef inputs, ArrayRef outputs, + StringRef args_communication_key, StringRef retvals_communication_key, + StringRef serialized_func_module, int num_cores_per_replica, + std::string& common_split_sharding, SmallVector& full_outputs, + SmallVector& host_compute_out_ops) { + // Get output types. + llvm::SmallVector shard_output_types; + llvm::SmallVector full_output_types; + shard_output_types.reserve(outputs.size()); + full_output_types.reserve(outputs.size()); + for (const auto& output : outputs) { + Type shard_type; + if (failed(GetShardShapedType(original_op, num_cores_per_replica, + output.getType(), shard_type))) + return mlir::failure(); + shard_output_types.push_back(shard_type); + full_output_types.push_back(output.getType()); + } + + // Convert split sharded inputs to MANUAL sharded inputs. + // common_split_sharding is the split sharding that is common to all inputs + // and outputs. + llvm::SmallVector manual_inputs; + manual_inputs.reserve(inputs.size()); + for (Value in : inputs) { + Type shard_type; + if (failed(GetShardShapedType(original_op, num_cores_per_replica, + in.getType(), shard_type))) + return mlir::failure(); + std::string in_sharding; + if (failed(GetShardingOfValue(original_op, in, in_sharding))) + return mlir::failure(); + if (common_split_sharding.empty()) { + common_split_sharding = std::move(in_sharding); + } else { + if (common_split_sharding != in_sharding) + return original_op->emitOpError() + << "All inputs and outputs of map_outside_compilation should " + "have the same sharding."; + } + auto in_manual = mlir::TF::XlaSpmdFullToShardShapeOp::create( + builder, loc, shard_type, in, common_split_sharding, /*dim=*/-1, + /*unspecified_dims=*/builder.getI64ArrayAttr({})); + manual_inputs.push_back(in_manual); + } + + // Create the _XlaHostComputeMlirOp + auto host_compute = mlir::TF::_XlaHostComputeMlirOp::create( + builder, loc, shard_output_types, manual_inputs, + /*send_key=*/builder.getStringAttr(args_communication_key), + /*recv_key=*/builder.getStringAttr(retvals_communication_key), + /*host_mlir_module=*/builder.getStringAttr(serialized_func_module), + /*manual_sharding=*/builder.getBoolAttr(true)); + host_compute_out_ops.push_back(host_compute); + + // Convert MANUAL sharded outputs to split sharded outputs. + for (auto [full_type, out] : + llvm::zip(full_output_types, host_compute.getResults())) { + RankedTensorType full_type_ranked = + mlir::dyn_cast(full_type); + if (!full_type_ranked) + return original_op->emitOpError() + << "map_outside_compilation must have ranked outputs"; + auto out_full = mlir::TF::XlaSpmdShardToFullShapeOp::create( + builder, loc, full_type, out, common_split_sharding, + full_type_ranked.getShape(), + /*dim=*/-1, + /*unspecified_dims=*/builder.getI64ArrayAttr({})); + host_compute_out_ops.push_back(out_full); + full_outputs.push_back(out_full); + } + + return mlir::success(); +} + +// Create the _XlaHostComputeMlir with `inputs` and `outputs` for the ordinary +// outside_compilation case. +// Output `full_outputs`, which is the outputs of the `_XlaHostComputeMlir` and +// add the `_XlaHostComputeMlir` to `host_compute_out_ops`. +void CreateHostComputeNotMap(OpBuilder& builder, Location loc, + ArrayRef inputs, ArrayRef outputs, + StringRef args_communication_key, + StringRef retvals_communication_key, + StringRef serialized_func_module, + SmallVector& full_outputs, + SmallVector& host_compute_out_ops) { + llvm::SmallVector device_output_types; + for (const auto& output : outputs) + device_output_types.push_back(output.getType()); + auto host_compute = mlir::TF::_XlaHostComputeMlirOp::create( + builder, loc, device_output_types, inputs, + builder.getStringAttr(args_communication_key), + builder.getStringAttr(retvals_communication_key), + /*host_mlir_module=*/builder.getStringAttr(serialized_func_module)); + host_compute_out_ops.push_back(host_compute); + for (Value v : host_compute.getResults()) full_outputs.push_back(v); +} + +// Create the _XlaHostComputeMlir with `inputs` and `outputs`. +// Set `common_split_sharding` if it has not yet been set. Output +// `full_outputs`, which is the outputs of the `_XlaHostComputeMlir` and add the +// `_XlaHostComputeMlir` to `host_compute_out_ops`. +LogicalResult CreateHostCompute( + Operation* original_op, OpBuilder& builder, Location loc, + ArrayRef inputs, ArrayRef outputs, + StringRef args_communication_key, StringRef retvals_communication_key, + StringRef serialized_func_module, bool is_map_oc, int num_cores_per_replica, + std::string& common_split_sharding, SmallVector& full_outputs, + SmallVector& host_compute_out_ops) { + if (is_map_oc) { + return CreateHostComputeMap( + original_op, builder, loc, inputs, outputs, args_communication_key, + retvals_communication_key, serialized_func_module, + num_cores_per_replica, common_split_sharding, full_outputs, + host_compute_out_ops); + } else { + CreateHostComputeNotMap(builder, loc, inputs, outputs, + args_communication_key, retvals_communication_key, + serialized_func_module, full_outputs, + host_compute_out_ops); + return mlir::success(); + } +} + +void MarkOutsideCompiled(Operation* op) { + op->setAttr(kXlaOutsideCompilationAttr, + StringAttr::get(op->getContext(), "temp")); +} + +// Returns whether an outside compilation cluster should be closed. True when: +// 1. There is no non-XLA output. +// 2. There is a dynamically shaped output consumed by a non-outside compiled +// op. +// 3. There is no dynamically shaped output. +bool ShouldCloseCluster(llvm::ArrayRef outputs) { + bool has_dynamic_output = false; + bool has_nonxla_output = false; + for (Value v : outputs) { + if (mlir::TF::CanBeRefined(v.getType())) { + has_dynamic_output = true; + for (Operation* user : v.getUsers()) { + if (!HasOutsideCompilationAncestor(user) && + !llvm::isa(user)) + return true; + } + } + if (!tensorflow::TypeValidForXLA(v.getType())) + for (const Operation* user : v.getUsers()) + if (!llvm::isa(user)) + has_nonxla_output = true; + } + + return !has_nonxla_output && !has_dynamic_output; +} + +// Replaces `external_operands` with the results from `recv_at_host`. +// For non-static shapes, only replace operand usage if op is in the same +// region as insertion. +// For static-shapes, Replace operand usages if op is in the same region as +// insertion or if the op is outside compiled and will be moved to host later. +void ReplaceExternalOperandUsage(ArrayRef external_operands, + Operation* recv_at_host, + Operation* insertion_point, + Block* original_op_block) { + auto replace_operand_usage = [&](OpOperand& operand) { + if (mlir::TF::CanBeRefined(operand.get().getType()) || + HasDynamicOutputs(operand.getOwner())) { + return insertion_point->getParentRegion()->isAncestor( + operand.getOwner()->getParentRegion()); + } + return insertion_point->getParentRegion()->isAncestor( + operand.getOwner()->getParentRegion()) || + (HasOutsideCompilationAncestor(operand.getOwner()) && + original_op_block == operand.getOwner()->getBlock()); + }; + for (auto result : llvm::zip(external_operands, recv_at_host->getResults())) { + Value external_operand = std::get<0>(result); + external_operand.replaceUsesWithIf(std::get<1>(result), + replace_operand_usage); + } +} + +bool HasDynamicOutputs(llvm::ArrayRef outputs) { + for (Value v : outputs) { + if (mlir::TF::CanBeRefined(v.getType())) { + return true; + } + } + return false; +} + +// Replaces usages of `external_outputs` which are values returned by outside +// compilation with the corresponding outputs from `host_compute`. +void ReplaceExternalOutputUsage(ArrayRef external_outputs, + ArrayRef host_compute_outputs) { + bool has_dynamic_outputs = HasDynamicOutputs(external_outputs); + + auto replace_output_usage = [&](OpOperand& operand) { + // Don't replace output usages if in host computation (defining op and user + // in same region). + bool in_same_region = + operand.get().getDefiningOp()->getParentRegion()->isAncestor( + operand.getOwner()->getParentRegion()); + if (has_dynamic_outputs || HasDynamicOutputs(operand.getOwner())) { + return !in_same_region; + } else { + // Don't replace output usages in host computation or for outside + // compiled ops. + return !in_same_region && + !HasOutsideCompilationAncestor(operand.getOwner()); + } + }; + for (auto result : llvm::zip(external_outputs, host_compute_outputs)) { + Value external_output = std::get<0>(result); + external_output.replaceUsesWithIf(std::get<1>(result), + replace_output_usage); + } +} + +std::pair MakeCommunicationKeys( + ArrayRef clustered_ops, ArrayRef external_operands, + int communication_key_index, Operation& op) { + std::string args_communication_key = + llvm::formatv("host_compute_channel_{0}_args", (communication_key_index)) + .str(); + std::string retvals_communication_key = + llvm::formatv("host_compute_channel_{0}_retvals", + (communication_key_index)) + .str(); + + // Use a unique name when sending just the IfRegion predicate. This is + // for readable and to match the key in the TF2XLA bridge. + if (clustered_ops.size() == 1 && llvm::isa(op) && + external_operands.size() == 1) { + args_communication_key = + llvm::formatv("if_predicate_channel_{0}", (communication_key_index)) + .str(); + } + return std::pair(args_communication_key, retvals_communication_key); +} + +// Add ops to the host-side. These are `RecvAtHost`, `clustered_ops` moved from +// device cluster, `SendFromHost`. Add these host-side ops to `host_ops`. Return +// the `RecvAtHost` op. +Operation* CreateHostOps(ArrayRef clustered_ops, + ArrayRef external_operands, + ArrayRef external_outputs, + Operation* host_insertion_point, Value compilation_key, + Value device_ordinal, StringAttr device_type_attr, + OpBuilder& builder, Operation& op, + std::string args_communication_key, + std::string retvals_communication_key, + SmallVector& host_ops) { + builder.setInsertionPoint(host_insertion_point); + llvm::SmallVector host_operand_types; + for (const auto& operand : external_operands) + host_operand_types.push_back(operand.getType()); + + Operation* recv_at_host = CreateRecvAtHostOp( + builder, op.getLoc(), host_operand_types, compilation_key, device_ordinal, + device_type_attr, args_communication_key); + + if (!external_operands.empty()) host_ops.push_back(recv_at_host); + Operation* after_op = recv_at_host; + for (Operation* cluster_op : clustered_ops) { + cluster_op->moveAfter(after_op); + cluster_op->removeAttr(StringAttr::get(op.getContext(), kDeviceAttr)); + after_op = cluster_op; + host_ops.push_back(cluster_op); + } + + if (!external_outputs.empty()) { + Operation* send_from_host = CreateSendFromHostOp( + builder, op.getLoc(), external_outputs, compilation_key, device_ordinal, + device_type_attr, retvals_communication_key); + host_ops.push_back(send_from_host); + } + + return recv_at_host; +} + +// Clone the first outside compiled region to one for each TPU core. This is +// used for map_outside_compilation. +// Message identification arguments to RecvAtHost and SendFromHost are changed. +void CloneFirstHost(llvm::SmallVector& core_to_mapping, + ArrayRef core_to_host_insertion_point, + ArrayRef core_to_compilation_key, + ArrayRef core_to_device_ordinal, + int num_cores_per_replica, ArrayRef host0_ops, + OpBuilder& builder) { + for (int core = 1; core < num_cores_per_replica; ++core) { + for (Operation* op : host0_ops) { + builder.setInsertionPoint(core_to_host_insertion_point[core]); + Operation* clone = builder.clone(*op, core_to_mapping[core]); + core_to_mapping[core].map(op, clone); + if (auto recv_at_host = + llvm::dyn_cast(clone)) { + recv_at_host.setDeviceOrdinal(core); + clone->setOperand(0, core_to_compilation_key[core]); + } else if (auto send_from_host = + llvm::dyn_cast(clone)) { + send_from_host.setDeviceOrdinal(core); + clone->setOperand(1, core_to_compilation_key[core]); + } else if (auto recv_at_host = + llvm::dyn_cast(clone)) { + recv_at_host.setOperand(0, core_to_compilation_key[core]); + builder.setInsertionPoint(recv_at_host); + recv_at_host.setOperand(1, core_to_device_ordinal[core]); + } else if (auto send_from_host = + llvm::dyn_cast(clone)) { + send_from_host.setOperand(1, core_to_compilation_key[core]); + builder.setInsertionPoint(send_from_host); + send_from_host.setOperand(2, core_to_device_ordinal[core]); + } + } + } +} + +// Move `clustered_ops` to run on host and adds communication ops to transfer +// `external_operands` and `external_outputs` to/from device/host. Inserts +// ops at `insertion_point` and uses `compilation_key` and `device_ordinal` when +// creating comm ops. +LogicalResult MoveToHostSingleCluster( + ArrayRef clustered_ops, ArrayRef external_operands, + ArrayRef external_outputs, + llvm::SmallVector& core_to_mapping, + ArrayRef core_to_host_insertion_point, + ArrayRef core_to_compilation_key, + ArrayRef core_to_device_ordinal, StringAttr device_type_attr, + bool is_map_oc, int num_cores_per_replica, + std::string& common_split_sharding, int& communication_key_index) { + OpBuilder builder(core_to_host_insertion_point[0]); + Operation& op = *clustered_ops.back(); + Block* original_op_block = op.getBlock(); + auto [args_communication_key, retvals_communication_key] = + MakeCommunicationKeys(clustered_ops, external_operands, + communication_key_index, op); + + std::string serialized_func_module; + if (HasDynamicOutputs(external_outputs)) { + FuncOp shape_op = BuildFunction(clustered_ops, external_operands, + external_outputs, &builder); + EncapsulateFuncAndSerialize(shape_op, &serialized_func_module); + } + + builder.setInsertionPoint(&op); + SmallVector host_compute_outputs; + SmallVector host_compute_out_ops; + if (failed(CreateHostCompute( + &op, builder, op.getLoc(), external_operands, external_outputs, + args_communication_key, retvals_communication_key, + serialized_func_module, is_map_oc, num_cores_per_replica, + common_split_sharding, host_compute_outputs, host_compute_out_ops))) + return mlir::failure(); + + // Insert ops on the host side computation to receive data from device. + // host0_ops are the ops that will make up the first host process. In the + // map_outside_compilation case, there are multiple host processes, which will + // be created by cloning. + SmallVector host0_ops; + Operation* recv_at_host = CreateHostOps( + clustered_ops, external_operands, external_outputs, + core_to_host_insertion_point[0], core_to_compilation_key[0], + core_to_device_ordinal.empty() ? nullptr : core_to_device_ordinal[0], + device_type_attr, builder, op, std::move(args_communication_key), + std::move(retvals_communication_key), host0_ops); + + if (external_operands.empty()) { + recv_at_host->erase(); + } else { + ReplaceExternalOperandUsage( + external_operands, recv_at_host, + /*insertion_point=*/core_to_host_insertion_point[0], original_op_block); + } + + ReplaceExternalOutputUsage(external_outputs, host_compute_outputs); + + // Clone the first outside compiled region to one for each TPU core. + if (is_map_oc) + CloneFirstHost(core_to_mapping, core_to_host_insertion_point, + core_to_compilation_key, core_to_device_ordinal, + num_cores_per_replica, host0_ops, builder); + + ReplaceExternalOutputUsage(external_outputs, host_compute_outputs); + + if (external_operands.empty() && external_outputs.empty()) { + for (Operation* op : host_compute_out_ops) op->erase(); + } else { + ++communication_key_index; + } + + return mlir::success(); +} + +// Update is_map_oc the true if op has attribute _xla_map_outside_compilation +// and false otherwise. Check that this is consistent with the previous setting +// of is_map_oc. +LogicalResult UpdateIsMapOutsideCompilation(Operation& op, bool control_above, + std::optional& is_map_oc) { + bool op_is_map_oc = op.hasAttr(kXlaMapOutsideCompilationAttr); + if (is_map_oc) { + if (op_is_map_oc != *is_map_oc) { + return op.emitOpError() + << "Cannot mix map_outside_compilation with ordinary " + "outside_compilation in the same graph."; + } + } else { + is_map_oc = op_is_map_oc; + } + if (control_above && op_is_map_oc) { + return op.emitOpError() << "map_outside_compilation inside control flow " + "is not implemented."; + } + return mlir::success(); +} + +// Move outside compiled ops in `src` to `insertion_point` in host +// computation (may be temporarily with `device_cluster` but moved in subsequent +// call to this method). Communication ops are added in both `src` and at +// `insertion_point` using `compilation_key`, `device_ordinal` and +// `communication_key_index` which is incremented when used. Communication ops +// are added only when needed and at the location need. There are checks to +// ensure that duplicate communication between device and host is not added. +// When `return_value_from_host` is not nullptr, MoveToHostMultiCluster will +// also update its value. `control_above` means that this Block is within +// control flow, which is not currently supported with map_outside_compilation. +// `is_map_oc` tracks whether map_outside_compilation is used, for the whole +// program. Currently only map_outside_compilation-only or ordinary +// outside_compilation only is supported. +LogicalResult MoveToHostMultiCluster( + mlir::tf_device::ClusterOp device_cluster, Block* src, + ArrayRef core_to_host_insertion_point, + ArrayRef core_to_compilation_key, + ArrayRef core_to_device_ordinal, bool control_above, + std::optional& is_map_oc, int& communication_key_index, + llvm::SmallVector* return_value_from_host = nullptr) { + const int num_cores_per_replica = core_to_host_insertion_point.size(); + // common_split_sharding is set upon the first use of map_outside_compilation. + std::string common_split_sharding; + llvm::SmallVector core_to_mapping; + for (int core = 0; core < num_cores_per_replica; ++core) + core_to_mapping.emplace_back(); + // Contains all of the outside compiled operations that should be moved to the + // host using a single `_XlaHostComputeMlir` op. This should only contain a + // single op except in the case where some of the input/output shapes are + // non-static. + llvm::SmallSetVector clustered_ops; + auto device_type_attr = device_cluster->getAttrOfType( + mlir::TF::kCompileDeviceTypeAttr); + + for (Operation& op : llvm::make_early_inc_range(*src)) { + if (HasOutsideCompilationAncestorExclusive(&op) || + !op.hasAttr(kXlaOutsideCompilationAttr)) + continue; + + if (failed(UpdateIsMapOutsideCompilation(op, control_above, is_map_oc))) + return mlir::failure(); + + llvm::SmallSetVector external_outputs; + llvm::SmallVector host_outputs; + // We want to move the clustered_ops if the op to be added has all + // statically shaped operands since we can't ensure that the static shapes + // has been sent back to host in all cases. See + // @static_shapes_sandwiched_outside_compilation MLIR test for an example. + if (!HasDynamicExternalValues(&op) && !clustered_ops.empty()) { + llvm::SmallSetVector external_operands = + GetExternalOperands(device_cluster, clustered_ops); + GetExternalOutputs(clustered_ops, external_outputs, host_outputs); + if (return_value_from_host) { + for (auto& output : host_outputs) { + return_value_from_host->push_back(output); + } + } + if (failed(MoveToHostSingleCluster( + clustered_ops.getArrayRef(), external_operands.getArrayRef(), + external_outputs.getArrayRef(), core_to_mapping, + core_to_host_insertion_point, core_to_compilation_key, + core_to_device_ordinal, device_type_attr, *is_map_oc, + num_cores_per_replica, common_split_sharding, + communication_key_index))) + return mlir::failure(); + clustered_ops.clear(); + } + + clustered_ops.insert(&op); + + // Get the outputs that need to be communicated from host -> device. + external_outputs.clear(); + host_outputs.clear(); + GetExternalOutputs(clustered_ops, external_outputs, host_outputs); + + if (ShouldCloseCluster(external_outputs.getArrayRef())) { + // Get the operands that need to be communicated from device -> host. + llvm::SmallSetVector external_operands = + GetExternalOperands(device_cluster, clustered_ops); + if (return_value_from_host) { + for (auto& output : host_outputs) { + return_value_from_host->push_back(output); + } + } + + if (failed(MoveToHostSingleCluster( + clustered_ops.getArrayRef(), external_operands.getArrayRef(), + external_outputs.getArrayRef(), core_to_mapping, + core_to_host_insertion_point, core_to_compilation_key, + core_to_device_ordinal, device_type_attr, *is_map_oc, + num_cores_per_replica, common_split_sharding, + communication_key_index))) + return mlir::failure(); + clustered_ops.clear(); + } + } + return mlir::success(); +} + +void GetReturnValueFromDevice( + mlir::tf_device::ClusterOp device_cluster, + const llvm::SmallVector& return_value_from_host, + llvm::SmallVector& return_value_from_device) { + if (auto return_op = llvm::dyn_cast_or_null( + device_cluster.GetBody().getTerminator())) { + for (auto v : return_op.getOperands()) { + if (absl::c_count(return_value_from_host, v) == 0) { + return_value_from_device.push_back(v); + } + } + } +} + +// Decompose control flow in `device_cluster` into device computation and host +// (outside compiled) computation into two separate control flow ops with +// communication between the device/host for data dependencies. Both device and +// host control flow initially remain within `device_cluster` and a subsequency +// call to MoveToHostSingleCluster moves the host side control flow to the host +// launch in tf_device.parallel_execute. Uses `compilation_key, +// `device_ordinal` and `communication_key_index` when creating communication +// ops. +LogicalResult DecomposeControlFlow(mlir::tf_device::ClusterOp device_cluster, + ArrayRef core_to_compilation_key, + ArrayRef core_to_device_ordinal, + int& communication_key_index, + std::optional& is_map_oc) { + auto result = device_cluster.GetBody().walk([&](Operation* op) { + if (auto if_op = llvm::dyn_cast(op)) { + if (!HasOutsideCompilationNested(op)) return WalkResult::advance(); + OpBuilder builder(if_op); + auto host_if = CloneEmptyIfWithPredicate(if_op, builder); + if (failed(MoveToHostMultiCluster( + device_cluster, &if_op.getThenBranch().front(), + {host_if.getThenBranch().front().getTerminator()}, + core_to_compilation_key, core_to_device_ordinal, + /*control_above=*/true, is_map_oc, communication_key_index))) + return WalkResult::interrupt(); + if (failed(MoveToHostMultiCluster( + device_cluster, &if_op.getElseBranch().front(), + {host_if.getElseBranch().front().getTerminator()}, + core_to_compilation_key, core_to_device_ordinal, + /*control_above=*/true, is_map_oc, communication_key_index))) + return WalkResult::interrupt(); + // Mark op as stateful due to side-effecting communication ops. + if_op->setAttr("is_stateless", builder.getBoolAttr(false)); + MarkOutsideCompiled(host_if.getOperation()); + } + if (auto while_op = llvm::dyn_cast(op)) { + if (!HasOutsideCompilationNested(op)) return WalkResult::advance(); + OpBuilder builder(while_op); + auto host_while = CloneEmptyWhile(while_op.getParallelIterations(), + while_op.getLoc(), builder); + const auto condition_send_recv_key = + llvm::formatv("while_condition_channel_{0}", + communication_key_index++) + .str(); + auto& cond = host_while.getCond(); + cond.push_back(new Block); + auto condition = + while_op.getCond().front().getTerminator()->getOperand(0); + builder.setInsertionPoint(while_op.getCond().front().getTerminator()); + mlir::TF::XlaSendToHostOp::create(builder, while_op.getLoc(), condition, + condition_send_recv_key); + // device_ordinal0 is the ordinal of TPU_REPLICATED_CORE_0 and is only + // used in the replicated case. + Value device_ordinal0 = nullptr; + if (!core_to_device_ordinal.empty()) + device_ordinal0 = core_to_device_ordinal[0]; + builder.setInsertionPointToEnd(&cond.front()); + auto recv_condition_at_host = CreateRecvAtHostOp( + builder, while_op.getLoc(), TypeRange{condition.getType()}, + core_to_compilation_key[0], device_ordinal0, + device_cluster->getAttrOfType( + mlir::TF::kCompileDeviceTypeAttr), + condition_send_recv_key); + mlir::TF::YieldOp::create(builder, while_op.getLoc(), + recv_condition_at_host->getResults()); + + if (failed(MoveToHostMultiCluster( + device_cluster, &while_op.getCond().front(), + {recv_condition_at_host}, core_to_compilation_key, + core_to_device_ordinal, + /*control_above=*/true, is_map_oc, communication_key_index))) + return WalkResult::interrupt(); + if (failed(MoveToHostMultiCluster( + device_cluster, &while_op.getBody().front(), + {host_while.getBody().front().getTerminator()}, + core_to_compilation_key, core_to_device_ordinal, + /*control_above=*/true, is_map_oc, communication_key_index))) + return WalkResult::interrupt(); + // Mark op as stateful due to side-effecting communication ops. + while_op->setAttr("is_stateless", builder.getBoolAttr(false)); + MarkOutsideCompiled(host_while.getOperation()); + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return mlir::failure(); + return mlir::success(); +} + +// Removes outside compilation from all ops inside `host_launch_op`. Should +// only be run after all outside compiled ops have been moved to +// `host_launch_op`. +void RemoveOutsideCompilation(mlir::tf_device::LaunchOp host_launch_op) { + host_launch_op.GetBody().walk([&](Operation* op) { + if (op->hasAttr(kXlaOutsideCompilationAttr)) { + op->removeAttr( + StringAttr::get(op->getContext(), kXlaOutsideCompilationAttr)); + } + }); +} + +// The results of parallel executes is the combination of return values from +// both host and device. +llvm::SmallVector GetParallelExecuteResultsTypes( + ArrayRef return_value_from_host, + ArrayRef return_value_from_device) { + llvm::SmallVector parallel_execute_result_types; + const int num_of_outputs = + return_value_from_host.size() + return_value_from_device.size(); + parallel_execute_result_types.reserve(num_of_outputs); + for (Value result : return_value_from_host) + parallel_execute_result_types.push_back(result.getType()); + for (Value result : return_value_from_device) + parallel_execute_result_types.push_back(result.getType()); + return parallel_execute_result_types; +} + +// Remap the device cluster results with parallel execute op results +llvm::SmallVector GetRemappedTpuClusterResults( + mlir::tf_device::ClusterOp device_cluster, + const llvm::SmallVector& return_value_from_host, + const llvm::SmallVector& return_value_from_device) { + llvm::SmallVector remapped_device_cluster_results; + + std::vector order_from_new_output_to_previous_output; + order_from_new_output_to_previous_output.reserve( + return_value_from_host.size() + return_value_from_device.size()); + + llvm::SmallDenseMap> return_operand_map; + auto return_op = llvm::dyn_cast( + device_cluster.GetBody().getTerminator()); + + for (OpOperand& operand : return_op->getOpOperands()) { + auto operand_idx = operand.getOperandNumber(); + return_operand_map[return_op.getOperand(operand_idx)].push_back( + operand_idx); + } + // `return_value_from_host` and `return_value_from_device` together contain + // all elements in operands in `return_op`, including duplicated ones. They + // are one to one mapping. Therefore, when we iterate `return_value_from_host` + // and `return_value_from_device`, it is safe to access return_operand_map and + // call `front()` method. + for (const auto& return_val : return_value_from_host) { + order_from_new_output_to_previous_output.push_back( + return_operand_map[return_val].front()); + return_operand_map[return_val].pop_front(); + } + + for (const Value& return_val : return_value_from_device) { + order_from_new_output_to_previous_output.push_back( + return_operand_map[return_val].front()); + return_operand_map[return_val].pop_front(); + } + + for (auto idx : order_from_new_output_to_previous_output) + remapped_device_cluster_results.push_back(device_cluster.getResult(idx)); + + return remapped_device_cluster_results; +} + +// Remap cluster results with parallel_execute results if user is outside of +// parallel_execute. +void RemapDeviceClusterResultsWithParallelExecuteResults( + mlir::tf_device::ClusterOp device_cluster, + mlir::tf_device::ParallelExecuteOp parallel_execute_op, + const llvm::SmallVector& return_value_from_host, + const llvm::SmallVector& return_value_from_device) { + llvm::SmallVector remapped_device_cluster_results = + GetRemappedTpuClusterResults(device_cluster, return_value_from_host, + return_value_from_device); + + for (auto result : llvm::zip(remapped_device_cluster_results, + parallel_execute_op.getResults())) { + Value device_cluster_result = std::get<0>(result); + Value parallel_execute_result = std::get<1>(result); + for (auto& use : + llvm::make_early_inc_range(device_cluster_result.getUses())) + if (!parallel_execute_op.getOperation()->isProperAncestor(use.getOwner())) + use.set(parallel_execute_result); + } +} + +// Get the vector of results for new device cluster +llvm::SmallVector GetNewDeviceResults( + ArrayRef return_value_from_device) { + llvm::SmallVector device_results; + device_results.reserve(return_value_from_device.size()); + for (Value old_result : return_value_from_device) + device_results.push_back(old_result); + return device_results; +} + +// Get the vector of types of results for new device cluster +llvm::SmallVector GetNewDeviceTypes( + ArrayRef return_value_from_device) { + llvm::SmallVector device_result_types; + device_result_types.reserve(return_value_from_device.size()); + for (Value old_result : return_value_from_device) + device_result_types.push_back(old_result.getType()); + return device_result_types; +} + +// Move ops in old device cluster to new device cluster +void MoveOldTpuClusterToNewTpuCluster(mlir::tf_device::ClusterOp device_cluster, + Operation* after_op_r) { + for (Operation& op : llvm::make_early_inc_range(device_cluster.GetBody())) { + if (&op != device_cluster.GetBody().getTerminator()) { + op.moveBefore(after_op_r); + } + } +} + +// Move ops in the tmp host launch op to new host launch op +void MoveTmpLaunchOpToNewLaunchOp(mlir::tf_device::LaunchOp tmp_host_launch_op, + Operation* after_op_host_cluster) { + for (Operation& op : + llvm::make_early_inc_range(tmp_host_launch_op.GetBody())) { + if (&op != tmp_host_launch_op.GetBody().getTerminator()) { + op.moveBefore(after_op_host_cluster); + } + } +} + +// Since we have the outputs from host and device computation after moving +// outside compiled ops, we can create the actual parallel_execute regions. +// Still, one region is for the host computation for outside compilation and +// the other one is for the original Device cluster computation. +mlir::tf_device::ParallelExecuteOp CreateFinalParallelExecuteOp( + OpBuilder& builder, int num_regions, ArrayRef core_to_host, + mlir::tf_device::ClusterOp device_cluster, + ArrayRef core_to_tmp_host_launch, + ArrayRef return_value_from_host, + ArrayRef return_value_from_device) { + llvm::SmallVector parallel_execute_result_types = + GetParallelExecuteResultsTypes(return_value_from_host, + return_value_from_device); + + builder.setInsertionPoint(device_cluster); + auto parallel_execute_op = mlir::tf_device::ParallelExecuteOp::create( + builder, device_cluster.getLoc(), num_regions, + parallel_execute_result_types); + SmallVector core_to_host_launch; + for (int core = 0; core < core_to_tmp_host_launch.size(); ++core) { + Block& host_computation_block = + parallel_execute_op.GetRegionBlockWithIndex(core); + builder.setInsertionPointToEnd(&host_computation_block); + + // map_outside_compilation with return values from host is not implemented. + // This would only be needed if head-tail-outside-compilation supports + // map_outside_compilation"; + assert(core == 0 || return_value_from_host.empty()); + + // Create a single launch op for all outside compiled ops. + llvm::SmallVector host_results; + host_results.insert(host_results.end(), return_value_from_host.begin(), + return_value_from_host.end()); + mlir::tf_device::LaunchOp host_launch_op = CreateLaunchOpForOutsideCluster( + builder, device_cluster, core_to_host[core], host_results); + core_to_host_launch.push_back(host_launch_op); + + // Create a return op for host computation block + builder.setInsertionPointToEnd(&host_computation_block); + mlir::tf_device::ReturnOp::create(builder, device_cluster.getLoc(), + host_launch_op->getResults()); + } + + // Move the launch body to last parallel_execute block. + Block& parallel_execute_device_block = + parallel_execute_op.GetRegionBlockWithIndex( + core_to_tmp_host_launch.size()); + builder.setInsertionPointToEnd(¶llel_execute_device_block); + + // Get the vector of results and types of results for new device cluster + llvm::SmallVector device_results = + GetNewDeviceResults(return_value_from_device); + llvm::SmallVector device_result_types = + GetNewDeviceTypes(return_value_from_device); + + // Create a empty device cluster op with same attribute but different return + // type + auto new_device_cluster = mlir::tf_device::ClusterOp::create( + builder, device_cluster.getLoc(), device_result_types, + /*operands=*/llvm::ArrayRef{}, device_cluster->getAttrs()); + + new_device_cluster.getBody().push_back(new Block); + builder.setInsertionPointToEnd(&new_device_cluster.GetBody()); + + // Create return op for device computation region in the paralle_execute op + Operation* after_op_r = mlir::tf_device::ReturnOp::create( + builder, new_device_cluster.getLoc(), device_results); + + builder.setInsertionPointToEnd(¶llel_execute_device_block); + + // Create return op for the new device cluster op + mlir::tf_device::ReturnOp::create(builder, device_cluster.getLoc(), + new_device_cluster.getResults()); + + MoveOldTpuClusterToNewTpuCluster(device_cluster, after_op_r); + + // Move each host-side Launch op. + for (int core = 0; core < core_to_tmp_host_launch.size(); ++core) { + Operation* after_op_host_cluster = + core_to_host_launch[core].GetBody().getTerminator(); + MoveTmpLaunchOpToNewLaunchOp(core_to_tmp_host_launch[core], + after_op_host_cluster); + } + + return parallel_execute_op; +} + +// Creates a `parallel_execute` op with a region for host computation and +// a region for `device_cluster` computation by extracting outside compiled ops +// to host computation. +LogicalResult CreateParallelExecuteForOutsideCompilation( + mlir::tf_device::ClusterOp device_cluster, + llvm::SmallVector& ops, + std::optional& is_map_oc, ArrayRef core_to_host, + bool has_tpu_device) { + OpBuilder builder(device_cluster); + llvm::SmallVector returns_from_host; + + // Create a temporary parallel_execute. This is temporary because the result + // type is not determined until after it is filled. The parallel_execute has + // `num_host_regions` assigned to hosts and 1 region for the Device cluster. + // In the ordinary outside compilation case `num_host_regions` is 1 and in the + // `map_outside_compilation` case `num_host_regions == num_cores_per_replica`. + const int num_host_regions = core_to_host.size(); + const int num_regions = 1 + num_host_regions; + auto tmp_parallel_execute_op = mlir::tf_device::ParallelExecuteOp::create( + builder, device_cluster.getLoc(), num_regions, llvm::ArrayRef{}); + SmallVector core_to_host_insertion_point; + SmallVector core_to_tmp_launch; + SmallVector compilation_key_ops; + SmallVector core_to_compilation_key; + SmallVector core_to_device_ordinal_op; + SmallVector core_to_device_ordinal; + for (int core = 0; core < num_host_regions; ++core) { + Block& tmp_host_computation_block = + tmp_parallel_execute_op.GetRegionBlockWithIndex(core); + builder.setInsertionPointToEnd(&tmp_host_computation_block); + // Create a single tmp launch op for all outside compiled ops. + llvm::SmallVector tmp_host_results; + mlir::tf_device::LaunchOp tmp_host_launch_op = + CreateLaunchOpForOutsideCluster(builder, device_cluster, + core_to_host[core], tmp_host_results); + core_to_tmp_launch.push_back(tmp_host_launch_op); + // Create a tmp return op for tmp host computation block + builder.setInsertionPointToEnd(&tmp_host_computation_block); + mlir::tf_device::ReturnOp::create(builder, device_cluster.getLoc(), + llvm::ArrayRef{}); + core_to_host_insertion_point.push_back( + tmp_host_launch_op.GetBody().getTerminator()); + + builder.setInsertionPoint(tmp_host_launch_op.GetBody().getTerminator()); + + // Create message identification ops. + Operation* compilation_key_op = nullptr; + Value compilation_key = nullptr; + Operation* device_ordinal_op = nullptr; + compilation_key_op = + CreateCompilationKeyPlaceholder(device_cluster.getLoc(), builder); + compilation_key = + llvm::dyn_cast( + compilation_key_op) + .getProgram(); + if (has_tpu_device) { + device_ordinal_op = mlir::TF::_TPUDeviceOrdinalPlaceholderOp::create( + builder, device_cluster.getLoc(), + RankedTensorType::get({}, builder.getI64Type()), + builder.getI64IntegerAttr(core)); + } else { + device_ordinal_op = mlir::TF::ConstOp::create( + builder, device_cluster.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get({}, builder.getI64Type()), + static_cast(0))); + } + compilation_key_ops.push_back(compilation_key_op); + core_to_compilation_key.push_back(compilation_key); + core_to_device_ordinal_op.push_back(device_ordinal_op); + if (device_cluster->getParentOfType()) + core_to_device_ordinal.push_back( + core_to_device_ordinal_op[core]->getResults()[0]); + } + + builder.setInsertionPoint(tmp_parallel_execute_op); + // communication_key_index is part of the message identifier and is + // incremented for each _XlaHostComputeMlir. + int communication_key_index = 0; + + // Decompose control flow into device and host control flow when outside + // compilation is included. + if (failed(DecomposeControlFlow(device_cluster, core_to_compilation_key, + core_to_device_ordinal, + communication_key_index, is_map_oc))) + return mlir::failure(); + + // Move all outside compiled ops including control flow to tmp host launch. + // Also set the values returned from the host when ops are moved. + if (failed(MoveToHostMultiCluster( + device_cluster, &device_cluster.GetBody(), + core_to_host_insertion_point, core_to_compilation_key, + core_to_device_ordinal, + /*control_above=*/false, is_map_oc, communication_key_index, + &returns_from_host))) + return mlir::failure(); + + llvm::SmallVector returns_from_device; + GetReturnValueFromDevice(device_cluster, returns_from_host, + returns_from_device); + + // Remove unused message identification ops. + if (communication_key_index == 0) + for (auto op : compilation_key_ops) op->erase(); + if (communication_key_index == 0 || core_to_device_ordinal.empty()) + for (auto op : core_to_device_ordinal_op) op->erase(); + + for (mlir::tf_device::LaunchOp tmp_host_launch_op : core_to_tmp_launch) + RemoveOutsideCompilation(tmp_host_launch_op); + + mlir::tf_device::ParallelExecuteOp parallel_execute_op = + CreateFinalParallelExecuteOp(builder, num_regions, core_to_host, + device_cluster, core_to_tmp_launch, + returns_from_host, returns_from_device); + + ops.push_back(tmp_parallel_execute_op); + // Remap cluster results with parallel_execute results if user is outside of + // parallel_execute. + RemapDeviceClusterResultsWithParallelExecuteResults( + device_cluster, parallel_execute_op, returns_from_host, + returns_from_device); + + device_cluster.erase(); + + return mlir::success(); +} + +// Check that cluster results are valid. An result is invalid when it does not +// have a valid XLA type. +LogicalResult CheckClusterResults(mlir::tf_device::ClusterOp cluster) { + for (OpResult result : cluster.getResults()) { + if (!tensorflow::TypeValidForXLA(result.getType())) { + return cluster.emitError() + << "The ExtractHeadTailOutsideCompilation pass produced a Device " + "cluster with a result with a non-XLA type: " + << result.getType(); + } + } + return mlir::success(); +} + +// Check that op marked for outside compilation has an ancestor also marked for +// outside compilation. +LogicalResult CheckAncestorNotOutsideComp(Operation* op) { + if (!op->getAttrOfType(kXlaOutsideCompilationAttr)) + return mlir::success(); + Operation* iter_op = op; + while (auto* parent_op = iter_op->getParentOp()) { + if (parent_op->getAttrOfType(kXlaOutsideCompilationAttr)) { + return op->emitOpError() + << "An op marked for outside compilation (having attribute " + << kXlaOutsideCompilationAttr + << ") has an ancestor marked for outside compilation."; + } + iter_op = parent_op; + } + return mlir::success(); +} + +// Check the validity of the module, pre-pass. +LogicalResult CheckPreconditions(ModuleOp module) { + auto walk_result = module.walk([&](Operation* op) { + if (failed(CheckAncestorNotOutsideComp(op))) return WalkResult::interrupt(); + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return mlir::failure(); + return mlir::success(); +} + +// Check the validity of the module, post-pass. +LogicalResult CheckPostconditions(ModuleOp module) { + auto walk_result = module.walk([&](mlir::tf_device::ClusterOp cluster) { + if (failed(CheckClusterResults(cluster))) return WalkResult::interrupt(); + return WalkResult::advance(); + }); + if (walk_result.wasInterrupted()) return mlir::failure(); + return mlir::success(); +} + +void ExtractOutsideCompilation::runOnOperation() { + // Get runtime devices information from the closest parent module. + auto module = getOperation(); + if (failed(CheckPreconditions(module))) signalPassFailure(); + + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(module, &devices))) + return signalPassFailure(); + + llvm::SmallVector + tmp_parallel_execute_ops; + std::optional is_map_oc; + + module.walk([&](mlir::tf_device::ClusterOp device_cluster) { + if (HasOutsideCompilationNested(device_cluster.getOperation())) { + SmallVector core_to_host; + if (failed(tensorflow::GetDeviceToHostMap(device_cluster, core_to_host))) + return signalPassFailure(); + if (failed(CreateParallelExecuteForOutsideCompilation( + device_cluster, tmp_parallel_execute_ops, is_map_oc, core_to_host, + tensorflow::HasTPUDevice(devices)))) + return signalPassFailure(); + } + }); + + for (auto parallel_execute_op : tmp_parallel_execute_ops) { + parallel_execute_op.erase(); + } + // Remove `_xla_outside_compilation` attribute from all ops. These ops will + // be outside of the device cluster. The `_xla_outside_compilation` attribute + // on ops outside of tf_device.cluster don't have any meaning and can lead to + // errors later on. These ops were likely lifted out of the + // tf_device.cluster in an earlier pass. + module.walk([](Operation* op) { + op->removeAttr(kXlaOutsideCompilationAttr); + op->removeAttr(kXlaMapOutsideCompilationAttr); + }); + + if (failed(CheckPostconditions(module))) return signalPassFailure(); +} +} // namespace + +std::unique_ptr> CreateExtractOutsideCompilationPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/hoist_broadcast_read.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/hoist_broadcast_read.cc new file mode 100644 index 0000000000000000000000000000000000000000..fabeb8f6fdf228a12cebed62e4c50c5ab7be39cd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/hoist_broadcast_read.cc @@ -0,0 +1,154 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::BlockArgument; +using mlir::failure; +using mlir::LogicalResult; +using mlir::Operation; +using mlir::OperationPass; +using mlir::OpOperand; +using mlir::StringAttr; +using mlir::success; +using mlir::Value; +using mlir::WalkResult; +using mlir::func::FuncOp; +using mlir::TF::ReadVariableOp; +using mlir::tf_device::ReplicateOp; + +#define GEN_PASS_DEF_HOISTBROADCASTREADPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +constexpr char kFuncDeviceAttr[] = "tf.device"; +constexpr char kCpuDeviceType[] = "CPU"; + +struct HoistBroadcastRead + : public impl::HoistBroadcastReadPassBase { + void runOnOperation() override; +}; + +// Get the ancestor of `descendant` that is a direct child of `ancestor`. +Operation* GetAncestorBelow(Operation* descendant, Operation* ancestor) { + Operation* parent = descendant->getParentOp(); + if (!parent) return nullptr; + if (parent == ancestor) return descendant; + return GetAncestorBelow(parent, ancestor); +} + +// `is_cpu_read` is set to `true` iff `read` is on a resource with device type +// CPU. +LogicalResult IsCpuRead(FuncOp func, ReadVariableOp read, bool& is_cpu_read) { + if (auto arg = mlir::dyn_cast(read->getOperand(0))) { + if (arg.getOwner() != &(func.front())) { + is_cpu_read = false; + return success(); + } + if (auto attr = func.getArgAttrOfType(arg.getArgNumber(), + kFuncDeviceAttr)) { + std::string device = attr.getValue().str(); + tensorflow::DeviceNameUtils::ParsedName parsed_name; + if (!tensorflow::DeviceNameUtils::ParseFullName(device, &parsed_name)) { + return read->emitOpError() << "invalid device '" << device << "'"; + } + is_cpu_read = parsed_name.type == kCpuDeviceType; + return success(); + } + } + is_cpu_read = false; + return success(); +} + +// Get the reads to hoist in the `replicate`. +LogicalResult GetReads(FuncOp func, ReplicateOp replicate, + llvm::SmallVector& reads) { + for (Operation& op : replicate.getBody().front()) { + if (auto read = llvm::dyn_cast(&op)) { + bool is_cpu_read; + if (failed(IsCpuRead(func, read, is_cpu_read))) return failure(); + if (is_cpu_read) reads.push_back(read); + } + } + return success(); +} + +// Move reads above the `replicate`. Skip reads that come after a write to the +// same resource. +void MoveReads(ReplicateOp replicate, + llvm::SmallVector& reads) { + for (ReadVariableOp read : reads) { + Value res = read.getResource(); + Operation* scope = res.getParentBlock()->getParentOp(); + if (!scope->isProperAncestor(replicate)) continue; + bool has_conflicting_write = false; + for (OpOperand& use : res.getUses()) { + Operation* using_op = use.getOwner(); + if (using_op == read) continue; + if (!replicate->isProperAncestor(using_op)) continue; + Operation* peer = GetAncestorBelow(using_op, replicate); + if (read->isBeforeInBlock(peer)) continue; + if (llvm::isa(peer)) continue; + has_conflicting_write = true; + } + if (has_conflicting_write) continue; + read->moveBefore(replicate); + } +} + +// Hoist `ReadVariableOp`s above the `tf_device.replicate`s. +void HoistBroadcastRead::runOnOperation() { + FuncOp func = getOperation(); + + auto result = func.walk([&](ReplicateOp replicate) { + llvm::SmallVector reads; + if (failed(GetReads(func, replicate, reads))) + return WalkResult::interrupt(); + MoveReads(replicate, reads); + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) return signalPassFailure(); +} + +} // namespace + +std::unique_ptr> CreateHoistBroadcastReadPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..5f9611a62cbaf2f555e4dda08e85a9b11310e4c2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass.cc @@ -0,0 +1,88 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/DenseSet.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" +#include "tensorflow/core/lib/monitoring/counter.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::Operation; +using mlir::WalkResult; + +#define GEN_PASS_DEF_INPUTLOWERINGMETRICSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h.inc" + +auto* dynamism_op_counter = tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/api/v2/dynamism_op_counter", + "Counts how many ops are dynamic", "op_name"); + +auto* dynamism_function_counter = tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/api/v2/dynamism_function_counter", + "Counts how many functions are dynamic", "has_dynamism"); + +constexpr char kNotDynamicFunctionName[] = "kNotDynamicFunction"; +constexpr char kDynamicFunctionName[] = "kDynamicFunction"; + +class InputMetricsLoweringPass + : public impl::InputLoweringMetricsPassBase { + public: + void runOnOperation() override; +}; + +void InputMetricsLoweringPass::runOnOperation() { + bool has_dynamic_op = false; + Operation* func_op = getOperation(); + + func_op->walk([&](Operation* op) { + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return WalkResult::advance(); + + if (mlir::hlo::IsDynamicPadderOp(abstractOp->getTypeID())) { + has_dynamic_op = true; + dynamism_op_counter->GetCell(op->getName().getStringRef().str()) + ->IncrementBy(1); + } + + return WalkResult::advance(); + }); + + if (has_dynamic_op) { + dynamism_function_counter->GetCell(kDynamicFunctionName)->IncrementBy(1); + } else { + dynamism_function_counter->GetCell(kNotDynamicFunctionName)->IncrementBy(1); + } +} +} // namespace + +std::unique_ptr> +CreateInputLoweringMetricsPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..aa9bd1c72e4d928b7baa39c3f57abc59f624cd21 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/input_lowering_metrics_pass_test.cc @@ -0,0 +1,124 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using ::mlir::LogicalResult; +using ::mlir::ModuleOp; +using ::mlir::hlo::test::GetMlirModuleFromString; +using ::tensorflow::monitoring::testing::CellReader; + +constexpr char kNotDynamicFunctionName[] = "kNotDynamicFunction"; +constexpr char kDynamicFunctionName[] = "kDynamicFunction"; +static constexpr char kDynamismOpCounterStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/dynamism_op_counter"; +static constexpr char kDynamismFunctionCounterStreamzName[] = + "/tensorflow/core/tf2xla/api/v2/dynamism_function_counter"; + +class InputLoweringMetricsPassTest : public testing::Test { + protected: + void CreateModule(const char* module_string) { + TF_ASSERT_OK_AND_ASSIGN(module_, + GetMlirModuleFromString(module_string, &context_)); + pm_ = std::make_unique(&context_); + pm_->addNestedPass(CreateInputLoweringMetricsPass()); + } + + bool ModulesEqual(const ModuleOp& module_before, + const ModuleOp& module_after) { + return mlir::OperationEquivalence::isEquivalentTo( + module_before, module_after, mlir::OperationEquivalence::None); + } + + mlir::LogicalResult Run() { + mlir::OwningOpRef module_before = module_->clone(); + LogicalResult run_result = pm_->run(module_.get()); + + EXPECT_TRUE(ModulesEqual(*module_before, *module_)); + return run_result; + } + + private: + mlir::MLIRContext context_; + mlir::OwningOpRef module_; + std::unique_ptr pm_; +}; + +TEST_F(InputLoweringMetricsPassTest, CountsNoDynamicOps) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + return %0 : tensor<1xi32> + } + })"; + + CellReader dynamism_op_counter(kDynamismOpCounterStreamzName); + CellReader dynamism_function_counter( + kDynamismFunctionCounterStreamzName); + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + EXPECT_EQ(dynamism_function_counter.Delta(kNotDynamicFunctionName), 1); +} + +TEST_F(InputLoweringMetricsPassTest, CountsDynamicOps) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %cst0 = "tf.Const"(){ value = dense<0> : tensor<3x5xi1>} : () -> tensor<3x5xi1> + %0 = "tf.Where"(%cst0) : (tensor<3x5xi1>) -> tensor + func.return + } + })"; + + CellReader dynamism_counter(kDynamismOpCounterStreamzName); + CellReader dynamism_function_counter( + kDynamismFunctionCounterStreamzName); + + CreateModule(kMlirModuleStr); + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + EXPECT_EQ(dynamism_counter.Delta("tf.Where"), 1); + EXPECT_EQ(dynamism_function_counter.Delta(kDynamicFunctionName), 1); +} + +} // namespace +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..1712bf8cf05a0f1ecf6258642ebecc30ef7322a1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h @@ -0,0 +1,38 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_LOWERING_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_LOWERING_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Create a pass that just collects metrics about the input MLIR. Does not +// logically transform the program. +std::unique_ptr> +CreateInputLoweringMetricsPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_INPUTLOWERINGMETRICSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.h.inc" + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_LOWERING_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..e05c29d55e94aeb861520319b10c819ea93085b6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/lowering_passes.td @@ -0,0 +1,25 @@ +/* Copyright 2024 TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/Pass/PassBase.td" + +def InputLoweringMetricsPass : Pass<"input-lowering-metrics-pass", "mlir::func::FuncOp"> { + + let summary = "Collects various metrics about the input to the lowering " + "portion of the bridge. This is a logical no-op."; + + let description = [{ + Gathers metrics about the input MLIR to Phase 2 of the TFXLA Bridge, which + does a strict semantic lowering from Tensorflow ops to XLA HLO. + }]; + + let constructor = "tensorflow::tf2xla::internal::CreateInputLoweringMetricsPass()"; +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mark_ops_for_outside_compilation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mark_ops_for_outside_compilation.cc new file mode 100644 index 0000000000000000000000000000000000000000..61bfa5aedc1d041d632d17aaf1e2a09210fcdc7b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mark_ops_for_outside_compilation.cc @@ -0,0 +1,524 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/str_join.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Rewrite/PatternApplicator.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/string_util.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/core/lib/monitoring/gauge.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::Block; +using mlir::BoolAttr; +using mlir::Dialect; +using mlir::LogicalResult; +using mlir::MLIRContext; +using mlir::ModuleOp; +using mlir::Operation; +using mlir::OperationName; +using mlir::OperationPass; +using mlir::Pattern; +using mlir::PatternApplicator; +using mlir::RewritePatternSet; +using mlir::StringAttr; +using mlir::TensorType; +using mlir::Type; +using mlir::Value; +using mlir::WalkResult; + +auto* auto_outside_compilation_gauge = + tensorflow::monitoring::Gauge::New( + "/tensorflow/core/use_auto_outside_compilation", + "Tracks if auto outside compilation is enabled"); + +#define GEN_PASS_DEF_MARKOPSFOROUTSIDECOMPILATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +struct MarkOpsForOutsideCompilation + : public impl::MarkOpsForOutsideCompilationPassBase< + MarkOpsForOutsideCompilation> { + void runOnOperation() override; +}; + +// Adds any canonicalization patterns to list of supported `patterns`. +// TODO(b/161726307): Move or import the relevant patterns to LowerTF pass and +// remove this. +void AddCanonicalizationPatterns(MLIRContext* context, + RewritePatternSet* patterns) { + for (auto op : context->getRegisteredOperations()) + op.getCanonicalizationPatterns(*patterns, context); +} + +// Adds the list of ops that are supported on TPU through constant folding which +// may depend on the inputs shapes not known at this point. Such ops may not +// have any legalization or canonicalization patterns but shouldn't be marked +// for outside compilation. +// +// TODO(b/177523289): Remove manual handling once we support constant folding +// and shape inference through the computation on the host side. +void AddSupportedOpsUsingFolding(MLIRContext* context, + llvm::DenseSet* supported_ops) { + llvm::SmallDenseSet allowlist_ops = { + OperationName(mlir::TF::BroadcastArgsOp::getOperationName(), context), + OperationName(mlir::TF::BroadcastGradientArgsOp::getOperationName(), + context), + OperationName(mlir::TF::ConcatOffsetOp::getOperationName(), context), + OperationName(mlir::TF::EmptyOp::getOperationName(), context), + OperationName(mlir::TF::ListDiffOp::getOperationName(), context), + OperationName(mlir::TF::RankOp::getOperationName(), context), + OperationName(mlir::TF::RangeOp::getOperationName(), context), + OperationName(mlir::TF::ShapeOp::getOperationName(), context), + OperationName(mlir::TF::ShapeNOp::getOperationName(), context), + OperationName(mlir::TF::SizeOp::getOperationName(), context), + }; + + supported_ops->insert(allowlist_ops.begin(), allowlist_ops.end()); +} + +// Adds the list of ops that are only supported in the old bridge. +// TODO(b/168036682): Remove bounded dynamism ops now that MLIR bridge supports +// bounded dynamism. +// TODO(b/257574556): Remove the need for this manual list by making use of old +// bridge phase 2 op list. +void AddOldBridgeOnlyOps(MLIRContext* context, + llvm::DenseSet* supported_ops) { + llvm::SmallDenseSet allowlist_ops = { + OperationName(mlir::TF::DynamicPartitionOp::getOperationName(), context), + OperationName(mlir::TF::OutfeedEnqueueOp::getOperationName(), context), + OperationName(mlir::TF::WhereOp::getOperationName(), context), + OperationName(mlir::TF::UniqueOp::getOperationName(), context), + OperationName(mlir::TF::XlaSetDynamicDimensionSizeOp::getOperationName(), + context), + OperationName(mlir::TF::XlaSpmdFullToShardShapeOp::getOperationName(), + context), + OperationName(mlir::TF::XlaSpmdShardToFullShapeOp::getOperationName(), + context), + }; + + supported_ops->insert(allowlist_ops.begin(), allowlist_ops.end()); +} + +// TODO(b/159128666): Check the control flow legalization passes instead once +// added. +void AddSupportedFunctionalOps(MLIRContext* context, + llvm::DenseSet* supported_ops) { + supported_ops->insert( + OperationName(mlir::TF::CaseRegionOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::IfRegionOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::InplaceAddOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::WhileRegionOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaCallModuleOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaHostComputeOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaReduceOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaReduceWindowOp::getOperationName(), context)); + supported_ops->insert(OperationName( + mlir::TF::XlaRngBitGeneratorOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaScatterOp::getOperationName(), context)); + supported_ops->insert(OperationName( + mlir::TF::XlaSelectAndScatterOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::SymbolicGradientOp::getOperationName(), context)); + supported_ops->insert(OperationName( + mlir::TF::XlaVariadicReduceOp::getOperationName(), context)); + supported_ops->insert(OperationName( + mlir::TF::XlaVariadicReduceV2Op::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaVariadicSortOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::XlaReplicaIdOp::getOperationName(), context)); + supported_ops->insert( + OperationName(mlir::TF::YieldOp::getOperationName(), context)); +} + +// These embedding ops are rewritten when running TPUCompileOp. +void AddRewrittenEmbeddingOps(MLIRContext* context, + llvm::DenseSet* supported_ops) { + supported_ops->insert(OperationName( + mlir::TF::RecvTPUEmbeddingActivationsOp::getOperationName(), context)); + supported_ops->insert(OperationName( + mlir::TF::SendTPUEmbeddingGradientsOp::getOperationName(), context)); +} + +// Stack, TensorList and TensorArray ops are rewritten during the second phase +// of the bridge (compilation of TPUCompile op). They would not match any +// legalization/canonicalization pattern and have to be manually added to the +// list of supported ops. +void AddRewrittenCompositeOps(MLIRContext* context, + llvm::DenseSet* supported_ops) { +#define GET_OPERATION_NAME(op) OperationName(op::getOperationName(), context) + llvm::SmallDenseSet allowlist_ops = { + // Stack ops. + GET_OPERATION_NAME(mlir::TF::StackV2Op), + GET_OPERATION_NAME(mlir::TF::StackPushV2Op), + GET_OPERATION_NAME(mlir::TF::StackPopV2Op), + // Tensor Array ops. + GET_OPERATION_NAME(mlir::TF::TensorArrayV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayReadV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayWriteV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayConcatV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArraySplitV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArraySizeV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayGradV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayGatherV3Op), + GET_OPERATION_NAME(mlir::TF::TensorArrayScatterV3Op), + // Tensor List Ops. + GET_OPERATION_NAME(mlir::TF::EmptyTensorListOp), + GET_OPERATION_NAME(mlir::TF::TensorListReserveOp), + GET_OPERATION_NAME(mlir::TF::TensorListFromTensorOp), + GET_OPERATION_NAME(mlir::TF::TensorListPushBackOp), + GET_OPERATION_NAME(mlir::TF::TensorListPopBackOp), + GET_OPERATION_NAME(mlir::TF::TensorListGetItemOp), + GET_OPERATION_NAME(mlir::TF::TensorListSetItemOp), + GET_OPERATION_NAME(mlir::TF::TensorListLengthOp), + GET_OPERATION_NAME(mlir::TF::TensorListElementShapeOp), + GET_OPERATION_NAME(mlir::TF::TensorListGatherOp), + GET_OPERATION_NAME(mlir::TF::TensorListScatterIntoExistingListOp), + GET_OPERATION_NAME(mlir::TF::TensorListStackOp), + }; +#undef GET_OPERATION_NAME + + supported_ops->insert(allowlist_ops.begin(), allowlist_ops.end()); +} + +bool IsStringType(Type type) { + if (mlir::isa(type)) return true; + + auto sub_type = mlir::dyn_cast(type); + if (!sub_type) return false; + + bool has_string = llvm::any_of(sub_type.GetSubtypes(), [](TensorType type) { + return mlir::isa(type.getElementType()); + }); + return has_string; +} + +bool HasStringOperand(Operation& op) { + for (auto operand : op.getOperands()) { + auto operand_type = getElementTypeOrSelf(operand); + if (IsStringType(operand_type)) return true; + } + return false; +} + +bool HasStringResult(Operation& op) { + for (auto result : op.getResults()) { + auto result_type = getElementTypeOrSelf(result); + if (IsStringType(result_type)) return true; + } + return false; +} + +bool MatchesPattern(Operation& op, + const llvm::DenseSet& supported_ops) { + return (supported_ops.contains(op.getName())); +} + +// Checks if the op is supported inside of a device cluster. Ops not +// in `tf_dialect` are considered supported. +bool IsSupportedOp(Operation& op, + const llvm::DenseSet& supported_ops, + const Dialect* tf_dialect) { + if (op.getDialect() != tf_dialect) return true; + // Assert has a legalization that later removes it so we don't want to outside + // compile it ever for performance reasons. + if (llvm::isa(op)) return true; + + if (HasStringOperand(op)) return false; + if (HasStringResult(op)) return false; + if (MatchesPattern(op, supported_ops)) return true; + + auto abstractOp = op.getRegisteredInfo(); + if (!abstractOp) return false; + return mlir::hlo::HasTf2XlaFallback(abstractOp->getTypeID()); +} + +bool IsVariant(Value value) { + return mlir::isa( + getElementTypeOrSelf(value.getType())); +} + +bool HasOutsideCompiledAncestor(Operation* op) { + Operation* parent = op->getParentOp(); + while (parent) { + if (parent->getAttrOfType(kXlaOutsideCompilationAttr)) + return true; + parent = parent->getParentOp(); + } + return false; +} + +// If any tf.variants are inputs/outputs to the another outside compiled +// Operation, `op`, mark them for outside compilation unless they are already +// marks with outside compilation attribute. +void MarkVariantInputsOutputs(mlir::tf_device::ClusterOp tpu_cluster) { + std::queue outside_compiled_ops; + tpu_cluster.walk([&](Operation* op) { + if (op->hasAttrOfType(kXlaOutsideCompilationAttr)) + outside_compiled_ops.push(op); + }); + + while (!outside_compiled_ops.empty()) { + Operation* host_op = outside_compiled_ops.front(); + outside_compiled_ops.pop(); + host_op->walk([&](Operation* op) { + // Add any operations that provide variant inputs to the cluster. + for (auto value : op->getOperands()) { + Operation* input_defining_op = value.getDefiningOp(); + if (IsVariant(value) && input_defining_op && + !HasOutsideCompiledAncestor(input_defining_op) && + !input_defining_op->hasAttrOfType( + kXlaOutsideCompilationAttr)) { + input_defining_op->setAttr( + kXlaOutsideCompilationAttr, + StringAttr::get(input_defining_op->getContext(), "auto")); + outside_compiled_ops.push(input_defining_op); + } + } + // Mark for outside compilation any operations that consume variant + // outputs from an outside compiled operation. + for (auto value : op->getResults()) { + if (IsVariant(value)) { + for (auto user : value.getUsers()) { + if (!user->hasTrait() && + !HasOutsideCompiledAncestor(user) && + !user->getAttrOfType(kXlaOutsideCompilationAttr)) { + user->setAttr(kXlaOutsideCompilationAttr, + StringAttr::get(user->getContext(), "auto")); + outside_compiled_ops.push(user); + } + } + } + } + }); + } +} + +// Marks uncompilable ops that are in `tf_dialect` for outside compilation. +LogicalResult MarkUncompilableOps( + const Dialect* tf_dialect, Block* block, + llvm::DenseSet& supported_ops) { + // Automatically marked ops for outside compilation have + // `_xla_outside_compilation` attribute value of "auto" plus + // an increasing counter. Manually marked ops for outside compilation only + // have an increasing counteri for the attribute value. Therefore there is no + // collision in + // `_xla_outside_compilation` attribute between automatically and manually + // marking ops. + int outside_compiled_cluster_counter = 0; + block->walk([&](Operation* op) { + if (!IsSupportedOp(*op, supported_ops, tf_dialect)) { + VLOG(3) << "Cloud TPU: Op " << op->getName().getStringRef().str() + << " isn't compilable, adding outside_compilation attr. " + "This op will automatically be placed on CPU."; + op->setAttr(kXlaOutsideCompilationAttr, + StringAttr::get( + op->getContext(), + llvm::formatv("auto{0}", outside_compiled_cluster_counter) + .str())); + outside_compiled_cluster_counter++; + } + }); + if (outside_compiled_cluster_counter > 0) { + auto_outside_compilation_gauge->GetCell()->Set(true); + } + return mlir::success(); +} + +// Check for uncompilable ops that are in `tf_dialect` and are not already +// marked for outside compilation. +bool ContainsUncompilableOps(const Dialect* tf_dialect, Block* block, + llvm::DenseSet& supported_ops) { + int uncompilable_op_count = 0; + // Check if op or any parent is already marked for outside compilation. + block->walk([&](Operation* op) { + Operation* iter_op = op; + while (iter_op && !llvm::isa(iter_op)) { + if (iter_op->hasAttrOfType(kXlaOutsideCompilationAttr)) { + return; + } + iter_op = iter_op->getParentOp(); + } + + if (!IsSupportedOp(*op, supported_ops, tf_dialect)) { + op->emitOpError() << "isn't compilable for TPU device. enable " + "soft_device_placement option to run on CPU"; + ++uncompilable_op_count; + } + }); + return uncompilable_op_count > 0; +} + +// Unmarks outside compilation for any op that has parents already +// marked for outside compilation since the child will be extracted +// anyways. +void UnmarkChildren(ModuleOp module) { + module->walk([&](Operation* op) { + if (!op->getAttrOfType(kXlaOutsideCompilationAttr)) return; + Operation* iter_op = op; + bool remove_attr = false; + while (auto* parent_op = iter_op->getParentOp()) { + if (parent_op->getAttrOfType(kXlaOutsideCompilationAttr)) { + remove_attr = true; + break; + } + iter_op = parent_op; + } + if (remove_attr) op->removeAttr(kXlaOutsideCompilationAttr); + }); +} + +constexpr int kTooManyOutsideCompileRegionThreshold = 32; +constexpr int kOpDetailCount = 8; + +void WarnOnExcessOutsideCompilationOps(ModuleOp module) { + // Count the number of outside compilation ops. If it exceeds the reporting + // threshold, warn the user that their model may run slowly. + llvm::SmallVector outside_compile_ops; + module->walk([&](Operation* op) { + if (op->getAttrOfType(kXlaOutsideCompilationAttr)) { + outside_compile_ops.push_back(op); + } + }); + + if (outside_compile_ops.size() > kTooManyOutsideCompileRegionThreshold) { + llvm::SmallVector op_info; + for (int i = 0; i < kOpDetailCount; ++i) { + auto& op = outside_compile_ops[i]; + op_info.push_back(tensorflow::OpAsString(*op)); + } + + LOG(WARNING) << outside_compile_ops.size() << " outside compilation " + << "regions found while processing " + << module->getName().getStringRef().str() + << ". This may result in excessively slow model execution. " + << "First " << op_info.size() + << " ops: " << absl::StrJoin(op_info, "\n"); + } else { + LOG(INFO) << "Found " << outside_compile_ops.size() + << " outside compilation regions."; + } +} + +void MarkOpsForOutsideCompilation::runOnOperation() { + auto module = getOperation(); + const Dialect* tf_dialect = getContext().getLoadedDialect("tf"); + if (!tf_dialect) { + getOperation().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); + } + RewritePatternSet patterns(&getContext()); + mlir::hlo::PopulateLegalizeTfPatterns(module.getContext(), &patterns); + mlir::TF::PopulateTFLoweringBeforeHLOPatterns(module.getContext(), &patterns); + mlir::TF::PopulateLoweringQuantizedPatterns(module.getContext(), &patterns); + AddCanonicalizationPatterns(module.getContext(), &patterns); + + // `supported_ops` contains the name of all of the ops that can potentially be + // lowered into HLO on the device. This doesn't always mean that the op can + // be lowered in the future passes but if the op is not in this set, it can't + // be lowered in a subsequent pass. + llvm::DenseSet supported_ops; + PatternApplicator(std::move(patterns)) + .walkAllPatterns([&](const Pattern& pattern) { + std::optional root_kind = pattern.getRootKind(); + if (root_kind.has_value()) supported_ops.insert(root_kind.value()); + }); + AddSupportedFunctionalOps(module.getContext(), &supported_ops); + AddSupportedOpsUsingFolding(module.getContext(), &supported_ops); + AddOldBridgeOnlyOps(module.getContext(), &supported_ops); + AddRewrittenEmbeddingOps(module.getContext(), &supported_ops); + AddRewrittenCompositeOps(module.getContext(), &supported_ops); + + auto result = module.walk([&](mlir::tf_device::ClusterOp cluster) { + // Only if `allow_soft_placement` attribute is true should we mark ops + // for outside compilation. + auto soft_placement_attr = + cluster->getAttrOfType(mlir::TF::kAllowSoftPlacementAttr); + if ((soft_placement_attr && soft_placement_attr.getValue())) { + if (failed(MarkUncompilableOps(tf_dialect, &cluster.GetBody(), + supported_ops))) + return WalkResult::interrupt(); + } else { + if (ContainsUncompilableOps(tf_dialect, &cluster.GetBody(), + supported_ops)) + return WalkResult::interrupt(); + } + MarkVariantInputsOutputs(cluster); + + return WalkResult::advance(); + }); + + if (result.wasInterrupted()) return signalPassFailure(); + + UnmarkChildren(module); + + WarnOnExcessOutsideCompilationOps(module); +} + +} // namespace + +std::unique_ptr> +CreateMarkOpsForOutsideCompilationPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..582ee6c1354917af24802a78e460db0ee5de8c65 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h @@ -0,0 +1,35 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_MLIR_TO_GRAPH_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_MLIR_TO_GRAPH_PASSES_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Verifies that Executor input is of the expected format. +std::unique_ptr> +CreateVerifyInputDialectToExecutorPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_VERIFYINPUTDIALECTTOEXECUTORPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h.inc" + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_MLIR_TO_GRAPH_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..eb029a2f4bafcaf78c4e482d3b3250f07d93d746 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.td @@ -0,0 +1,20 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/Pass/PassBase.td" + +def VerifyInputDialectToExecutorPass : Pass<"verify-input-dialect-to-executor-pass", "mlir::func::FuncOp"> { + let summary = "Verify that TF dialect to executor converter receives the correct input."; + let description = [{ + Verifies the input before exporting to TF executor. This includes checking whether the Ops are in TF functional, have device attributes & there are no tf_device.cluster_func ops. + }]; + let constructor = "tensorflow::tf2xla::internal::CreateVerifyInputDialectToExecutorPass()"; +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_cluster_formation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_cluster_formation.cc new file mode 100644 index 0000000000000000000000000000000000000000..fc2ca3dd197f459e484267142231ec872b22e21b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_cluster_formation.cc @@ -0,0 +1,977 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/string_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::Block; +using mlir::DialectRegistry; +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::NamedAttribute; +using mlir::NamedAttrList; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OpResult; +using mlir::Region; +using mlir::StringAttr; +using mlir::success; +using mlir::Type; +using mlir::Value; +using mlir::ValueRange; +using mlir::WalkResult; + +constexpr llvm::StringRef kDeviceAttr = "device"; +constexpr llvm::StringRef kNameAttr = "name"; +constexpr llvm::StringRef kNumCoresPerReplicaAttr = "num_cores_per_replica"; +constexpr llvm::StringRef kNumReplicasAttr = "num_replicas"; +constexpr llvm::StringRef kMirroredVariableIndicesAttr = + "_mirrored_variable_indices"; + +constexpr llvm::StringRef kBadReplicateInfoAttrMsg = + "requires '_replication_info' string attribute"; + +// Mapping for `_replication_info` attribute to TPUReplicateMetadata attributes. +using MetadataMap = llvm::SmallDenseMap; + +// A set of operations. We use a `SmallSetVector` in order to have deterministic +// traversal order (= insertion order), independent of the pointer keys. +using OpSetVector = llvm::SmallSetVector; + +// Mapping for `_replication_info` attribute to ops of a cluster. +using ClusterMap = llvm::SmallDenseMap; + +#define GEN_PASS_DEF_TPUCLUSTERFORMATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +class TPUClusterFormationPass + : public impl::TPUClusterFormationPassBase { + public: + explicit TPUClusterFormationPass(bool strict_clusters) + : strict_clusters_(strict_clusters) {} + + void getDependentDialects(DialectRegistry& registry) const override { + registry.insert(); + } + + void runOnOperation() override; + + private: + bool strict_clusters_; +}; + +// Creates a mapping from the TPUReplicateMetadata ops `_replication_info` +// attribute to its attributes and removes the ops. If multiple +// TPUReplicateMetadata ops have the same `_replication_info` attribute, an +// error will be returned. +LogicalResult CollectMetadata(Block* block, MetadataMap* metadata_map) { + // Just look at top-level operations in the block (not nested ones) + for (Operation& op : llvm::make_early_inc_range(*block)) { + auto metadata_op = llvm::dyn_cast(op); + if (!metadata_op) continue; + + NamedAttrList attrs(metadata_op->getAttrDictionary()); + + // Missing or bad `_replication_info` attribute. + auto replication_info_attr = attrs.get(mlir::TF::kReplicationInfoAttr); + if (!replication_info_attr) + return metadata_op.emitError() << kBadReplicateInfoAttrMsg; + + auto replication_info_attr_str = + mlir::dyn_cast(replication_info_attr); + if (!replication_info_attr_str || + replication_info_attr_str.getValue().empty()) + return metadata_op.emitError() << kBadReplicateInfoAttrMsg; + + // Remove `name` attribute. + attrs.erase(StringAttr::get(metadata_op.getContext(), kNameAttr)); + + auto it = metadata_map->try_emplace(replication_info_attr_str.getValue(), + std::move(attrs)); + + // There are multiple TPUReplicateMetadata ops with the same + // `_replication_info` attribute. + if (!it.second) { + return metadata_op.emitError() + << "multiple TPUReplicateMetadata ops with the same '" + << mlir::TF::kReplicationInfoAttr << "' attribute '" + << replication_info_attr_str.getValue() << "' found"; + } + metadata_op.erase(); + } + return success(); +} + +struct OpDevice { + Operation* op; + std::string device; +}; + +LogicalResult HasValidDeviceTypeAttribute(Block* block) { + // Use ordered set here to make error message below deterministic. + std::set device_types; + for (Operation& op : *block) { + // Collect device types which currently must be consistent per block + // (checked later). + if (auto device_type_attr = + op.getAttrOfType(mlir::TF::kCompileDeviceTypeAttr)) { + // tf.StatefulPartitionedCall ops with and without + // _tpu_replicate attributes may exist in the same graph. Ops without + // the attribute but with _XlaMustCompile=true would have + // _xla_compile_device_type="" after + // CanonicalizeCompileAndReplicateAttributesPass. Skip empty value here. + if (!device_type_attr.getValue().empty()) { + device_types.insert(device_type_attr); + } + } + } + + if (device_types.size() > 1) { + return block->getParentOp()->emitError() + << "found different '" << mlir::TF::kCompileDeviceTypeAttr + << "' attribute values (" << llvm::join(device_types, ",") + << ") in same block which is not supported"; + } + return success(); +} + +// Collects and clusters ops based on `_replication_info` attribute. Returns +// an error in case of invalid compilation or replication attribute(s). +LogicalResult CollectAndGroupClusterOps(Block* block, ClusterMap* clusters) { + LogicalResult result = HasValidDeviceTypeAttribute(block); + if (failed(result)) return result; + + for (Operation& op : *block) { + LogicalResult result = + mlir::TF::HasValidCompilationAndReplicationAttributes(op); + if (failed(result)) return result; + + // Skip ops with non-TPU device type, they are handled elsewhere. + auto device_type_attr = + op.getAttrOfType(mlir::TF::kCompileDeviceTypeAttr); + if (device_type_attr) { + if (device_type_attr.getValue().empty()) continue; + if (device_type_attr.getValue() != mlir::TF::kTpuDevice) continue; + } + + if (op.hasAttr(mlir::TF::kReplicationInfoAttr)) { + // For replicated case, borrow cluster structure from replication info. + // Following condition is already checked in + // `HasValidCompilationAndReplicationAttributes` above, assert here for + // documentation and to avoid breakage when that function is changed. + assert(op.hasAttr(mlir::TF::kCompileDeviceTypeAttr)); + auto attr = op.getAttrOfType(mlir::TF::kReplicationInfoAttr); + auto it = clusters->try_emplace(attr.getValue()); + it.first->getSecond().insert(&op); + } + } + return success(); +} + +// Returns true iff `op` has a direct control dependency from (`incoming` == +// true) or to (`incoming` == false) any op in `cluster_ops` or +// `cluster_dependent_ops`. +Operation* getOpClusterControlDependency( + Operation* op, bool incoming, const OpSetVector& cluster_ops, + const OpSetVector& cluster_dependent_ops, + const mlir::TF::SideEffectAnalysis::Info& side_effect_analysis) { + auto filter = [&](Operation* other_op) { + return cluster_ops.contains(other_op) || + cluster_dependent_ops.contains(other_op); + }; + auto other_ops = + incoming ? side_effect_analysis.DirectControlPredecessors(op, filter) + : side_effect_analysis.DirectControlSuccessors(op, filter); + if (other_ops.empty()) + return nullptr; + else + return *other_ops.begin(); +} + +// Returns true iff `op` has a direct data dependency from (`incoming` == true +// or to (`incoming` == false) any op in `cluster_ops` or +// `cluster_dependent_ops`. +Operation* getOpClusterDataDependency( + Operation* op, bool incoming, const OpSetVector& cluster_ops, + const OpSetVector& cluster_dependent_ops) { + Operation* other_op = nullptr; + op->walk([&](Operation* inner_op) { + ValueRange values = incoming ? ValueRange(inner_op->getOperands()) + : ValueRange(inner_op->getResults()); + llvm::SmallVector candidates; + for (Value value : values) { + if (incoming) { + candidates = {value.getDefiningOp()}; + } else { + candidates.assign(value.getUsers().begin(), value.getUsers().end()); + } + for (Operation* candidate_op : candidates) { + if (cluster_ops.contains(candidate_op) || + cluster_dependent_ops.contains(candidate_op)) { + other_op = candidate_op; + return WalkResult::interrupt(); + } + } + } + return WalkResult::advance(); + }); + return other_op; +} + +// Collects ops that need to be moved behind the cluster due to data or control +// dependencies. +mlir::FailureOr> CollectClusterSuccessorOps( + Block* block, const OpSetVector& cluster_ops, + const mlir::TF::SideEffectAnalysis::Info& side_effect_analysis, + bool strict_clusters) { + OpSetVector cluster_predecessor_ops; + OpSetVector cluster_successor_ops; + + // Collect non-cluster ops that have a dependency to the cluster. For this + // traverse all ops from last to first cluster op and keep track of in-between + // non-cluster ops that have some outgoing (transitive) dependency to some + // cluster op (`cluster_predecessor_ops`). + auto rfront = Block::reverse_iterator(cluster_ops.front()); + auto rback = Block::reverse_iterator(cluster_ops.back()); + for (Operation& op : llvm::make_range(rback, rfront)) { + if (cluster_ops.contains(&op)) continue; + bool has_dependency_to_cluster = + getOpClusterDataDependency(&op, /*incoming=*/false, cluster_ops, + cluster_predecessor_ops) != nullptr || + getOpClusterControlDependency(&op, /*incoming=*/false, cluster_ops, + cluster_predecessor_ops, + side_effect_analysis) != nullptr; + if (has_dependency_to_cluster) cluster_predecessor_ops.insert(&op); + } + // Collect non-cluster ops that have a dependency from the cluster. For this + // traverse all ops from first to last cluster op and keep track of in-between + // non-cluster ops that have some incoming (transitive) dependency from some + // cluster op (`cluster_successor_ops`). + auto front = Block::iterator(cluster_ops.front()); + auto back = Block::iterator(cluster_ops.back()); + for (Operation& op : llvm::make_range(front, back)) { + if (cluster_ops.contains(&op)) continue; + Operation* data_predecessor = getOpClusterDataDependency( + &op, /*incoming=*/true, cluster_ops, cluster_successor_ops); + Operation* control_predecessor = getOpClusterControlDependency( + &op, /*incoming=*/true, cluster_ops, cluster_successor_ops, + side_effect_analysis); + if (data_predecessor || control_predecessor) { + if (cluster_predecessor_ops.contains(&op)) { + // Op has a dependency from and to the cluster which is invalid. Instead + // of erroring out we don't add the op to `cluster_successor_ops` which + // is in line with previous behavior when certain control dependencies + // were not considered. + // TODO(b/216706460) Establish some contract here: Should we expect only + // valid clusters, or should we split clusters accordingly? The latter + // might have runtime impact for existing models. + // We should make this message an error once there is such a contract + // and once existing cases have been fixed. + mlir::InFlightDiagnostic error = + strict_clusters ? mlir::emitError(op.getLoc(), "") + : mlir::emitWarning(op.getLoc(), ""); + error << "Op has cyclic dependency with a compilation cluster:\n"; + error << "The cluster depends on\n"; + error << op.getName() << "\n" + << "which is outside of cluster, but itself depends "; + if (data_predecessor) { + error << "on\n" << data_predecessor->getName() << "\n"; + } else if (control_predecessor) { + error << "(via control) on\n" + << control_predecessor->getName() << "\n"; + } + if (cluster_ops.contains(data_predecessor) || + cluster_ops.contains(control_predecessor)) + error << "which is inside the cluster.\n"; + else + error << "which the cluster depends on.\n"; + } else { + cluster_successor_ops.insert(&op); + } + } + } + return cluster_successor_ops; +} + +// Collects results and associated types of the cluster that are used outside of +// the cluster. These results and types are used to create the clusters +// `tf_device.cluster` and associated terminator. Results that have no uses +// outside of the cluster (i.e. results of ops in the cluster are only consumed +// by other ops in the cluster) are pruned. +llvm::SmallVector CollectClusterResults( + Block* block, const OpSetVector& cluster_ops) { + llvm::SmallVector results; + + for (Operation* op : cluster_ops) { + for (Value result : op->getResults()) { + for (Operation* user : result.getUsers()) { + // Check if user is not an op in the cluster. + if (cluster_ops.count(block->findAncestorOpInBlock(*user)) == 0) { + results.push_back(result); + break; + } + } + } + } + + return results; +} + +// Creates a `tf_device.cluster` to wrap cluster ops. +mlir::tf_device::ClusterOp CreateClusterOp( + Block* block, const OpSetVector& cluster_ops, llvm::ArrayRef results, + llvm::ArrayRef cluster_successor_ops) { + // `tf_device.cluster` will be placed at where the last op of the cluster is. + Operation* last_cluster_op = cluster_ops.back(); + OpBuilder builder(last_cluster_op); + + llvm::SmallVector result_types; + for (Value result : results) result_types.push_back(result.getType()); + auto cluster = mlir::tf_device::ClusterOp::create( + builder, last_cluster_op->getLoc(), result_types); + + Block* body = new Block; + cluster.getBody().push_back(body); + + // Move cluster ops to the cluster body. Also remove `_replication_info` and + // `device` attribute from ops in the cluster when that information is + // redundant will the `tf_device.cluster`. Do this for all ops including + // nested ops. + for (Operation* cluster_op : cluster_ops) { + cluster_op->moveBefore(body, body->end()); + cluster_op->walk([&](Operation* inner_op) { + inner_op->removeAttr(mlir::TF::kReplicationInfoAttr); + inner_op->removeAttr(mlir::TF::kCompileDeviceTypeAttr); + + if (auto attr = inner_op->getAttrOfType(kDeviceAttr)) { + // Preserve device attribute if the op is placed on a replicated core + // device. Device attribute is used to infer the appropriate sharding + // within TPUs for this op. + // TODO(b/183598857): Use explicit sharding ops from the front-end. + // For example, dequeue ops generated by + // tensorflow/python/tpu/tpu_feed.py + if (!tensorflow::IsTPUReplicatedCore(attr.getValue())) { + inner_op->removeAttr(kDeviceAttr); + } + } + }); + } + + // Add terminator. + builder.setInsertionPointToEnd(body); + mlir::tf_device::ReturnOp::create(builder, last_cluster_op->getLoc(), + results); + + // Replaces uses of cluster ops results outside of cluster with the associated + // `tf_device.cluster` results. + for (auto ret_vals : llvm::zip(results, cluster.getResults())) { + Value old_ret = std::get<0>(ret_vals); + Value new_ret = std::get<1>(ret_vals); + for (auto& use : llvm::make_early_inc_range(old_ret.getUses())) { + Operation* user = use.getOwner(); + if (!body->findAncestorOpInBlock(*user)) use.set(new_ret); + } + } + + // Move ops that depend on something in the cluster behind the cluster. + Operation* op_after_cluster = cluster.getOperation()->getNextNode(); + for (Operation* op : cluster_successor_ops) op->moveBefore(op_after_cluster); + return cluster; +} + +// Returns an op of the given type that uses the result, along with +// a list of identity ops along the way. +template +std::tuple> GetSingleUserOfType( + OpResult result) { + llvm::SmallVector identity_ops; + + do { + Operation* user = result.hasOneUse() ? *result.getUsers().begin() : nullptr; + if (auto t = llvm::dyn_cast_or_null(user)) { + return std::make_tuple(t, identity_ops); + } else if (auto identity = + llvm::dyn_cast_or_null(user)) { + identity_ops.emplace_back(identity); + result = identity->getResult(0); + } else { + result = OpResult(); // reset to stop iterating + } + } while (result); + + return std::make_tuple(T(), identity_ops); +} + +using PartitionedClusterOutputMap = absl::flat_hash_map< + uint64_t, llvm::SmallVector>; + +// Returns the partitioned output ops from the cluster if there are any, +// along with any single user identity ops between them. Not all outputs +// of a cluster must be partitioned, so the output is a map from cluster +// output ids to ops. +std::tuple> +GetPartitionedOutputsAndIdentityOps(mlir::tf_device::ClusterOp cluster) { + PartitionedClusterOutputMap partitioned_outputs; + llvm::SmallVector erase_list; + + for (auto [cluster_result_id, cluster_result] : + llvm::enumerate(cluster.getResults())) { + auto [replicated_output, _] = + GetSingleUserOfType(cluster_result); + if (replicated_output) { + for (OpResult per_replica_result : replicated_output->getResults()) { + auto [partitioned_output, id_ops] = + GetSingleUserOfType( + per_replica_result); + if (partitioned_output) { + erase_list.insert(erase_list.end(), id_ops.begin(), id_ops.end()); + partitioned_outputs[cluster_result_id].emplace_back( + partitioned_output); + } + } + } + } + + return std::forward_as_tuple(partitioned_outputs, erase_list); +} + +// Inlines the partitioned output ops into the cluster, and updates +// their users to point to the replicate op instead. +Operation* BuildPartitionedOutputs( + OpBuilder& builder, mlir::tf_device::ClusterOp cluster, + mlir::tf_device::ReplicateOp replicate_op, + PartitionedClusterOutputMap& partitioned_outputs, + llvm::SmallVector& erase_list, + llvm::SmallVector& result_types, int num_replicas) { + Operation* result_op; + llvm::SmallVector results; + uint64_t num_results = cluster.getNumResults(); + for (uint64_t result_id = 0; result_id < num_results; ++result_id) { + auto search = partitioned_outputs.find(result_id); + if (search == partitioned_outputs.end()) { + // If the output is not partitioned, directly pass it through. + results.emplace_back(cluster.getResult(result_id)); + + continue; + } + + // Otherwise, "inline" the partitioned output ops by: + // - Building a new op within the cluster. + // - Replacing all the uses of the original ops with the cluster's outputs. + llvm::SmallVector& ops = + search->second; + for (auto [replica_id, partitioned_output] : llvm::enumerate(ops)) { + for (auto [core_id, result] : + llvm::enumerate(partitioned_output->getResults())) { + // outputs from replicate op are interleaved: + // [(replica:0,core:0), (replica:1,core:0), ..., + // (replica:0,core:1), (replica:1,core:1), ...] + uint64_t output_id = + core_id * num_replicas + replica_id + results.size(); + result.replaceAllUsesWith(replicate_op.getResult(output_id)); + } + } + + // Assume all the replicas have the same structure. + mlir::TF::TPUPartitionedOutputV2Op first_op = *(ops.begin()); + mlir::ArrayAttr dims = first_op.getPartitionDimsAttr(); + StringAttr sharding = first_op.get_XlaShardingAttr(); + StringAttr sharding_v2 = first_op.get_XlaShardingV2Attr(); + Operation::result_type_range output_types = first_op.getResultTypes(); + result_op = mlir::TF::TPUPartitionedOutputV2Op::create( + builder, replicate_op.getLoc(), output_types, + cluster.getResult(result_id), dims, sharding, sharding_v2); + + results.insert(results.end(), result_op->getResults().begin(), + result_op->getResults().end()); + } + + // Once we've accumulated all the cluster's results, build a return op. + mlir::tf_device::ReturnOp::create(builder, result_op->getLoc(), results); + + // Then erase all the identity and partitioned output ops. + for (const auto& [_, ops] : partitioned_outputs) { + for (mlir::TF::TPUPartitionedOutputV2Op op : ops) { + op->erase(); + } + } + + for (mlir::TF::IdentityOp to_erase : erase_list) { + to_erase->erase(); + } + + return result_op; +} + +// Return the cluster's per-replica result type, converting any full-shaped +// tensor types into sharded-shaped ones if they're partitioned. +llvm::SmallVector GetClusterResultTypes( + mlir::tf_device::ClusterOp cluster, + const PartitionedClusterOutputMap& partitioned_outputs) { + llvm::SmallVector result_types; + Operation::result_type_range cluster_result_types = cluster.getResultTypes(); + if (partitioned_outputs.empty()) { + // Directly pass through the cluster's outputs if none are partitioned. + result_types.insert(result_types.end(), cluster_result_types.begin(), + cluster_result_types.end()); + } else { + // For each output of the cluster... + for (auto [output_id, result_type] : + llvm::enumerate(cluster_result_types)) { + auto search = partitioned_outputs.find(output_id); + if (search == std::end(partitioned_outputs)) { + // If it's not partitioned, directly pass it through. + result_types.emplace_back(result_type); + } else { + // Otherwise, pass through the result shard types. + Operation::result_type_range partitioned_result_types = + (*search->second.begin())->getResultTypes(); + result_types.insert(result_types.end(), + partitioned_result_types.begin(), + partitioned_result_types.end()); + } + } + } + return result_types; +} + +// Creates a `tf_device.replicate` to represent replication for the cluster, if +// necessary. Erases Identity ops between partitioned and replicated output ops. +LogicalResult ReplicateCluster(mlir::tf_device::ClusterOp cluster, + int num_replicas, int num_cores_per_replica) { + OpBuilder builder(cluster); + auto [partitioned_outputs, erase_list] = + GetPartitionedOutputsAndIdentityOps(cluster); + + for (auto [_, ops] : partitioned_outputs) { + if (!(ops.empty() || ops.size() == num_replicas)) { + return (ops.begin())->emitOpError() + << "expected zero or " << num_replicas + << " 'TPUPartitionedOutput' op(s), instead got " + << partitioned_outputs.size(); + } + } + + // No need to replicate. + if (num_replicas == 1) { + // Collapse all the Identity ops between the TRO and TPO ops. + if (!partitioned_outputs.empty()) { + for (mlir::TF::IdentityOp to_erase : erase_list) { + Value in = to_erase->getOperand(0); + OpResult out = to_erase->getResult(0); + out.replaceAllUsesWith(in); + to_erase->erase(); + } + } + + return success(); + } + + if (num_replicas < 1) + return cluster.emitError() << "requires '" << kNumReplicasAttr + << "' int attribute to be at least 1"; + + LogicalResult status = success(); + // Collect all used TPUReplicatedInput ops. + llvm::SmallVector replicated_input_ops; + llvm::SmallSet seen_ops; + mlir::visitUsedValuesDefinedAbove( + cluster.getBody(), cluster.getBody(), [&](mlir::OpOperand* operand) { + Operation* def = operand->get().getDefiningOp(); + if (auto ri = + llvm::dyn_cast_or_null(def)) { + if (!seen_ops.contains(ri)) { + seen_ops.insert(ri); + replicated_input_ops.push_back(ri); + } + } + // When model parallelism is used in conjunction with data parallelism + // for resource inputs, we need to collect the per replica resource + // inputs from input to `tf.TPUPartitionedInputV2` ops. + if (auto pi = llvm::dyn_cast_or_null( + def)) { + if (pi->getNumOperands() != num_cores_per_replica) + status = pi.emitOpError() + << "requires " << num_cores_per_replica + << " operands but found " << pi->getNumOperands(); + for (auto operand : pi.getInputs()) { + if (auto ri = + llvm::dyn_cast_or_null( + operand.getDefiningOp())) { + if (!seen_ops.contains(ri)) { + seen_ops.insert(ri); + replicated_input_ops.push_back(ri); + } + } + } + } + }); + + if (failed(status)) return mlir::failure(); + + // Indices of the replicate op's arguments that are mirrored variables. + llvm::SmallVector mirrored_variable_indices; + + // Check if number of operands of each used TPUReplicatedInput op matches + // `num_replicas` or 1. Collect all their operands and associated type for + // creating the replicate op. + llvm::SmallVector, 8> replicated_inputs; + llvm::SmallVector packed_inputs; + llvm::SmallVector replicated_ops; + llvm::SmallVector packed_ops; + for (const auto& pos_and_input : llvm::enumerate(replicated_input_ops)) { + auto input = pos_and_input.value(); + bool is_packed = input.getIsPacked(); + const int num_operands = input->getNumOperands(); + int num_inputs = is_packed ? 1 : num_replicas; + if (num_operands != num_inputs) + return input->emitOpError() << "requires " << num_inputs << " operands"; + if (is_packed) { + packed_inputs.push_back(input->getOperand(0)); + packed_ops.push_back(input); + } else { + replicated_inputs.push_back( + {input->getOperands(), input->getOperand(0).getType()}); + replicated_ops.push_back(input); + } + } + + // Create `ordered_tpu_replicate_inputs` which contains the final ordered + // replicate inputs. All packed arguments are moved to the end of the arg + // list. + llvm::SmallVector + ordered_tpu_replicate_inputs = replicated_ops; + ordered_tpu_replicate_inputs.append(packed_ops.begin(), packed_ops.end()); + + // Assign `mirrored_variable_indices` based on the ordered replicated inputs. + for (const auto& pos_and_input : + llvm::enumerate(ordered_tpu_replicate_inputs)) { + auto tpu_replicated_input = pos_and_input.value(); + if (tpu_replicated_input.getIsMirroredVariable()) { + mirrored_variable_indices.push_back(pos_and_input.index()); + } + } + + // Create replicate op. + auto result_types = GetClusterResultTypes(cluster, partitioned_outputs); + auto replicate_op = mlir::tf_device::ReplicateOp::create( + builder, cluster.getLoc(), num_replicas, + llvm::SmallDenseMap>(), + replicated_inputs, packed_inputs, result_types); + + if (!mirrored_variable_indices.empty()) + replicate_op->setAttr(kMirroredVariableIndicesAttr, + builder.getI64ArrayAttr(mirrored_variable_indices)); + + // Replace replicated cluster results with replicate op results. + uint64_t offset = 0; + for (auto [idx, result] : llvm::enumerate(cluster.getResults())) { + if (partitioned_outputs.contains(idx)) { + // Partitioned output propagation happens in BuildPartitionedOutputs. + offset += num_replicas * num_cores_per_replica; + continue; + } + + auto replicate_outputs = llvm::make_range( + std::next(replicate_op.result_begin(), offset), + std::next(replicate_op.result_begin(), offset + num_replicas)); + for (auto& use : llvm::make_early_inc_range(result.getUses())) { + Operation* def = use.getOwner(); + if (!llvm::isa(def)) { + // If user is not a `tf.TPUReplicatedOutput`, simply forward the first + // replica output. Certain Graphs under V1 create `tf.Identity` users of + // replicated ops to pin the TPU computation for execution. + use.set(*replicate_outputs.begin()); + continue; + } + + const int def_num_results = def->getNumResults(); + if (def_num_results != num_replicas) + return def->emitOpError() << "requires " << num_replicas << " results"; + + def->replaceAllUsesWith(replicate_outputs); + } + + offset += num_replicas; + } + + // Collect all `tf.TPUPartitionedInputV2` ops to be moved inside the + // `tf_device.replicate` later. + llvm::SmallSet partitioned_inputs; + for (auto input_and_block_arg : + llvm::zip(ordered_tpu_replicate_inputs, + replicate_op.GetBody().getArguments())) { + mlir::TF::TPUReplicatedInputOp input = std::get<0>(input_and_block_arg); + Value block_arg = std::get<1>(input_and_block_arg); + mlir::replaceAllUsesInRegionWith(input->getResult(0), block_arg, + cluster.getBody()); + // Update replicated input use in tf.TPUPartitionedInputV2 op. + for (auto& use : input->getUses()) { + auto pi = + llvm::dyn_cast(use.getOwner()); + if (pi) { + pi.setOperand(use.getOperandNumber(), block_arg); + partitioned_inputs.insert(pi.getOperation()); + } + } + } + + // Create terminator for replicate op and move `tf_device.cluster` and + // `tf.TPUPartitionedInputV2`(s) into replicate body. + builder.setInsertionPointToEnd(&replicate_op.GetBody()); + + Operation* result_op; + if (!partitioned_outputs.empty()) { + result_op = BuildPartitionedOutputs(builder, cluster, replicate_op, + partitioned_outputs, erase_list, + result_types, num_replicas); + } else { + result_op = mlir::tf_device::ReturnOp::create( + builder, replicate_op.getLoc(), cluster.getResults()); + } + + for (auto pi : partitioned_inputs) pi->moveBefore(result_op); + + cluster.getOperation()->moveBefore(result_op); + + return success(); +} + +// Forms clusters with ops of the same `_replication_info` attribute under a +// block. +// +// For a given block, clusters are formed via grouping ops by +// `_replication_info` attributes. For every cluster formed: +// 1. Find associated TPUReplicateMetadata attributes with the same +// `_replication_info` attribute. +// 2. Find users not in cluster that are interleaved between cluster ops. +// 3. Find external uses of cluster ops. +// 4. Create `tf_device.cluster` with results consisting of the external uses +// of cluster ops determined at 3. +// 5. Move cluster ops to `tf_device.cluster` body. +// 6. Replace external uses of cluster ops uses with `tf_device.cluster` +// results. +// 7. Move users from 2 to after the `tf_device.cluster`. +// 8. Wrap cluster (`tf_device.cluster`) in a `tf_device.replicate` if +// attribute `num_replicas` is greater than 1. +// 9. Copy over TPUReplicateMetadata attributes to `tf_device.cluster`. +LogicalResult FormClustersInBlock( + Block* block, + const mlir::TF::SideEffectAnalysis::Info& side_effect_analysis, + bool strict_clusters) { + MetadataMap metadata_map; + LogicalResult result = CollectMetadata(block, &metadata_map); + if (failed(result)) return result; + + // If there is no TPUReplicateMetadata op in this block, process blocks in + // regions attached to the op's in the block. + if (metadata_map.empty()) { + for (Operation& op : *block) { + for (Region& region : op.getRegions()) { + if (!llvm::hasSingleElement(region)) + return op.emitOpError("Expected single block region"); + if (failed(FormClustersInBlock(®ion.front(), side_effect_analysis, + strict_clusters))) + return mlir::failure(); + } + } + return success(); + } + + ClusterMap clusters; + result = CollectAndGroupClusterOps(block, &clusters); + if (failed(result)) return result; + + for (const auto& cluster_metadata_and_ops : clusters) { + const auto& cluster_ops = cluster_metadata_and_ops.getSecond(); + + auto cluster_metadata = + metadata_map.find(cluster_metadata_and_ops.getFirst()); + + // llvm::errs() << __func__ << "\n"; + // No TPUReplicateMetadata for a `_replication_info` attribute. + if (cluster_metadata == metadata_map.end()) { + block->getParentOp()->emitWarning() + << "TPUReplicateMetadata for associated '" + << mlir::TF::kReplicationInfoAttr << "' attribute '" + << cluster_metadata_and_ops.getFirst() << "' is missing"; + continue; + } + + auto status = CollectClusterSuccessorOps( + block, cluster_ops, side_effect_analysis, strict_clusters); + if (failed(status)) return status; + OpSetVector cluster_successor_ops = *status; + + llvm::SmallVector results = + CollectClusterResults(block, cluster_ops); + + mlir::tf_device::ClusterOp cluster = CreateClusterOp( + block, cluster_ops, results, cluster_successor_ops.getArrayRef()); + + auto num_replicas = cluster_metadata->getSecond().get(kNumReplicasAttr); + if (!num_replicas || !mlir::isa(num_replicas)) + return cluster.emitError() + << "requires '" << kNumReplicasAttr << "' int attribute"; + + int num_cores_per_replica = 1; + auto num_cores_per_replica_attr = mlir::dyn_cast_or_null( + cluster_metadata->getSecond().get(kNumCoresPerReplicaAttr)); + if (num_cores_per_replica_attr) + num_cores_per_replica = num_cores_per_replica_attr.getInt(); + if (failed(ReplicateCluster( + cluster, mlir::cast(num_replicas).getInt(), + num_cores_per_replica))) + return mlir::failure(); + + // Copy TPUReplicateMetadata attributes to `tf_device.cluster`. + cluster->setAttrs( + cluster_metadata->second.getDictionary(cluster.getContext())); + // Exclude `num_replicas` as cluster should be replicated if necessary. + cluster->removeAttr(kNumReplicasAttr); + } + + return success(); +} + +LogicalResult FormClustersInFunction( + mlir::func::FuncOp func, + const mlir::TF::SideEffectAnalysis::Info& side_effect_analysis, + bool strict_clusters) { + if (!llvm::hasSingleElement(func)) + return func.emitOpError("Expecting a single block function"); + + if (failed(FormClustersInBlock(&func.front(), side_effect_analysis, + strict_clusters))) + return mlir::failure(); + + // Remove TPUReplicatedInput and TPUReplicatedOutput nodes. + auto remove_result = func.walk([&](Operation* op) { + if (!llvm::isa(op)) + return WalkResult::advance(); + + // Forward operand to result. When `num_replicas` attribute is 1, no + // `tf_device.replicate` is created and replicated (1) operands/results are + // untouched. + if (op->getNumOperands() == 1 && op->getNumResults() == 1) + op->getResult(0).replaceAllUsesWith(op->getOperand(0)); + + // Leftover TPUReplicatedInput/TPUReplicatedOutput that are not of + // `num_replicas` to 1. + if (!op->use_empty()) { + op->emitOpError() << "is expected to have no uses, but it is operand#" + << op->use_begin()->getOperandNumber() << " of " + << *op->use_begin()->getOwner(); + return WalkResult::interrupt(); + } + + op->erase(); + + return WalkResult::advance(); + }); + + return mlir::failure(remove_result.wasInterrupted()); +} + +void TPUClusterFormationPass::runOnOperation() { + // Attributes on tf.Constant aren't reliable: CSE will merge ConstantLike ops + // with the same value (but different attributes!) into the same tf.Const + // definition, potentially leading to bogus _replication_info attributes. So + // we just scrub all tf.Constants of all extra attributes. + // TODO(kramm): Remove this once tf.Const's folder is aware of extra + // attributes. + auto value_str_attr = StringAttr::get(&getContext(), "value"); + getOperation().walk([&](mlir::TF::ConstOp cst) { + auto dict = cst->getAttrDictionary(); + if (dict.size() == 1) { + return; // Optimization. Assume the one attribute is "value". + } + // Recreate the attributes dictionary to only contain "value". + NamedAttrList attributes; + attributes.append(NamedAttribute(value_str_attr, cst->getAttr("value"))); + cst->setAttrs(attributes.getDictionary(&getContext())); + }); + + auto& side_effect_analysis = getAnalysis(); + for (auto func : getOperation().getOps()) + if (!func.isExternal() && + failed(FormClustersInFunction( + func, side_effect_analysis.GetAnalysisForFunc(func), + strict_clusters_))) + return signalPassFailure(); +} +} // anonymous namespace + +std::unique_ptr> CreateTPUClusterFormationPass( + bool strict_clusters) { + return std::make_unique(strict_clusters); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_sharding_identification_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_sharding_identification_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..bffa4522a9c765f0214700499c846b6c2b0efb67 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_sharding_identification_pass.cc @@ -0,0 +1,814 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "xla/hlo/builder/sharding_builder.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla_data.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +using OpShardingVariant = std::variant; +using OpShardingVector = llvm::SmallVector; +using OptionalOpShardingVector = + llvm::SmallVector, 8>; +using llvm::StringRef; +using mlir::Block; +using mlir::BlockArgument; +using mlir::BoolAttr; +using mlir::Builder; +using mlir::IntegerAttr; +using mlir::LogicalResult; +using mlir::ModuleOp; +using mlir::Operation; +using mlir::OpOperand; +using mlir::OpResult; +using mlir::RankedTensorType; +using mlir::StringAttr; +using mlir::Value; +using mlir::WalkResult; + +constexpr char kReplicateSharding[] = ""; +constexpr char kShardingAttr[] = "mhlo.sharding"; +constexpr char kUseSpmdAttr[] = "use_spmd_for_xla_partitioning"; +constexpr char kAliasingAttr[] = "tf.aliasing_output"; +constexpr char kNumCoresPerReplicaAttr[] = "num_cores_per_replica"; +const char kShardingAttribute[] = "_XlaSharding"; +const char kShardingAttributeV2[] = "_XlaShardingV2"; + +#define GEN_PASS_DEF_TPUSHARDINGIDENTIFICATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +struct TPUShardingIdentificationPass + : public impl::TPUShardingIdentificationPassBase< + TPUShardingIdentificationPass> { + void runOnOperation() final; +}; + +std::string CreateMissingAttributeMsg(llvm::StringRef attribute) { + return llvm::formatv("requires attribute '{0}'", attribute).str(); +} + +// Returns nullptr if the op does not have a sharding attribute. +template +mlir::Operation* NullUnlessSharded(PartitionedOp op) { + return op.get_XlaSharding() ? op : nullptr; +} + +// Returns true if unitary op has one of the traits that meets the requirements +// for sharding, otherwise returns false. +bool UnaryOpHasTraitsForSharding(Operation* op) { + // Trait "SameOperandsAndResultTypeResolveRef" for Cast, real/imag, etc. + if (op->hasTrait()) + return true; + // Trait "SameOperandsAndResultType" for Exp, ceil, etc. + if (op->hasTrait()) return true; + // Trait "OperandsSameAsResultsTypeOrRef" for Identity. + if (op->hasTrait()) + return true; + return false; +} + +// Returns true if binary op has one of the traits that meets the requirements +// for sharding, otherwise returns false. +bool BinaryOpHasTraitsForSharding(Operation* op) { + // Trait "CwiseBinary" and "SameOperandsAndResultElementTypeResolveRef" for + // AddV2, Sub, etc. + if (op->hasTrait< + mlir::OpTrait::TF::SameOperandsAndResultElementTypeResolveRef>() && + op->hasTrait()) + return true; + return false; +} + +bool DoTypesHavePartialSameShape(Value value_0, Value value_1) { + auto shape_0 = + mlir::dyn_cast_or_null(value_0.getType()); + auto shape_1 = + mlir::dyn_cast_or_null(value_1.getType()); + if (shape_0 && shape_1) { + if (shape_0.hasStaticShape() && shape_1.hasStaticShape()) + return shape_0.getShape() == shape_1.getShape(); + int i = 0, j = 0; + while (i < shape_0.getShape().size() && j < shape_1.getShape().size()) { + if (shape_0.getShape()[i] != shape_1.getShape()[j] && + !shape_0.isDynamicDim(i) && !shape_1.isDynamicDim(j)) { + return false; + } + if (shape_0.getShape()[i] == shape_1.getShape()[j]) { + i++; + j++; + } else { + if (shape_0.isDynamicDim(i)) { + i++; + } + if (shape_1.isDynamicDim(j)) { + j++; + } + } + } + return i == shape_0.getShape().size() && j == shape_1.getShape().size(); + } + return false; +} +// Returns a TPUPartitionedInput op connected to a `tf_device.cluster_func` +// operand value if it has an XLA sharding. If value is a resource type then +// TPUPartitionedInput op will be connected to a ReadVariable op that feeds into +// a `tf_device.cluster_func`. +mlir::Operation* GetXlaShardingFromOperand(Value value) { + Value value_to_visit = value; + if (auto read_var = value_to_visit.getDefiningOp()) + value_to_visit = read_var.getResource(); + + if (auto partitioned_input = + value_to_visit.getDefiningOp()) { + return NullUnlessSharded(partitioned_input); + } + + return nullptr; +} + +// Returns the op sharding attribute from a partitioned operator. +std::optional GetXlaShardingFromOperator(mlir::Operation* op) { + if (auto partitioned_output = + llvm::dyn_cast(op)) { + if (partitioned_output.get_XlaShardingV2().has_value()) { + return partitioned_output.get_XlaShardingV2(); + } + return partitioned_output.get_XlaSharding(); + } else if (auto partitioned_input = + llvm::dyn_cast(op)) { + if (partitioned_input.get_XlaShardingV2().has_value()) { + return partitioned_input.get_XlaShardingV2(); + } + return partitioned_input.get_XlaSharding(); + } + return std::nullopt; +} + +// Returns the sharding string from a op-sharding variant if it is available. +std::optional GetShardingStringFromVariant( + const OpShardingVariant& sharding_or_op) { + return std::visit( + [](auto&& sharding_or_op) -> std::optional { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return sharding_or_op; + } else { + return GetXlaShardingFromOperator(sharding_or_op); + } + }, + sharding_or_op); +} + +// Returns the sharding from a op-sharding variant if it is available and valid. +std::optional GetShardingFromVariant( + const OpShardingVariant& sharding_or_op) { + xla::OpSharding sharding; + const auto sharding_string = GetShardingStringFromVariant(sharding_or_op); + if (!sharding_string) return std::nullopt; + if (tensorflow::DecodeShardingAttribute(sharding_string->str(), sharding, + false) + .failed()) { + return std::nullopt; + } + return sharding; +} + +// Converts an op-sharding vector into a string attr using the builder. +mlir::ArrayAttr GetStrArrayAttr(Builder* builder, + const OpShardingVector& vect) { + llvm::SmallVector strings; + for (const auto& sharding_or_op : vect) { + if (const auto sharding = GetShardingStringFromVariant(sharding_or_op)) { + strings.emplace_back(builder->getStringAttr(*sharding)); + } + } + return builder->getArrayAttr(strings); +} + +// Verify whether the given sharding can be applied to the given (tensor) type. +// (A bad sharding might mean failing tf.Split ops if the graph later executes +// on CPU) +// If the sharding is incorrect, return failure. If it's good, or if we can't +// verify it, return success. +LogicalResult VerifySharding(mlir::Type type, + const OpShardingVariant& sharding_or_op) { + auto* partitioned_op = + std::holds_alternative(sharding_or_op) + ? std::get(sharding_or_op) + : nullptr; + const auto sharding = GetShardingFromVariant(sharding_or_op); + if (!sharding || sharding->type() != xla::OpSharding::OTHER) { + // Some test cases use \01\02\03 as sharding, to test propagation. Treat + // a non-proto sharding as valid, and don't verify further. We also only + // verify shardings that actually break a tensor apart. + return mlir::success(); + } + if (RankedTensorType ranked_type = mlir::dyn_cast(type)) { + const int64_t tensor_rank = ranked_type.getRank(); + int tile_assignment_rank = sharding->tile_assignment_dimensions_size(); + + // When a tensor is partial or subgroup tiled, its tile assignment will + // have one or more dimension(s) than its rank; so, we subtract them to + // determine which rank the sharding is compatible with. + tile_assignment_rank -= (int)sharding->replicate_on_last_tile_dim(); + tile_assignment_rank -= sharding->last_tile_dims_size(); + + if (tensor_rank < tile_assignment_rank) { + if (partitioned_op) { + partitioned_op->emitError() + << "tensor of type " << ranked_type << " (rank=" << tensor_rank + << ") sharded in " << (tile_assignment_rank - tensor_rank) + << " extra dimension(s) by: " << sharding->DebugString(); + } + + return mlir::failure(); + } + } + return mlir::success(); +} + +// Verify sharding for all arguments and return values. +LogicalResult VerifyShardings(mlir::func::FuncOp func, + const OpShardingVector& sharding_for_args, + const OpShardingVector& sharding_for_rets) { + Block& function_block = func.front(); + for (auto sharding_and_arg : + llvm::zip(sharding_for_args, function_block.getArguments())) { + const auto& sharding = std::get<0>(sharding_and_arg); + BlockArgument arg = std::get<1>(sharding_and_arg); + if (failed(VerifySharding(arg.getType(), sharding))) return mlir::failure(); + } + Operation* terminator = function_block.getTerminator(); + for (auto sharding_and_retval : + llvm::zip(sharding_for_rets, terminator->getOpOperands())) { + const auto& sharding = std::get<0>(sharding_and_retval); + OpOperand& retval = std::get<1>(sharding_and_retval); + if (failed(VerifySharding(retval.get().getType(), sharding))) + return mlir::failure(); + } + return mlir::success(); +} + +// Assign the logical device if an op has an attribute `TPU_REPLICATED_CORE:n`, +// the corresponding input sharding arg will be associated with +// logical device `n`. +std::optional AssignLogicalDeviceFromTPUReplicatedCoreAttr( + Operation* op, const llvm::SmallVector& logical_device_vec) { + if (auto device = op->getAttrOfType("device")) { + if (!device.getValue().empty() && !device.getValue().str().empty()) { + tensorflow::DeviceNameUtils::ParsedName name; + if (tensorflow::DeviceNameUtils::ParseFullName(device.str(), &name)) { + if (name.type == "TPU_REPLICATED_CORE") { + // TODO(hanxiongwang): Add check for out of bound of name.id + return llvm::StringRef(logical_device_vec[name.id]); + } + } + } + } + return std::nullopt; +} + +absl::StatusOr> GetXlaShardingFromShardingOp( + mlir::TF::XlaShardingOp sharding) { + TF_ASSIGN_OR_RETURN(auto attr, GetXlaShardingAttrFromShardingOp(sharding)); + return attr ? ::std::optional<::llvm::StringRef>(attr.getValue()) + : (::std::nullopt); +} + +// Returns XLA sharding from a XlaSharding op connected to an argument value. If +// value is a resource type then XlaSharding op will be connected to a +// ReadVariable op. XlaSharding op may be direct user of inputs but it may also +// be followed by an Identity op and, in the case where bfloat16 type is used, +// Cast op may be added right after the input. +// +// TODO(hongjunchoi): Add logic to parse XlaSharding op inside control flow (If, +// Case) ops and Caller return values. +// TODO(hongjunchoi): Consider explicitly checking op patterns to detect sharded +// inputs. +absl::StatusOr> GetXlaShardingFromArg( + Value value, const llvm::SmallVector& logical_device_vec) { + llvm::SmallPtrSet visited_values; + llvm::SmallVector values_to_visit{value}; + while (!values_to_visit.empty()) { + llvm::SmallVector next_values_to_visit; + for (Value value_to_visit : values_to_visit) { + if (!visited_values.insert(value_to_visit).second) continue; + + for (auto& use : value_to_visit.getUses()) { + Operation* owner = use.getOwner(); + if (auto sharding = llvm::dyn_cast(owner)) + return GetXlaShardingFromShardingOp(sharding); + + if (auto logical_device = AssignLogicalDeviceFromTPUReplicatedCoreAttr( + owner, logical_device_vec)) { + return logical_device; + } + + if (auto while_op = llvm::dyn_cast(owner)) { + const int operand_number = use.getOperandNumber(); + next_values_to_visit.push_back( + while_op.getCond().front().getArgument(operand_number)); + next_values_to_visit.push_back( + while_op.getBody().front().getArgument(operand_number)); + continue; + } + + if (UnaryOpHasTraitsForSharding(owner)) { + next_values_to_visit.push_back(use.getOwner()->getResult(0)); + continue; + } + + if (BinaryOpHasTraitsForSharding(owner)) { + if (DoTypesHavePartialSameShape(value_to_visit, + owner->getResult(0))) { + next_values_to_visit.push_back(use.getOwner()->getResult(0)); + continue; + } + } + + if (llvm::isa(owner)) { + next_values_to_visit.push_back(use.getOwner()->getResult(0)); + continue; + } + + if (auto call_op = llvm::dyn_cast(owner)) { + mlir::func::FuncOp func = + llvm::dyn_cast(call_op.resolveCallable()); + if (!func) continue; + next_values_to_visit.push_back( + func.getArgument(use.getOperandNumber())); + } + } + } + + values_to_visit.swap(next_values_to_visit); + } + + return std::nullopt; +} + +// Tries to extract sharding configurations for all inputs by parsing +// XlaSharding/ TPUPartitionedInput op connected to the operands/arguments. If +// argument to the `cluster_func` directly feeds into another function call op, +// then recursively walk the function definition to find the connected +// XlaSharding op. +absl::Status IdentifyXlaShardingForComputationInputs( + const llvm::SmallVector& logical_device_vec, + bool infer_from_computation, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::func::FuncOp func, Builder* builder, + OptionalOpShardingVector& sharding_for_args) { + // Look up function definition from module. + Block& function_block = func.front(); + + sharding_for_args.reserve(function_block.getNumArguments()); + + // Iterate through operands of `cluster_func`. + // The computation operand can either be: + // 1) a TPUPartitionedInput Op if the input has a non-resource type; + // 2) a ReadVariableOp else. + // + // Iterate through input arguments to the entry block of + // tf_device.ClusterFunc. For input ops, look for XlaSharding ops. + // XlaSharding ops can: + // 1) Directly follow the input argument if input argument has non-resource + // types. + // 2) Follow ReadVariableOp if the input type is of resource type. + // 3) Follow IdentityOp or CastOp after above cases (1), (2). + // + // Sharding configurations are added to the tf_device.ClusterFunc as an + // attribute and the function as an argument attribute. + for (auto operand_and_arg : + llvm::zip(cluster_func.getOperands(), function_block.getArguments())) { + Value operand = std::get<0>(operand_and_arg); + BlockArgument arg = std::get<1>(operand_and_arg); + + if (auto operand_sharding = GetXlaShardingFromOperand(operand)) { + sharding_for_args.push_back(operand_sharding); + continue; + } + + if (infer_from_computation) { + TF_ASSIGN_OR_RETURN(auto arg_sharding, + GetXlaShardingFromArg(arg, logical_device_vec)); + if (arg_sharding) { + sharding_for_args.push_back(arg_sharding.value()); + continue; + } + } + + sharding_for_args.push_back(std::nullopt); + } + return absl::OkStatus(); +} + +// Returns a TPUPartitionedOutput or TPUPartitionedInput op with XLA sharding +// connected to a `tf_device.cluster_func` result value (via AssignVariableOp/ +// resource write). +mlir::Operation* GetXlaShardingFromResult(Value value) { + if (!value.hasOneUse()) return nullptr; + + Operation* user = *value.getUsers().begin(); + if (auto partitioned_output = + llvm::dyn_cast(user)) + return NullUnlessSharded(partitioned_output); + + if (auto assign_var = llvm::dyn_cast(user)) + if (auto partitioned_input = + assign_var.getResource() + .getDefiningOp()) + return NullUnlessSharded(partitioned_input); + + return nullptr; +} + +absl::Status DetermineShardingFromAlias( + mlir::func::FuncOp func, OptionalOpShardingVector& input_shardings, + OptionalOpShardingVector& output_shardings) { + for (int arg_idx = 0; arg_idx < func.getNumArguments(); ++arg_idx) { + if (auto v = + func.getArgAttrOfType(arg_idx, kAliasingAttr)) { + if (int retval_idx = v.getInt(); + retval_idx >= 0 && retval_idx < func.getNumResults()) { + auto& input_sharding = input_shardings[arg_idx]; + auto& output_sharding = output_shardings[retval_idx]; + + if (input_sharding.has_value() && output_sharding.has_value() && + input_sharding.value() != output_sharding.value()) { + return absl::InvalidArgumentError(absl::StrCat( + "arg#", arg_idx, " is aliased to retval#", retval_idx, + " but their sharding configurations don't match.")); + } else if (input_sharding.has_value() && !output_sharding.has_value()) { + output_sharding = input_sharding; + } else if (!input_sharding.has_value() && output_sharding.has_value()) { + input_sharding = output_sharding; + } + } + } + } + + return absl::OkStatus(); +} + +// Returns XLA sharding from XlaSharding op connected to a result value. +// XlaSharding op may be directly connected to output but it may also be +// followed by Identity or simple arithmetic ops. In case where bfloat16 type is +// used, we might see a Cast op. +// +// TODO(hongjunchoi): Add logic to parse XlaSharding op inside control flow (If, +// Case) ops and Caller argument values. +// TODO(hongjunchoi): Consider explicitly checking op patterns to detect sharded +// inputs. +absl::StatusOr> GetXlaShardingFromRetval( + Value value, const llvm::SmallVector& logical_device_vec) { + llvm::SmallPtrSet visited_values; + llvm::SmallVector values_to_visit; + values_to_visit.push_back(value); + + while (!values_to_visit.empty()) { + Value value_to_visit = values_to_visit.pop_back_val(); + + if (!visited_values.insert(value_to_visit).second) { + continue; + } + + Operation* def = value_to_visit.getDefiningOp(); + if (!def) { + continue; + } + + if (auto sharding = llvm::dyn_cast_or_null(def)) + return GetXlaShardingFromShardingOp(sharding); + + if (auto sharding = def->getAttrOfType(kShardingAttributeV2)) { + return sharding.strref(); + } + + if (auto sharding = def->getAttrOfType(kShardingAttribute)) { + return sharding.strref(); + } + + if (auto logical_device = AssignLogicalDeviceFromTPUReplicatedCoreAttr( + def, logical_device_vec)) { + return logical_device; + } + + if (UnaryOpHasTraitsForSharding(def) || BinaryOpHasTraitsForSharding(def)) { + for (auto operand : def->getOperands()) { + values_to_visit.push_back(operand); + } + continue; + } + + if (auto call_op = llvm::dyn_cast_or_null(def)) { + mlir::func::FuncOp func = + llvm::dyn_cast(call_op.resolveCallable()); + if (!func) continue; + value_to_visit = func.front().getTerminator()->getOperand( + mlir::cast(value_to_visit).getResultNumber()); + values_to_visit.push_back(value_to_visit); + continue; + } + + if (auto while_op = llvm::dyn_cast(def)) { + if (auto op_result = mlir::cast(value_to_visit)) { + int result_idx = op_result.getResultNumber(); + if (auto yield_op = llvm::dyn_cast( + while_op.getBody().front().getTerminator())) { + values_to_visit.push_back(yield_op.getOperand(result_idx)); + } + } + continue; + } + } + + return std::nullopt; +} + +// Tries to extract sharding configurations for all outputs by parsing +// XlaSharding/ TPUPartitionedOutput op connected to the retvals/results. +absl::Status IdentifyXlaShardingForComputationOutputs( + const llvm::SmallVector& logical_device_vec, + bool infer_from_computation, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::func::FuncOp func, Builder* builder, + OptionalOpShardingVector& sharding_for_rets) { + Block& function_block = func.front(); + Operation* terminator = function_block.getTerminator(); + sharding_for_rets.reserve(terminator->getNumOperands()); + + // Iterate through results of `cluster_func`. For output ops, look for + // TPUPartitionedOutput ops. + // + // Iterate through operands of the terminator. If the preceding op is + // XlaShardingOp, then the provided sharding configuration is added to the + // tf_device.ClusterFunc as an attribute and the function as a result + // attribute. + for (auto result_and_retval : + llvm::zip(cluster_func.getResults(), terminator->getOpOperands())) { + Value result = std::get<0>(result_and_retval); + OpOperand& retval = std::get<1>(result_and_retval); + + if (auto result_sharding = GetXlaShardingFromResult(result)) { + sharding_for_rets.push_back(result_sharding); + continue; + } + + if (infer_from_computation) { + TF_ASSIGN_OR_RETURN( + auto retval_sharding, + GetXlaShardingFromRetval(retval.get(), logical_device_vec)); + if (retval_sharding) { + sharding_for_rets.push_back(retval_sharding.value()); + continue; + } + } + + sharding_for_rets.push_back(std::nullopt); + } + return absl::OkStatus(); +} + +void SetReplicatedOrMaximalShardingIfNoShardingFound( + const llvm::SmallVector& logical_device_vec, bool use_spmd, + OptionalOpShardingVector& shardings) { + for (auto& sharding : shardings) { + if (sharding == std::nullopt) { + // If we haven't found sharding, default to either replicated or maximal + // sharding depending on whether XLA SPMD is enabled. + if (use_spmd) { + // If XLA SPMD is enabled, host variables or non-variable per-replica + // inputs, and outputs should take on replicate sharding, so that every + // device gets the whole tensor(s) (and can slice them up later eg. + // using dynamic-slice). + sharding = kReplicateSharding; + } else { + // Otherwise, default to maximal sharding core 0. + sharding = logical_device_vec[0]; + } + } + } +} + +// Moves shardings from `optional_shardings` to `shardings`. +absl::Status MoveSharding(OptionalOpShardingVector& optional_shardings, + OpShardingVector& shardings) { + shardings.clear(); + for (auto& sharding : optional_shardings) { + if (!sharding) { + return absl::InternalError( + "Couldn't find/assign sharding for an input/output. All shardings " + "should have been identified by this point."); + } + + shardings.push_back(std::move(sharding.value())); + } + + return absl::OkStatus(); +} + +// Determines XlaSharding for inputs and outputs. If there are aliased +// inputs/outputs for which no sharding was found directly, the corresponding +// output/input sharding is used (if it exists). If we still don't find sharding +// for some inputs/outputs, we default to replicated or maximal sharding +// depending on `use_spmd`. +absl::Status IdentifyXlaShardingForInputsAndOutputs( + const llvm::SmallVector& logical_device_vec, bool use_spmd, + bool infer_from_computation, mlir::tf_device::ClusterFuncOp cluster_func, + mlir::func::FuncOp func, Builder* builder, OpShardingVector& input_sharding, + OpShardingVector& output_sharding) { + OptionalOpShardingVector optional_input_sharding; + OptionalOpShardingVector optional_output_sharding; + TF_RETURN_IF_ERROR(IdentifyXlaShardingForComputationInputs( + logical_device_vec, infer_from_computation, cluster_func, func, builder, + optional_input_sharding)); + TF_RETURN_IF_ERROR(IdentifyXlaShardingForComputationOutputs( + logical_device_vec, infer_from_computation, cluster_func, func, builder, + optional_output_sharding)); + TF_RETURN_IF_ERROR(DetermineShardingFromAlias(func, optional_input_sharding, + optional_output_sharding)); + SetReplicatedOrMaximalShardingIfNoShardingFound(logical_device_vec, use_spmd, + optional_input_sharding); + SetReplicatedOrMaximalShardingIfNoShardingFound(logical_device_vec, use_spmd, + optional_output_sharding); + TF_RETURN_IF_ERROR(MoveSharding(optional_input_sharding, input_sharding)); + TF_RETURN_IF_ERROR(MoveSharding(optional_output_sharding, output_sharding)); + + return absl::OkStatus(); +} + +// Extracts input/output sharding configuration of `cluster_func` by parsing +// XlaSharding ops inside the `cluster_func`. +LogicalResult IdentifyXlaShardingForTPUComputation( + Builder* builder, mlir::tf_device::ClusterFuncOp cluster_func) { + // Look up function definition from module. + mlir::func::FuncOp func = + cluster_func->getParentOfType() + .lookupSymbol(cluster_func.getFunc()); + + bool use_spmd = false; + if (auto use_spmd_attr = cluster_func->getAttrOfType(kUseSpmdAttr)) + use_spmd = use_spmd_attr.getValue(); + + auto num_cores_per_replica_attr = + cluster_func->getAttrOfType(kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) + return cluster_func.emitOpError( + CreateMissingAttributeMsg(kNumCoresPerReplicaAttr)); + + int num_cores_per_replica = num_cores_per_replica_attr.getInt(); + llvm::SmallVector logical_device_vec(num_cores_per_replica); + + for (int idx = 0; idx < num_cores_per_replica; idx++) { + logical_device_vec[idx] = + xla::sharding_builder::SingleDevice(idx).SerializeAsString(); + } + + OpShardingVector sharding_for_args; + OpShardingVector sharding_for_rets; + if (auto status = IdentifyXlaShardingForInputsAndOutputs( + logical_device_vec, use_spmd, + /*infer_from_computation=*/true, cluster_func, func, builder, + sharding_for_args, sharding_for_rets); + !status.ok()) { + return cluster_func.emitOpError(status.message()); + }; + + auto has_maximal_sharding = + [](const OpShardingVariant& sharding_or_op) -> bool { + const auto sharding = GetShardingFromVariant(sharding_or_op); + return sharding && sharding->type() == xla::OpSharding::MAXIMAL; + }; + + // XLA SPMD only supports cases where all inputs/outputs exist on every + // partition (sharded or replicated). If any of the inputs/outputs have + // maximal sharding, then fallback to MPMD. Also fall back if any of the + // shardings aren't compatible with the rank of their tensor. + if ((use_spmd && (absl::c_any_of(sharding_for_args, has_maximal_sharding) || + absl::c_any_of(sharding_for_rets, has_maximal_sharding))) || + failed(VerifyShardings(func, sharding_for_args, sharding_for_rets))) { + LOG(WARNING) << "XLA SPMD only supports cases where all inputs/outputs " + "exist on every partition (sharded or replicated). If any " + "of the inputs/outputs have maximal sharding, then " + "fallback to MPMD."; + sharding_for_args.clear(); + sharding_for_rets.clear(); + cluster_func->setAttr(kUseSpmdAttr, builder->getBoolAttr(false)); + + if (auto status = IdentifyXlaShardingForInputsAndOutputs( + logical_device_vec, /*use_spmd=*/false, + /*infer_from_computation=*/false, cluster_func, func, builder, + sharding_for_args, sharding_for_rets); + !status.ok()) { + return cluster_func.emitOpError(status.message()); + } + } + + // Update sharding on function arguments and returns. + Block& function_block = func.front(); + for (auto sharding_and_arg : + llvm::zip(sharding_for_args, function_block.getArguments())) { + BlockArgument arg = std::get<1>(sharding_and_arg); + const auto& sharding_or_op = std::get<0>(sharding_and_arg); + if (auto sharding = GetShardingStringFromVariant(sharding_or_op)) { + func.setArgAttr(arg.getArgNumber(), kShardingAttr, + builder->getStringAttr(*sharding)); + } + } + + Operation* terminator = function_block.getTerminator(); + for (auto sharding_and_retval : + llvm::zip(sharding_for_rets, terminator->getOpOperands())) { + OpOperand& retval = std::get<1>(sharding_and_retval); + const auto& sharding_or_op = std::get<0>(sharding_and_retval); + if (auto sharding = GetShardingStringFromVariant(sharding_or_op)) { + func.setResultAttr(retval.getOperandNumber(), kShardingAttr, + builder->getStringAttr(*sharding)); + } + } + + // Update input/output sharding attributes on tf_device.cluster_func op. + cluster_func->setAttr(tensorflow::kInputShardingAttr, + GetStrArrayAttr(builder, sharding_for_args)); + cluster_func->setAttr(tensorflow::kOutputShardingAttr, + GetStrArrayAttr(builder, sharding_for_rets)); + return mlir::success(); +} + +void TPUShardingIdentificationPass::runOnOperation() { + Builder builder(getOperation().getContext()); + + auto result = + getOperation().walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + if (failed( + IdentifyXlaShardingForTPUComputation(&builder, cluster_func))) { + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) return signalPassFailure(); +} + +} // namespace + +std::unique_ptr> +CreateTPUShardingIdentificationPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..62d58f101c7058906a81b118303a9fa910bbf364 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.cc @@ -0,0 +1,645 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/strings/match.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h" +#include "xla/xla_data.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +#define GEN_PASS_DEF_TPUVALIDATEINPUTSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +constexpr char kXLAShardingAttr[] = "_XlaSharding"; +constexpr char kShardingAttr[] = "sharding"; + +using mlir::dyn_cast; +using mlir::isa; +using mlir::ModuleOp; +using mlir::Operation; +using mlir::OperationPass; +using mlir::StringAttr; +using mlir::StringRef; +using mlir::Type; +using mlir::TypeID; +using mlir::func::FuncOp; +using mlir::func::ReturnOp; +using mlir::TF::AssertOp; +using mlir::TF::ConstOp; +using mlir::TF::kCompileDeviceTypeAttr; +using mlir::TF::kTpuReplicateAttr; +using mlir::TF::OutfeedEnqueueTupleOp; +using mlir::TF::PartitionedCallOp; +using mlir::TF::StatefulPartitionedCallOp; +using mlir::TF::TPUPartitionedCallOp; +using mlir::TF::TPUPartitionedInputOp; +using mlir::TF::TPUPartitionedInputV2Op; +using mlir::TF::TPUPartitionedOutputOp; +using mlir::TF::TPUPartitionedOutputV2Op; +using mlir::TF::TPUReplicatedInputOp; +using mlir::TF::TPUReplicatedOutputOp; +using mlir::TF::TPUReplicateMetadataOp; +using mlir::TF::WhileOp; +using mlir::TF::XlaSetDynamicDimensionSizeOp; +using mlir::tf_executor::FetchOp; +using mlir::tf_executor::GraphOp; +using mlir::tf_executor::IslandOp; +using mlir::tf_executor::YieldOp; + +typedef std::unordered_map MetadataMap; + +struct TPUValidateInputsPass + : public impl::TPUValidateInputsPassBase { + void runOnOperation() override; +}; +bool IsTpuRegularOp(Operation* op) { + static auto* ops = [] { + llvm::SmallDenseSet* ops_set = + new llvm::SmallDenseSet{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + return ops_set; + }(); + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return true; + return ops->count(abstractOp->getTypeID()) == 0; +} + +bool IsIntersectionXlaNonXlaOps(Operation* op) { + static auto* ops = [] { + llvm::SmallDenseSet* ops_set = + new llvm::SmallDenseSet{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + return ops_set; + }(); + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return true; + return ops->count(abstractOp->getTypeID()) == 0; +} + +bool IsPartitionedOp(Operation* op) { + static auto* ops = [] { + llvm::SmallDenseSet* ops_set = + new llvm::SmallDenseSet{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + return ops_set; + }(); + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return false; + return ops->count(abstractOp->getTypeID()) != 0; +} + +// Gets the successors of an op wrapped in a tf_executor.island. +llvm::SmallVector GetSuccessors(Operation* op) { + llvm::SmallVector successors; + for (auto result : op->getParentOp()->getOpResults()) { + for (auto& use : result.getUses()) { + auto succ = use.getOwner(); + successors.push_back(succ); + } + } + return successors; +} +// Gets the predecessors of an op wrapped in tf_executor.island. +llvm::SmallVector GetPredecessors(Operation* op) { + llvm::SmallVector predecessors; + for (auto operand : op->getOperands()) { + if (Operation* pred = operand.getDefiningOp()) { + pred->walk([&](mlir::Operation* opinexecutor) { + predecessors.push_back(opinexecutor); + }); + } + } + return predecessors; +} + +bool CheckTpuReplicateAttr(Operation* op, StringAttr attr, + std::function errormsg) { + if (!op->hasAttr(kTpuReplicateAttr)) { + op->emitOpError("TF2XLA TPU bridge input check: " + errormsg() + + "missing _tpu_replicate attr"); + return false; + } + auto opattr = op->getAttr(kTpuReplicateAttr); + if (opattr != attr) { + op->emitOpError("TF2XLA TPU bridge input check: " + errormsg() + + "invalid _tpu_replicate attr.") + << " Expected attr: " << attr << ", Actual attr: " << opattr; + return false; + } + return true; +} + +bool ValidateReplicatedInput(TPUReplicatedInputOp rep, int num_replicas, + StringAttr attr) { + int arity = rep.getInputs().size(); + if (rep.getIsPacked() && arity != 1) { + rep.emitOpError( + "TF2XLA TPU bridge input check: packed with number of inputs not 1.") + << " num_replicas=" << num_replicas << " no. of inputs=" << arity; + return false; + } else if (!rep.getIsPacked() && arity != num_replicas) { + rep.emitOpError( + "TF2XLA TPU bridge input check: number of inputs inconsistent.") + << " num_replicas=" << num_replicas << " no. of inputs=" << arity; + return false; + } + for (auto& succ : GetSuccessors(rep)) { + if (!IsTpuRegularOp(succ)) continue; + auto errormsg = [&]() -> std::string { + return rep->getName().getStringRef().str() + " op has successor op " + + succ->getName().getStringRef().str() + " with error: "; + }; + if (!CheckTpuReplicateAttr(succ, attr, errormsg)) return false; + } + return true; +} +bool ValidateReplicatedOutput(TPUReplicatedOutputOp rep, int num_replicas, + StringAttr attr) { + int arity = rep.getOutputs().size(); + if (arity != num_replicas) { + rep.emitOpError( + "TF2XLA TPU bridge input check: number of outputs inconsistent.") + << " num_replicas=" << num_replicas << " no. of outputs=" << arity; + return false; + } + for (auto& pred : GetPredecessors(rep)) { + if (!IsTpuRegularOp(pred)) continue; + auto errormsg = [&]() -> std::string { + return rep->getName().getStringRef().str() + " op has predecessor op " + + pred->getName().getStringRef().str() + " with error: "; + }; + if (!CheckTpuReplicateAttr(pred, attr, errormsg)) return false; + } + return true; +} +bool ValidatePartitionedInput(TPUPartitionedInputOp rep, + int num_cores_per_replica) { + int arity = rep.getInputs().size(); + if (arity != num_cores_per_replica) { + rep.emitOpError( + "TF2XLA TPU bridge input check: number of inputs inconsistent.") + << " num_cores_per_replica=" << num_cores_per_replica + << " no. of inputs=" << arity; + return false; + } + return true; +} +bool ValidatePartitionedInputV2(TPUPartitionedInputV2Op rep, + int num_cores_per_replica) { + int arity = rep.getInputs().size(); + if (rep.getIsPacked() && arity != 1) { + rep.emitOpError( + "TF2XLA TPU bridge input check: packed with number of inputs not 1.") + << " num_cores_per_replicas=" << num_cores_per_replica + << " no. of inputs=" << arity; + return false; + } else if (!rep.getIsPacked() && arity != num_cores_per_replica) { + rep.emitOpError( + "TF2XLA TPU bridge input check: number of inputs inconsistent.") + << " num_cores_per_replica=" << num_cores_per_replica + << " no. of inputs=" << arity; + return false; + } + return true; +} +template +bool ValidatePartitionedOutput(T rep, int num_cores_per_replica) { + int arity = rep.getOutput().size(); + if (arity != num_cores_per_replica) { + rep.emitOpError( + "TF2XLA TPU bridge input check: number of outputs inconsistent.") + << " num_cores_per_replica=" << num_cores_per_replica + << " no. of outputs=" << arity; + return false; + } + return true; +} + +bool CheckReplicatedIOOp(Operation* op, TPUReplicateMetadataOp metadata, + Operation* parent) { + int num_replicas = metadata.getNumReplicas(); + int num_cores_per_replica = metadata.getNumCoresPerReplica(); + StringAttr tpu_replicate_attr = + metadata->getAttrOfType(kTpuReplicateAttr); + if (auto repinput = dyn_cast(op)) { + if (!ValidateReplicatedInput(repinput, num_replicas, tpu_replicate_attr)) + return false; + } + if (auto repoutput = dyn_cast(op)) { + if (!ValidateReplicatedOutput(repoutput, num_replicas, tpu_replicate_attr)) + return false; + } + if (auto partinput = dyn_cast(op)) { + if (!ValidatePartitionedInput(partinput, num_cores_per_replica)) + return false; + } + if (auto partinput = dyn_cast(op)) { + if (!ValidatePartitionedInputV2(partinput, num_cores_per_replica)) + return false; + } + if (auto partoutput = dyn_cast(op)) { + if (!ValidatePartitionedOutput(partoutput, num_cores_per_replica)) + return false; + } + if (auto partoutput = dyn_cast(op)) { + if (!ValidatePartitionedOutput(partoutput, num_cores_per_replica)) + return false; + } + return true; +} +// Checking op which is successor to a cluster op. +bool CheckClusterSuccessors(Operation* op, std::string cluster, + Operation* parent, MetadataMap& metadata_map) { + std::string cluster_succ = ""; + if (op->hasAttr(kTpuReplicateAttr)) { + cluster_succ = op->getAttrOfType(kTpuReplicateAttr).str(); + } + if (cluster_succ.empty()) { + // TODO (b/269195256#comment16): Change to error after resolving issue + // with test. Will fix it after the upstream code is fixed. + op->emitWarning("TF2XLA TPU bridge input check: cluster op = ") + << parent->getName() << " with cluster = " << cluster + << " has successor as non cluster op " << op->getName(); + return true; + } + if (cluster != cluster_succ) { + op->emitOpError( + "TF2XLA TPU bridge input check: mismatch clusters tpu_replicate " + "attr. Parent op ") + << parent->getName() << " with cluster = " << cluster + << " has successor cluster op " << op->getName() + << " with cluster = " << cluster_succ; + return false; + } + return true; +} + +// Checking op which is a predecessor to a non-cluster op. +bool CheckNonClusterSuccessors(Operation* op, Operation* parent, + MetadataMap& metadata_map) { + if (!IsTpuRegularOp(op)) { + if (isa(op)) { + op->emitOpError("TF2XLA TPU bridge input check: non-cluster op = ") + << parent->getName() + << " has invalid successor op = " << op->getName(); + return false; + } else { + return true; + } + } + return true; +} +// Checking op which is a successor to a non-cluster op. +bool CheckNonClusterPredecessors(Operation* op, Operation* parent, + MetadataMap& metadata_map) { + if (!IsTpuRegularOp(op)) { + if (isa(op)) { + op->emitOpError("TF2XLA TPU bridge input check: non-cluster op = ") + << parent->getName() + << " has invalid predecessor op = " << op->getName(); + return false; + } else { + return true; + } + } + return true; +} + +bool CheckOpsClusterIO(Operation* op, MetadataMap& metadata_map) { + bool is_cluster_op = false; + std::string cluster = ""; + if (op->hasAttr(kTpuReplicateAttr)) { + cluster = op->getAttrOfType(kTpuReplicateAttr).str(); + if (cluster.empty()) { + op->emitOpError("TF2XLA TPU bridge input check: empty _tpu_replicate") + << " attr for op = " << op->getName(); + return false; + } + is_cluster_op = true; + } + bool has_cluster_metadata = + (metadata_map.find(cluster) != metadata_map.end()); + + for (auto pred : GetPredecessors(op)) { + if (is_cluster_op && !IsTpuRegularOp(pred) && has_cluster_metadata) { + if (!CheckReplicatedIOOp(pred, metadata_map[cluster], op)) return false; + } + if (!is_cluster_op) { + if (!CheckNonClusterPredecessors(pred, op, metadata_map)) return false; + } + } + + for (auto succ : GetSuccessors(op)) { + if (is_cluster_op && !IsTpuRegularOp(succ) && has_cluster_metadata) { + if (!CheckReplicatedIOOp(succ, metadata_map[cluster], op)) return false; + } + if (is_cluster_op && IsTpuRegularOp(succ)) { + if (!CheckClusterSuccessors(succ, cluster, op, metadata_map)) + return false; + } + if (!is_cluster_op) { + if (!CheckNonClusterSuccessors(succ, op, metadata_map)) return false; + } + } + return true; +} + +bool TypeMustBeNonXLA(const Type& type) { + const Type elem = getElementTypeOrSelf(type); + return !mlir::isa(elem) && + !tensorflow::TypeValidForXLA(type); +} + +// Check if the op cannot be XLA compiled. If the op does not satisfy this +// criteria, then it is possible for the op to be XLA and non-XLA. But this +// function specifically checks if the op must be non-xla. +bool IsMustNotBeXlaOp(Operation* op) { + for (auto& input : op->getOpOperands()) { + if (TypeMustBeNonXLA(input.get().getType())) return true; + } + for (auto output_types : op->getResultTypes()) { + if (TypeMustBeNonXLA(output_types)) return true; + } + return false; +} + +// Check if the op must be compiled with XLA. If the op does not satisfy this +// critiria for "must be xla" then it is still possible for this op to be xla +// and non-xla as well. But below function specifically checks for the op to be +// only XLA op. +bool IsMustBeXlaOp(Operation* op, MetadataMap metadata_map) { + // All PartitionedCall are inlined-out before XLA. + // So MustBeXLA should return false + if (IsPartitionedOp(op)) return false; + if (!op->hasAttr(kTpuReplicateAttr)) return false; + auto cluster = op->getAttrOfType(kTpuReplicateAttr).str(); + if (metadata_map.find(cluster) == metadata_map.end()) return false; + auto metadata = metadata_map[cluster]; + if (!metadata.getAllowSoftPlacement() && + !op->hasAttr(mlir::TF::kXlaOutsideCompilationAttr)) + return true; + std::string device = ""; + if (op->hasAttr(mlir::TF::kDeviceAttr)) + device = op->getAttrOfType(mlir::TF::kDeviceAttr).str(); + else + return false; + if (absl::StrContains(device, mlir::TF::kTpuDevice)) return true; + return false; +} +bool ValidateIntersectionXlaNonXlaOps(Operation* op, MetadataMap metadata_map) { + if (isa(op) || isa(op) || + isa(op) || isa(op) || + isa(op) || isa(op) || + isa(op)) + return true; + if (IsMustBeXlaOp(op, metadata_map) && IsMustNotBeXlaOp(op)) { + // TODO(b/269195256#comment19) change the warning for Identity op to error + // when issue with input graph is resolved. Possible issue with python layer + // inserting Identity op incorrectly. + if (isa(op)) { + op->emitWarning("TF/XLA TPU bridge input check: found invalid op. ") + << op->getName() << " can't be both xla and non-xla"; + return true; + } + op->emitOpError("TF/XLA TPU bridge input check: found invalid op. ") + << "Can't be both xla and non-xla"; + return false; + } + return true; +} +void GetFlattenedShardings(llvm::SmallVector& shardings_result, + std::string shard_string) { + xla::OpSharding sharding; + if (shard_string.empty()) return; + if (!sharding.ParseFromString(shard_string)) return; + std::queue shardings; + shardings.push(sharding); + while (!shardings.empty()) { + auto sharding_next = shardings.front(); + shardings.pop(); + if (sharding_next.type() == xla::OpSharding::TUPLE) { + for (auto& child_sharding : sharding_next.tuple_shardings()) { + shardings.push(child_sharding); + } + } else { + shardings_result.push_back(sharding_next); + } + } +} + +bool IsValidShardingTupleForArity(Operation* op) { + if (!op->hasAttr(kXLAShardingAttr) && !op->hasAttr(kShardingAttr)) { + return true; + } + std::string shard_string; + if (op->hasAttr(kXLAShardingAttr)) { + shard_string = + op->getAttrOfType(kXLAShardingAttr).strref().str(); + } else { + shard_string = op->getAttrOfType(kShardingAttr).strref().str(); + } + xla::OpSharding sharding; + if (!shard_string.empty() && sharding.ParseFromString(shard_string)) { + // Only checking op with TUPLE sharding + if (sharding.type() != xla::OpSharding::TUPLE) { + return true; + } + // Each output is expected to have a corresponding sharding given by + // tuple shardings. So, the no. of outputs (arity) should be same as + // number of tuple shardings. + if (sharding.tuple_shardings().size() != op->getNumResults()) { + op->emitOpError( + "TF2XLA TPU bridge input check: invalid no. of tuple shardings ") + << sharding.tuple_shardings().size() + << " for arity = " << op->getNumResults() << "\n The sharding is " + << sharding.DebugString() << "\n"; + return false; + } + } + return true; +} + +bool IsValidMAXIMALSharding(Operation* op, MetadataMap& metadata_map) { + if (!op->hasAttr(kTpuReplicateAttr)) return true; + if (!op->hasAttr(kXLAShardingAttr) && !op->hasAttr(kShardingAttr)) { + return true; + } + + int num_cores_per_replica; + // Assuming that the op is a IsTpuRegularOp and has a cluster metadata + // for it. These checks are already performed in CheckOpsClusterIO. + // Also assuming that if there is sharding, then there must be + // cluster and the metadata corresponding to it. + auto cluster = op->getAttrOfType(kTpuReplicateAttr).str(); + if (cluster.empty()) { + return true; + } + if (metadata_map.find(cluster) == metadata_map.end()) { + return true; + } + num_cores_per_replica = metadata_map[cluster].getNumCoresPerReplica(); + std::optional shard_string; + if (op->hasAttr(kXLAShardingAttr)) { + shard_string = op->getAttrOfType(kXLAShardingAttr).strref(); + } else { + shard_string = op->getAttrOfType(kShardingAttr).strref(); + } + llvm::SmallVector shardings; + GetFlattenedShardings(shardings, shard_string.value().str()); + + for (auto& sharding : shardings) { + if (sharding.type() != xla::OpSharding::MAXIMAL) { + continue; + } + if (sharding.tile_assignment_devices_size() != 1) { + op->emitOpError("TF/XLA TPU bridge input check: There must be ") + << "exactly 1 device for MAXIMAL sharding." + << " Number of devices assigned are " + << sharding.tile_assignment_devices_size() << "\n"; + return false; + } else { + int sharding_device = sharding.tile_assignment_devices(0); + if (sharding_device >= num_cores_per_replica || sharding_device < 0) { + op->emitOpError( + "TF2XLA TPU bridge input check: invalid sharding device ") + << sharding_device + << " for num_cores_per_replica = " << num_cores_per_replica + << "\n The sharding is " << sharding.DebugString() << "\n"; + return false; + } + } + } + return true; +} + +bool HasSingleCoreTpu(Operation* op) { + if (auto compilation_attr = + op->getAttrOfType(kCompileDeviceTypeAttr)) { + if (compilation_attr.getValue().str() == mlir::TF::kTpuDevice) { + op->emitOpError( + "TF2XLA TPU bridge input check: found a single-core TPU graph"); + return true; + } + } + return false; +} + +void TPUValidateInputsPass::runOnOperation() { + ModuleOp module = getOperation(); + bool success = true; + int num_metadata = 0; + TPUReplicateMetadataOp metadata; + MetadataMap metadata_map; + module.walk([&](TPUReplicateMetadataOp meta) { + ++num_metadata; + metadata = meta; + metadata_map[meta->getAttrOfType(kTpuReplicateAttr).str()] = + meta; + }); + + getOperation().walk([&](mlir::Operation* op) { + if (IsPotentialUnsupportedOp(op)) { + LOG(WARNING) << "Potential unsupported op: " + << op->getName().getStringRef().str() + << ". TF2XLA MLIR bridge does not guarantee to support it."; + } + + if (IsTpuRegularOp(op)) { + success &= CheckOpsClusterIO(op, metadata_map); + } + if (IsIntersectionXlaNonXlaOps(op)) { + success &= ValidateIntersectionXlaNonXlaOps(op, metadata_map); + } + if (IsTpuRegularOp(op)) { + success &= IsValidMAXIMALSharding(op, metadata_map); + success &= IsValidShardingTupleForArity(op); + } + success &= !HasSingleCoreTpu(op); + + if (!success) { + signalPassFailure(); + } + }); + + module.walk([&](GraphOp graph) { + if (HasV1ControlFlow(graph)) { + LOG(WARNING) << "TF2XLA MLIR bridge does not support v1 control flow." + << " Use at your own risk."; + } + if (!success) { + signalPassFailure(); + } + }); +} + +} // anonymous namespace + +std::unique_ptr> CreateTPUValidateInputsPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a8b81b777fdc4477f17ec26242db2f14bf8cbf33 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs.mlir @@ -0,0 +1,279 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-validate-inputs | FileCheck %s + +// CHECK-LABEL: func @num_replicas_replicated +func.func @num_replicas_replicated(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @num_replicas_replicated_input(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + // expected-error @+1 {{'tf.TPUReplicatedInput' op TF2XLA TPU bridge input check: number of inputs inconsistent. num_replicas=2 no. of inputs=3}} + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @num_replicas_replicated_input_packed(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + // expected-error @+1 {{'tf.TPUReplicatedInput' op TF2XLA TPU bridge input check: packed with number of inputs not 1. num_replicas=2 no. of inputs=2}} + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = true} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @num_replicas_replicated_output(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + // expected-error @+1 {{'tf.TPUReplicatedOutput' op TF2XLA TPU bridge input check: number of outputs inconsistent. num_replicas=2 no. of outputs=3}} + %ro:3, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @num_core_per_replica_partitioned_input(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + // expected-error @+1 {{'tf.TPUPartitionedInput' op TF2XLA TPU bridge input check: number of inputs inconsistent. num_cores_per_replica=2 no. of inputs=3}} + %pi, %c0 = tf_executor.island wraps "tf.TPUPartitionedInput"(%arg0, %arg1, %arg1) {index = 1 : i64} : (tensor, tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%pi) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %po:2, %c2 = tf_executor.island wraps "tf.TPUPartitionedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %po#0, %po#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @num_core_per_replica_partitioned_output(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + %pi, %c0 = tf_executor.island wraps "tf.TPUPartitionedInput"(%arg0, %arg1) {index = 1 : i64} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%pi) {_tpu_replicate = "cluster"} : (tensor) -> tensor + // expected-error @+1 {{'tf.TPUPartitionedOutput' op TF2XLA TPU bridge input check: number of outputs inconsistent. num_cores_per_replica=2 no. of outputs=3}} + %po:3, %c2 = tf_executor.island wraps "tf.TPUPartitionedOutput"(%out) : (tensor) -> (tensor, tensor, tensor) + tf_executor.fetch %po#0, %po#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @validate_tpu_replicate_no_attr(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate="cluster"}: (tensor) -> tensor + // expected-warning @+1 {{TF2XLA TPU bridge input check: cluster op = tf.opA with cluster = cluster has successor as non cluster op tf.opB}} + %out2, %c2 = tf_executor.island wraps "tf.opB"(%out) : (tensor) -> tensor + // expected-error @+1 {{tf.TPUReplicatedOutput' op TF2XLA TPU bridge input check: non-cluster op = tf.opB has invalid successor op = tf.TPUReplicatedOutput}} + %ro:2, %c4 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out2) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @validate_tpu_replicate_wrong_attr(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster_wrong"}: (tensor) -> tensor + // expected-error @+1 {{'tf.opB' op TF2XLA TPU bridge input check: mismatch clusters tpu_replicate attr. Parent op tf.opA with cluster = cluster_wrong has successor cluster op tf.opB with cluster = cluster}} + %out2, %c2 = tf_executor.island wraps "tf.opB"(%out) {_tpu_replicate = "cluster"}: (tensor) -> tensor + %ro:2, %c3 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out2) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @valid_xla_nonxla(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster", device = "TPU"} : (tensor) -> tensor + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- + +func.func @valid_xla_nonxla_warning(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) { + %0:2 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor<*x!tf_type.string> + // expected-warning @+1 {{TF/XLA TPU bridge input check: found invalid op. tf.Identity can't be both xla and non-xla}} + %out, %c1 = tf_executor.island(%c0) wraps "tf.Identity"(%ri) {_tpu_replicate = "cluster", device = ""} : (tensor<*x!tf_type.string>) -> tensor<*x!tf_type.string> + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor<*x!tf_type.string>) -> (tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) + tf_executor.fetch %ro#0, %ro#1 : tensor<*x!tf_type.string>, tensor<*x!tf_type.string> + } + return %0#0, %0#1 : tensor<*x!tf_type.string>, tensor<*x!tf_type.string> +} + +// ----- + +// Serialized string: +// "\08\01\1A\01\01\22\01\00" +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 + +func.func @valid_MAXIMAL_sharding_device(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + %0, %c = tf_executor.island wraps "tf.Identity"(%arg0) {_tpu_replicate = "cluster", _XlaSharding = "\08\01\1A\01\01\22\01\00"} : (tensor) -> tensor + tf_executor.fetch %0 : tensor + } + return %0 : tensor +} +// ----- + +// Serialized string: +// "\08\01\1A\01\01\22\01\02" +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 2 + +func.func @invalid_MAXIMAL_sharding_device(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + // expected-error @+1 {{'tf.Identity' op TF2XLA TPU bridge input check: invalid sharding device 2 for num_cores_per_replica = 2}} + %0, %c = tf_executor.island wraps "tf.Identity"(%arg0) {_tpu_replicate = "cluster", _XlaSharding = "\08\01\1A\01\01\22\01\02"} : (tensor) -> tensor + tf_executor.fetch %0 : tensor + } + return %0 : tensor +} +// ----- + +// Serialized string: +// "\08\02\2a\08\08\01\1a\01\01\22\01\00\2a\08\08\01\1a\01\01\22\01\01" +// Proto debug string: +// type: TUPLE +// tuple_shardings { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// } +// tuple_shardings { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// } + +func.func @invalid_TUPLE_sharding_arity(%arg0: tensor) -> tensor { + %0 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + // expected-error @+1 {{'tf.Identity' op TF2XLA TPU bridge input check: invalid no. of tuple shardings 2 for arity = 1}} + %0, %c = tf_executor.island wraps "tf.Identity"(%arg0) {_tpu_replicate = "cluster", _XlaSharding = "\08\02\2a\08\08\01\1a\01\01\22\01\00\2a\08\08\01\1a\01\01\22\01\01"} : (tensor) -> tensor + tf_executor.fetch %0 : tensor + } + return %0 : tensor +} +// ----- + +// Serialized string: +// "\08\02\2a\08\08\01\1a\01\01\22\01\00\2a\08\08\01\1a\01\01\22\01\01" +// Proto debug string: +// type: TUPLE +// tuple_shardings { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// } +// tuple_shardings { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 1 +// } + +func.func @outfeed_enqueue_tuple_sharding_exception(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = tf_executor.graph { + %control = tf_executor.island wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_cores_per_replica = 2 : i64, num_replicas = 1 : i64, topology = "topology"} : () -> () + %0, %c0 = tf_executor.island wraps "tf.AddV2"(%arg0, %arg1) {_tpu_replicate = "cluster"} : (tensor, tensor) -> tensor + %c1 = tf_executor.island wraps "tf.OutfeedEnqueueTuple"(%arg0, %arg1) {_tpu_replicate = "cluster", _XlaSharding = "\08\02\2a\08\08\01\1a\01\01\22\01\00\2a\08\08\01\1a\01\01\22\01\01"} : (tensor, tensor) -> () + tf_executor.fetch %0 : tensor + } + return %0 : tensor +} + + +// ----- + +func.func @single_core_tpu(%arg0: tensor) -> () { + tf_executor.graph { + // expected-error @+1 {{found a single-core TPU graph}} + tf_executor.island wraps "tf.Identity"(%arg0) {_xla_compile_device_type = "TPU"} : (tensor) -> tensor + tf_executor.fetch + } + return +} + +// ----- + +// CHECK-LABEL: func @num_replicas_1 +func.func @num_replicas_1(%arg0: tensor) -> (tensor) { + %0 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 1, num_cores_per_replica = 1, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %ro, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> tensor + tf_executor.fetch %ro : tensor + } + return %0 : tensor +} + +// ----- +func.func @contians_InfeedDequeueTuple(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + // expected-warning @+1 {{TPU_REPLICATED_CORE:0 device is not supported for op = tf.InfeedDequeueTuple in TF2XLA MLIR Bridge}} + %infeed_output:3, %c2 = tf_executor.island wraps "tf.InfeedDequeueTuple"() {device = "/device:TPU_REPLICATED_CORE:0"} : () -> (tensor<3xi32>, tensor<4x?xf32>, tensor<*xi16>) + %ro:2, %c3 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- +func.func @graph_contains_v1_control_flow() { + tf_executor.graph { + // expected-warning @+1 {{ is v1 control flow op which is not supported in TF2XLA MLIR Bridge.}} + %control = tf_executor.ControlTrigger {} + tf_executor.fetch + } + func.return +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..0c1857ee7f8e6e734b877672b49cc9b596734258 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.cc @@ -0,0 +1,66 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h" + +#include + +#include "absl/strings/match.h" +#include "llvm/ADT/DenseSet.h" +#include "mlir/Support/TypeID.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +bool IsPotentialUnsupportedOp(Operation* op) { + static auto* ops = [] { + llvm::SmallDenseSet* ops_set = + new llvm::SmallDenseSet{ + TypeID::get(), + }; + return ops_set; + }(); + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return false; + + bool is_in_ops = ops->count(abstractOp->getTypeID()) != 0; + if (!is_in_ops) return false; + + std::string device = ""; + if (!op->hasAttr(kDeviceAttr)) return false; + device = op->getAttrOfType(kDeviceAttr).str(); + if (!absl::StrContains(device, kTpuReplicatedCoreZeroAttr)) return false; + op->emitWarning("TPU_REPLICATED_CORE:0 device is not supported for op = ") + << op->getName() << " in TF2XLA MLIR Bridge"; + + return true; +} + +bool HasV1ControlFlow(GraphOp graph) { + for (Operation& op : graph.GetBody().without_terminator()) { + auto island_op = llvm::dyn_cast(op); + if (!island_op) { + op.emitWarning() << " is v1 control flow op which is not supported in " + "TF2XLA MLIR Bridge."; + return true; + } + } + return false; +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..fc1b306051e6ae7c3af12c514337af4b9bb11048 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h @@ -0,0 +1,49 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_TPU_VALIDATE_INPUTS_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_TPU_VALIDATE_INPUTS_UTILS_H_ + +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +constexpr char kTpuReplicatedCoreZeroAttr[] = "TPU_REPLICATED_CORE:0"; + +using mlir::ModuleOp; +using mlir::Operation; +using mlir::StringAttr; +using mlir::TypeID; +using mlir::TF::InfeedDequeueTupleOp; +using mlir::TF::kDeviceAttr; +using mlir::tf_executor::GraphOp; + +bool IsPotentialUnsupportedOp(Operation* op); + +bool HasV1ControlFlow(GraphOp graph); + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_PASSES_TPU_VALIDATE_INPUTS_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..03108c7f4e35113effab07498ed4d2b294c7a449 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils_test.cc @@ -0,0 +1,94 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h" + +#include +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +using mlir::hlo::test::GetMlirModuleFromString; + +TEST(IsPotentialUnsupportedOp, ClusterOpReturnsFalse) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + auto cluster = mlir::tf_device::ClusterOp::create( + builder, mlir::UnknownLoc::get(&context), result_types); + cluster->dump(); + EXPECT_FALSE(IsPotentialUnsupportedOp(cluster)); +} + +TEST(IsPotentialUnsupportedOp, InfeedDequeueTupleOpReturnsTrue) { + mlir::MLIRContext context; + context.loadDialect(); + mlir::OwningOpRef module_ref = + mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); + mlir::OpBuilder builder(module_ref->getBodyRegion()); + + llvm::SmallVector result_types; + mlir::StringAttr _XlaSharding = mlir::StringAttr::get(&context, ""); + mlir::ArrayAttr layouts = mlir::ArrayAttr::get(&context, {}); + + auto infeed_dequeue_tuple = + InfeedDequeueTupleOp::create(builder, mlir::UnknownLoc::get(&context), + result_types, _XlaSharding, layouts); + + infeed_dequeue_tuple->setAttr( + kDeviceAttr, mlir::StringAttr::get(&context, kTpuReplicatedCoreZeroAttr)); + + EXPECT_TRUE(IsPotentialUnsupportedOp(infeed_dequeue_tuple)); +} + +TEST(HasV1ControlFlow, ReturnsTrue) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @graph_contains_v1_control_flow() { + tf_executor.graph { + %control = tf_executor.ControlTrigger {} + tf_executor.fetch + } + func.return + } + })"; + mlir::MLIRContext context; + context.loadDialect(); + auto module = GetMlirModuleFromString(kMlirModuleStr, &context); + + module->get().walk( + [&](GraphOp graph) { EXPECT_TRUE(HasV1ControlFlow(graph)); }); +} + +} // namespace + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.cc new file mode 100644 index 0000000000000000000000000000000000000000..d4351e98856b95745b012cec32e9d755481ee70e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.cc @@ -0,0 +1,75 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "absl/log/log.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_inputs_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +#define GEN_PASS_DEF_TPUVALIDATESESSIONINPUTSPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +using mlir::ModuleOp; + +struct TPUValidateSessionInputsPass + : public impl::TPUValidateSessionInputsPassBase< + TPUValidateSessionInputsPass> { + void runOnOperation() override; +}; + +void TPUValidateSessionInputsPass::runOnOperation() { + ModuleOp module = getOperation(); + bool success = true; + + module.walk([&](mlir::Operation* op) { + if (IsPotentialUnsupportedOp(op)) { + LOG(WARNING) << "Potential unsupported op: " + << op->getName().getStringRef().str() + << ". TF2XLA MLIR bridge does not guarantee to support it."; + } + if (!success) { + signalPassFailure(); + } + }); + + module.walk([&](GraphOp graph) { + if (HasV1ControlFlow(graph)) { + LOG(WARNING) << "TF2XLA MLIR bridge does not support v1 control flow." + << " Use at your own risk."; + } + if (!success) { + signalPassFailure(); + } + }); +} + +} // anonymous namespace + +std::unique_ptr> +CreateTPUValidateSessionInputsPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..9bcc81a0110fa4c948499cc781470ccc24600cfb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/tpu_validate_session_inputs.mlir @@ -0,0 +1,37 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-tpu-validate-session-inputs | FileCheck %s + +// CHECK-LABEL: func @does_not_contian_InfeedDequeueTuple +func.func @does_not_contian_InfeedDequeueTuple(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + %ro:2, %c2 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- +func.func @contians_InfeedDequeueTuple(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> (tensor, tensor) { + %0:2 = tf_executor.graph { + %control = tf_executor.island() wraps "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster", device = "/device:TPU:0", num_replicas = 2, topology = "topology"} : () -> () + %ri, %c0 = tf_executor.island wraps "tf.TPUReplicatedInput"(%arg0, %arg1) {index = 1 : i64, is_mirrored_variable = false, is_packed = false} : (tensor, tensor) -> tensor + %out, %c1 = tf_executor.island wraps "tf.opA"(%ri) {_tpu_replicate = "cluster"} : (tensor) -> tensor + // expected-warning @+1 {{TPU_REPLICATED_CORE:0 device is not supported for op = tf.InfeedDequeueTuple in TF2XLA MLIR Bridge}} + %infeed_output:3, %c2 = tf_executor.island wraps "tf.InfeedDequeueTuple"() {device = "/device:TPU_REPLICATED_CORE:0"} : () -> (tensor<3xi32>, tensor<4x?xf32>, tensor<*xi16>) + %ro:2, %c3 = tf_executor.island wraps "tf.TPUReplicatedOutput"(%out) : (tensor) -> (tensor, tensor) + tf_executor.fetch %ro#0, %ro#1 : tensor, tensor + } + return %0#0, %0#1 : tensor, tensor +} + +// ----- +func.func @graph_contains_v1_control_flow() { + tf_executor.graph { + // expected-warning @+1 {{ is v1 control flow op which is not supported in TF2XLA MLIR Bridge.}} + %control = tf_executor.ControlTrigger {} + tf_executor.fetch + } + func.return +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..00e4c06e00328b670aca0ef28077600505000baf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass.cc @@ -0,0 +1,80 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +#define GEN_PASS_DEF_VERIFYCLUSTERINGPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +using mlir::Operation; +using mlir::WalkResult; + +class VerifyClusteringPass + : public impl::VerifyClusteringPassBase { + public: + void runOnOperation() override; +}; + +void VerifyClusteringPass::runOnOperation() { + Operation* func_op = getOperation(); + + auto walk_result = func_op->walk([&](Operation* op) { + if (!tensorflow::tf2xla::internal::IsInBridgeAcceptableDialects(op)) { + std::string error = "op is in dialect " + + op->getDialect()->getNamespace().str() + + " not in tf functional dialect"; + op->emitError() << error; + return WalkResult::interrupt(); + } + + if (op->hasAttr(mlir::TF::kXlaOutsideCompilationAttr)) { + std::string error = + "op has outside compilation attribute _xla_outside_compilation which " + "is not allowed after clustering"; + op->emitError() << error; + return mlir::WalkResult::interrupt(); + } + + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + signalPassFailure(); + } +} +} // namespace + +std::unique_ptr> +CreateVerifyClusteringPass() { + return std::make_unique(); +} +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..fb16c85b8fa941e3a0a3c493106f8093264b907e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.cc @@ -0,0 +1,87 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "xla/tsl/platform/statusor.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::hlo::test::GetMlirModuleFromString; + +class VerifyClusteringPassTest : public testing::Test { + protected: + void CreateModule(const char* module_string) { + TF_ASSERT_OK_AND_ASSIGN(module_, + GetMlirModuleFromString(module_string, &context_)); + pm_ = std::make_unique(&context_); + pm_->addNestedPass(CreateVerifyClusteringPass()); + } + + mlir::LogicalResult Run() { return pm_->run(module_.get()); } + + private: + mlir::MLIRContext context_; + mlir::OwningOpRef module_; + std::unique_ptr pm_; +}; + +TEST_F(VerifyClusteringPassTest, OnlyTfFunctionalPasses) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + return %0 : tensor<1xi32> + } + })"; + CreateModule(kMlirModuleStr); + + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); +} + +TEST_F(VerifyClusteringPassTest, NotTfFunctionalFails) { + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<3x32x32x3xf32> { + %0 = mhlo.constant dense<2.550000e+02> : tensor<3x32x32x3xf32> + return %0 : tensor<3x32x32x3xf32> + } + })"; + CreateModule(kMlirModuleStr); + + auto result = Run(); + + EXPECT_TRUE(result.failed()); +} + +} // namespace +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3260fd33eb357a70038ee4d09160e904c1a42cd8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_clustering_pass_test.mlir @@ -0,0 +1,26 @@ +// RUN: tf-opt -verify-clustering-pass -split-input-file -verify-diagnostics %s | FileCheck %s +// Tests the VerifyClusteringPass Pass, ensures that an error is thrown when validation fails. + +func.func @testNotTfDialect(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // expected-error@below {{op is in dialect chlo not in tf functional dialect}} + %0 = "chlo.broadcast_add"(%arg0, %arg1) {broadcast_dimensions = array} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} + +// ----- + +// CHECK-LABEL: func @testTFDialect +func.func @testTFDialect(%arg0: tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> { + %0 = "tf.Identity"(%arg0) : (tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + + +// ----- + +func.func @testTFDialect(%arg0: tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> { + // expected-error@below {{op has outside compilation attribute _xla_outside_compilation which is not allowed after clustering}} + %0 = "tf.Identity"(%arg0) {_xla_outside_compilation = "cluster1"}: (tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..964cdce2f100c9ee7cad4a8978b3fd8e7933c3c8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass.cc @@ -0,0 +1,84 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { +using mlir::Operation; +using mlir::OperationPass; +using mlir::WalkResult; +using mlir::func::FuncOp; + +#define GEN_PASS_DEF_VERIFYINPUTDIALECTTOEXECUTORPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h.inc" + +class VerifyInputDialectToExecutorPass + : public impl::VerifyInputDialectToExecutorPassBase< + VerifyInputDialectToExecutorPass> { + public: + void runOnOperation() override; +}; + +bool IsTfDeviceClusterFuncOp(Operation* op) { + std::string kClusterFuncOpName = "tf_device.cluster_func"; + return op->getName().getStringRef().str() == kClusterFuncOpName; +} + +void VerifyInputDialectToExecutorPass::runOnOperation() { + Operation* func_op = getOperation(); + + auto walk_result = func_op->walk([&](Operation* op) { + if (!tensorflow::tf2xla::internal::IsInBridgeAcceptableDialects(op)) { + std::string error = "op is in dialect " + + op->getDialect()->getNamespace().str() + + " which is not an accepted dialect"; + op->emitError() << error; + return WalkResult::interrupt(); + } + + if (IsTfDeviceClusterFuncOp(op)) { + std::string error = + "failed TF functional to executor validation, op " + "tf_device.cluster_func is not allowed"; + op->emitError() << error; + return WalkResult::interrupt(); + } + + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + signalPassFailure(); + } +} + +} // namespace + +std::unique_ptr> +CreateVerifyInputDialectToExecutorPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass_test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass_test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..dbc157954217ebbef61fea916fba43b1025d4668 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/verify_input_dialect_to_executor_pass_test.mlir @@ -0,0 +1,38 @@ +// RUN: tf-opt -verify-input-dialect-to-executor-pass -split-input-file -verify-diagnostics %s | FileCheck %s +// Tests the VerifyClusteringPass Pass, ensures that an error is thrown when validation fails. + +// ----- + +// CHECK-LABEL: func @testNoClusterFuncOpPasses +func.func @testNoClusterFuncOpPasses(%arg0: tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> { + %0 = "tf.Identity"(%arg0) : (tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + +// ----- + +func.func @_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +func.func @testClusterFuncOpFails(%arg0: tensor) -> tensor { + // expected-error@below {{failed TF functional to executor validation, op tf_device.cluster_func is not allowed}} + %cluster = "tf_device.cluster_func"(%arg0) {func = @_func} : (tensor) -> tensor + func.return %cluster : tensor +} + +// ----- + +// CHECK-LABEL: func @testTFDialect +func.func @testTFDialect(%arg0: tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> { + %0 = "tf.Identity"(%arg0) : (tensor<4x?x!tf_type.stringref>) -> tensor<4x2x!tf_type.string> + func.return %0 : tensor<4x2x!tf_type.string> +} + +// ----- + +func.func @testNotTfDialect(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // expected-error@below {{op is in dialect chlo which is not an accepted dialect}} + %0 = "chlo.broadcast_add"(%arg0, %arg1) {broadcast_dimensions = array} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_broadcast.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_broadcast.cc new file mode 100644 index 0000000000000000000000000000000000000000..30a5081d1fd48fc719892994bed4f8045408cedb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_broadcast.cc @@ -0,0 +1,360 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_rewrite_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "tensorflow/core/ir/types/dialect.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using llvm::dyn_cast; +using mlir::Block; +using mlir::BlockArgument; +using mlir::DenseIntElementsAttr; +using mlir::failure; +using mlir::Location; +using mlir::LogicalResult; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OperationPass; +using mlir::OpOperand; +using mlir::RankedTensorType; +using mlir::StringAttr; +using mlir::success; +using mlir::Type; +using mlir::Value; +using mlir::ValueRange; +using mlir::WalkResult; +using mlir::func::FuncOp; +using mlir::TF::ConstOp; +using mlir::TF::IdentityOp; +using mlir::TF::ShapeAttr; +using mlir::TF::TPUDummyInputOp; +using mlir::TF::XlaAllReduceOp; +using mlir::tf_device::ClusterOp; +using mlir::tf_device::LaunchOp; +using mlir::tf_device::ParallelExecuteOp; +using mlir::tf_device::ReplicateOp; + +#define GEN_PASS_DEF_XLABROADCASTPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +struct XlaBroadcast : public impl::XlaBroadcastPassBase { + void runOnOperation() override; +}; + +// Returns true iff the broadcast val can be substituted with an XlaAllReduce. +// Sets `zero_type` and `shape` as params for the dummy zeros that will be +// created. +bool GetDummyParams(OpBuilder& builder, Value val_bcast, Type& zero_type, + ShapeAttr& shape) { + Type type = val_bcast.getType(); + Type elem_type = getElementTypeOrSelf(type); + // Xla's all_reduce legalizer bitcasts to 32 bits, so only + // element types size <= 4 bytes are supported. + if (elem_type.isBF16() || elem_type.isF16() || elem_type.isTF32() || + elem_type.isF32()) { + zero_type = type; + } else { + return false; + } + if (auto ranked_type = dyn_cast(type)) { + llvm::ArrayRef type_shape = ranked_type.getShape(); + for (int64_t i : type_shape) { + if (i < 0) return false; + } + shape = ShapeAttr::get(builder.getContext(), type_shape); + } else { + return false; + } + return true; +} + +// Create a dummy zero to be fed locally from the host to the TPUExecute. +Value CreateZeroInput(Location loc, OpBuilder& builder, Type zero_type, + ShapeAttr shape_attr) { + TPUDummyInputOp tpu_dummy_input = + TPUDummyInputOp::create(builder, loc, zero_type, shape_attr); + tpu_dummy_input->setAttr(kICIWeightDistributionMlirBridgeMarker, + builder.getBoolAttr(true)); + return tpu_dummy_input; +} + +// Add parallel collection of inputs to the replicated op. +LogicalResult AppendReplicatedInput(OpBuilder& builder, ReplicateOp replicate, + ValueRange inputs, Type type, + BlockArgument& block_arg) { + // check that inputs length is same as num_replicas. + if (inputs.size() != replicate.getN()) { + return replicate.emitError() + << "Expected numper of inputs (" << inputs.size() + << ") to append to replicate to be num_replicas (" + << replicate.getN() << ")"; + } + + // add block arg to region. This is in $body. + unsigned block_arg_idx = replicate.GetNumReplicatedBlockArguments(); + Block& block = replicate.GetBody(); + block_arg = block.insertArgument(block_arg_idx, type, replicate.getLoc()); + + // add to $replicated_inputs. This also updates OperandSegmentSizeAttr. + replicate.getReplicatedInputsMutable().append(inputs); + + return success(); +} + +// Insert an XlaAllReduce. +Value CreateAllReduce(ReplicateOp replicate, OpBuilder& builder, + Value block_arg) { + // This group_assignment is a list of all replicas. This says that the + // reduction sent to each replica is over all replicas. + uint32_t num_replicas = replicate.getN(); + llvm::SmallVector group_assignment_val; + for (int i = 0; i < num_replicas; ++i) group_assignment_val.push_back(i); + Value group_assignment = ConstOp::create( + builder, block_arg.getLoc(), + DenseIntElementsAttr::get( + RankedTensorType::get({1, num_replicas}, builder.getIntegerType(32)), + group_assignment_val)); + + StringAttr reduce_op = builder.getStringAttr("Add"); + StringAttr mode = builder.getStringAttr("CrossReplica"); + return XlaAllReduceOp::create(builder, block_arg.getLoc(), + block_arg.getType(), block_arg, + group_assignment, reduce_op, mode); +} + +// Creates a missing attribute error message. +std::string CreateMissingAttributeMsg(llvm::StringRef attribute) { + return llvm::formatv("requires attribute '{0}'", attribute).str(); +} + +LogicalResult GetTpuDeviceAssignment( + ClusterOp cluster, ReplicateOp replicate, mlir::ModuleOp module, + absl::StatusOr& status_or_tpu_device_assignment) { + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(module, &devices))) return failure(); + + uint32_t num_replicas = replicate.getN(); + + auto num_cores_per_replica_attr = cluster->getAttrOfType( + tensorflow::kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) + return cluster.emitOpError( + CreateMissingAttributeMsg(tensorflow::kNumCoresPerReplicaAttr)); + int num_cores_per_replica = num_cores_per_replica_attr.getInt(); + + auto topology_attr = cluster->getAttrOfType("topology"); + if (!topology_attr) + return cluster.emitOpError( + CreateMissingAttributeMsg(tensorflow::kTopologyAttr)); + + auto device_assignment_attr = cluster->getAttrOfType( + tensorflow::kDeviceAssignmentAttr); + if (!device_assignment_attr) + return cluster.emitOpError(llvm::formatv("requires attribute '{0}'", + tensorflow::kDeviceAssignmentAttr) + .str()); + + auto status_or_device_coodinates = + tensorflow::GetDeviceCoordinates(device_assignment_attr); + if (!status_or_device_coodinates.ok()) + return cluster.emitError() + << "error in fetching tpu device coordinates: " + << status_or_device_coodinates.status().message(); + + status_or_tpu_device_assignment = + tensorflow::GetTPUCompilationAndExecutionDevices( + devices.device_names(), num_replicas, num_cores_per_replica, + topology_attr.getValue(), status_or_device_coodinates.value()); + return success(); +} + +// Move a broadcast into the XLA cluster, converting it to an XlaAllReduce. This +// skips if the element type is not known to be valid for XlaAllReduce. +LogicalResult MoveBroadcastToCluster(OpBuilder& builder, + OpBuilder& inner_builder, + ClusterOp cluster, ReplicateOp replicate, + llvm::DenseMap& orig_to_new, + Value val_bcast, mlir::ModuleOp module) { + Type zero_type; + ShapeAttr shape_attr; + if (!GetDummyParams(builder, val_bcast, zero_type, shape_attr)) + return success(); + llvm::SmallVector inputs; + inputs.push_back(val_bcast); + uint32_t num_replicas = replicate.getN(); + + absl::StatusOr status_or_tpu_device_assignment; + if (failed(GetTpuDeviceAssignment(cluster, replicate, module, + status_or_tpu_device_assignment))) { + return failure(); + } + if (!status_or_tpu_device_assignment.ok()) + return cluster.emitError() + << "error in fetching TPU compilation/execution devices: " + << status_or_tpu_device_assignment.status().message(); + + llvm::ArrayRef> + tpu_devices = status_or_tpu_device_assignment.value().tpu_devices; + + std::unordered_map host_to_fill; + + for (uint32_t replica = 1; replica < num_replicas; ++replica) { + std::string host = tpu_devices[replica][0].host; + if (host_to_fill.find(host) == host_to_fill.end()) { + host_to_fill[host] = + CreateZeroInput(val_bcast.getLoc(), builder, zero_type, shape_attr); + } + inputs.push_back(host_to_fill[host]); + } + + BlockArgument block_arg; + if (failed(AppendReplicatedInput(builder, replicate, inputs, + val_bcast.getType(), block_arg))) { + return failure(); + } + + OpBuilder before_cluster_builder(cluster); + IdentityOp assigned_id = + IdentityOp::create(before_cluster_builder, val_bcast.getLoc(), + block_arg.getType(), block_arg); + assigned_id->setAttr(kICIWeightDistributionMlirBridgeMarker, + before_cluster_builder.getBoolAttr(true)); + std::string device = tensorflow::GetDeviceAliasForHostOfLogicalCore(0); + LaunchOp launch = tensorflow::WrapOpInLaunch( + &before_cluster_builder, val_bcast.getLoc(), assigned_id, device); + + launch->setAttr(kICIWeightDistributionMlirBridgeMarker, + before_cluster_builder.getBoolAttr(true)); + + Value all_reduce = + CreateAllReduce(replicate, inner_builder, launch.getResult(0)); + + orig_to_new[val_bcast] = all_reduce; + return success(); +} + +// Move all suitable broadcasts across replicas to the `cluster` into the +// `cluster`. +LogicalResult MoveAllBroadcastsToCluster(ClusterOp cluster, + ReplicateOp replicate, + mlir::ModuleOp module) { + // TODO(b/325153657): Fix tpu_rewrite_pass so the parallel_execute case does + // not need to be skipped. + if (cluster->getParentOfType()) return success(); + + auto num_cores_per_replica_attr = cluster->getAttrOfType( + tensorflow::kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) + return cluster.emitOpError( + CreateMissingAttributeMsg(tensorflow::kNumCoresPerReplicaAttr)); + + llvm::SetVector bcasts; + cluster->walk([&](Operation* op) { + if (op == cluster) return WalkResult::advance(); + for (auto operand : op->getOperands()) { + Operation* scope = operand.getParentBlock()->getParentOp(); + if (scope->isProperAncestor(replicate)) { + bcasts.insert(operand); + } + } + return WalkResult::advance(); + }); + OpBuilder builder(replicate); + OpBuilder inner_builder = OpBuilder::atBlockBegin(&cluster.getBody().front()); + // mapping from original operand to new XlaAllReduce op. + llvm::DenseMap orig_to_new; + + for (Value bcast : bcasts) { + if (failed(MoveBroadcastToCluster(builder, inner_builder, cluster, + replicate, orig_to_new, bcast, module))) { + return failure(); + } + } + + cluster->walk([&](Operation* op) { + if (op == cluster) return WalkResult::advance(); + for (OpOperand& operand : op->getOpOperands()) { + if (orig_to_new.count(operand.get())) { + operand.assign(orig_to_new[operand.get()]); + } + } + return WalkResult::advance(); + }); + + return success(); +} + +void XlaBroadcast::runOnOperation() { + FuncOp func = getOperation(); + mlir::ModuleOp module = func->getParentOfType(); + if (!module) return signalPassFailure(); + func.walk([&](ClusterOp cluster) { + if (auto replicate = cluster->getParentOfType()) { + if (failed(MoveAllBroadcastsToCluster(cluster, replicate, module))) { + return signalPassFailure(); + } + } + }); +} + +} // namespace + +std::unique_ptr> CreateXlaBroadcastPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_cluster_formation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_cluster_formation.cc new file mode 100644 index 0000000000000000000000000000000000000000..80818d087ba15792445393779544eb019a6f1adc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/passes/xla_cluster_formation.cc @@ -0,0 +1,145 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/call_graph_util.h" +#include "tensorflow/core/common_runtime/inline_function_utils.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +using mlir::Block; +using mlir::CallInterfaceCallable; +using mlir::CallOpInterface; +using mlir::ModuleOp; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OperationPass; +using mlir::SymbolTable; +using mlir::SymbolTableCollection; +using mlir::SymbolUserOpInterface; +using mlir::func::FuncOp; + +#define GEN_PASS_DEF_XLACLUSTERFORMATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h.inc" + +// Outlines partitioned call ops with `_XlaMustCompile` to device clusters. +struct XlaClusterFormationPass + : public impl::XlaClusterFormationPassBase { + void runOnOperation() override; +}; + +void EncapsulatePartitionedCall(Operation *call_op, + mlir::StringAttr callee_name) { + OpBuilder builder(call_op); + auto cluster = mlir::tf_device::ClusterOp::create(builder, call_op->getLoc(), + call_op->getResultTypes()); + cluster.getBody().push_back(new Block); + call_op->replaceAllUsesWith(cluster.getResults()); + call_op->moveBefore(&cluster.GetBody(), cluster.GetBody().end()); + builder.setInsertionPointToEnd(&cluster.GetBody()); + mlir::tf_device::ReturnOp::create(builder, call_op->getLoc(), + call_op->getResults()); + // Propagate necessary attributes to the cluster so that when it's outlined, + // the function will have correct attributes. + mlir::TF::CopyDeviceAndUnderscoredAttributes(call_op, cluster); + cluster->setAttr(mlir::TF::kClusterOutlinedFunctionNameAttr, callee_name); + cluster->setAttr(mlir::TF::kAllowSoftPlacementAttr, + builder.getBoolAttr(true)); +} + +// Encapsulate the first partitioned call that can be reached from +// `func` and is with compilation markers in a device cluster. For nested calls, +// if the outermost one has the markers, encapsulates the outermost call and +// returns. Otherwise, we'll keep going through inner calls until we found one. +mlir::LogicalResult EncapsulateFirstXlaCompilablePartitionedCalls( + FuncOp func, SymbolTableCollection &symbol_table_collection, + SymbolTable &symtab) { + auto has_no_compile_device_type = [](SymbolUserOpInterface op) { + return !op->hasAttr(mlir::TF::kCompileDeviceTypeAttr); + }; + + mlir::OpBuilder builder(func.getContext()); + auto noinline_attr_name = absl::StrCat("tf.", tensorflow::kNoInlineAttr); + llvm::SmallVector noinline_pcall_ops, + outermost_pcall_ops; + if (mlir::failed( + mlir::GetOpsOfTypeUntilMiss( + func, symtab, /*predicate*/ has_no_compile_device_type, + /*hits*/ noinline_pcall_ops, + /*first_misses*/ outermost_pcall_ops))) { + return mlir::failure(); + } + // Cluster outermost partitioned calls with _xla_compile_device_type + // attribute. + for (auto &pcall_op : outermost_pcall_ops) { + auto call = llvm::cast(pcall_op.getOperation()); + CallInterfaceCallable callable = call.getCallableForCallee(); + auto sym = callable.get(); + EncapsulatePartitionedCall(pcall_op, sym.getRootReference()); + } + // Partitioned calls are executed asynchronous. The calls outside of + // device clusters therefore should not be inlined to perserve run-time + // performance. + for (auto &pcall_op : noinline_pcall_ops) { + auto call = llvm::cast(pcall_op.getOperation()); + auto callee = llvm::cast( + call.resolveCallableInTable(&symbol_table_collection)); + callee->setAttr(noinline_attr_name, builder.getBoolAttr(true)); + } + return mlir::success(); +} + +void XlaClusterFormationPass::runOnOperation() { + ModuleOp module = getOperation(); + SymbolTableCollection symbol_table_collection; + SymbolTable symtab = symbol_table_collection.getSymbolTable(module); + llvm::SmallVector entry_funcs = GetEntryFunctions(module); + for (auto &entry_func : entry_funcs) { + if (EncapsulateFirstXlaCompilablePartitionedCalls( + entry_func, symbol_table_collection, symtab) + .failed()) { + return signalPassFailure(); + } + } +} + +std::unique_ptr> CreateXlaClusterFormationPass() { + return std::make_unique(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/reproducer.proto b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/reproducer.proto new file mode 100644 index 0000000000000000000000000000000000000000..d4fa78bee3a1949da1b6b7e9a15a5610d522374f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/reproducer.proto @@ -0,0 +1,38 @@ +// Copyright 2024 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package tensorflow.mlir.tf2xla.internal; + +import "tensorflow/core/framework/function.proto"; +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/protobuf/tpu/compile_metadata.proto"; + +option cc_enable_arenas = true; + +// The input to LegalizeMlirToHlo to reproduce a legalization failure. +message LegalizeMlirToHloReproducer { + // The TPUCompileMetadataProto used during legalization. + tensorflow.tpu.TPUCompileMetadataProto compile_metadata = 1; + + // The input shapes used during legalization. + repeated tensorflow.TensorShapeProto input_shapes = 2; + + oneof computation { + // The MLIR module to be legalized. + string mlir_module = 3; + // The FunctionDefLibrary to be legalized. + tensorflow.FunctionDefLibrary function_def_library = 4; + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h new file mode 100644 index 0000000000000000000000000000000000000000..aeb2aefbfc92585d841a196e98f03d728aa96e60 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h @@ -0,0 +1,91 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_TEST_MATCHERS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_TEST_MATCHERS_H_ + +#include +#include "absl/status/statusor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "xla/tsl/platform/statusor.h" + +template +bool WasGraphAnalysisFailure(const absl::StatusOr& status) { + return (status.status() == + tensorflow::CompileToHloGraphAnalysisFailedError()); +} + +/* The third party version of the Graph Analysis always returns disabled so + * these matchers short circuit on that error. */ +MATCHER(IsOkOrFiltered, + "Status was OK or equal to the Graph Analysis failure") { + bool is_ok = arg.ok(); + auto graph_analysis_failure = WasGraphAnalysisFailure(arg); + return testing::ExplainMatchResult( + testing::IsTrue(), is_ok || graph_analysis_failure, result_listener); +} + +MATCHER_P2(IncrementedOrFiltered, metric, value, + "Metric was incremented by value or Status equal to the Graph " + "Analysis failure") { + auto graph_analysis_failure = WasGraphAnalysisFailure(arg); + if (graph_analysis_failure) { + return testing::ExplainMatchResult(testing::IsTrue(), + graph_analysis_failure, result_listener); + } + return testing::ExplainMatchResult(testing::Eq(metric), value, + result_listener); +} + +MATCHER_P(ComputationProtoContains, regex, + "If not a Graph Analysis failure then matches the computation result " + "with the regex") { + auto graph_analysis_failure = WasGraphAnalysisFailure(arg); + if (graph_analysis_failure) { + return testing::ExplainMatchResult(testing::IsTrue(), + graph_analysis_failure, result_listener); + } + auto proto = arg.value().computation->proto().DebugString(); + return testing::ExplainMatchResult(testing::ContainsRegex(regex), proto, + result_listener); +} + +MATCHER_P(XlaComputationProtoContains, regex, + "If not a Graph Analysis failure then matches the computation result " + "with the regex") { + auto graph_analysis_failure = WasGraphAnalysisFailure(arg); + if (graph_analysis_failure) { + return testing::ExplainMatchResult(testing::IsTrue(), + graph_analysis_failure, result_listener); + } + auto proto = arg.value().proto().DebugString(); + return testing::ExplainMatchResult(testing::ContainsRegex(regex), proto, + result_listener); +} + +MATCHER_P( + HasMlirModuleWith, expected, + "If not a Graph Analysis failure then matches the mlir module result") { + auto graph_analysis_failure = WasGraphAnalysisFailure(arg); + if (graph_analysis_failure) { + return testing::ExplainMatchResult(testing::IsTrue(), + graph_analysis_failure, result_listener); + } + auto actual = arg.value(); + return testing::ExplainMatchResult(testing::ContainsRegex(expected), actual, + result_listener); +} + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_TEST_MATCHERS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..166e60bbbea81a68908f349f6939463b96c0812b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/test_matchers_test.cc @@ -0,0 +1,136 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/test_matchers.h" + +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/service/hlo.pb.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" +#include "tensorflow/core/lib/monitoring/counter.h" + +namespace { +using ::tensorflow::monitoring::testing::CellReader; +using ::testing::Not; + +constexpr char kMetric[] = "/tensorflow/metric"; +auto* counter = + tensorflow::monitoring::Counter<1>::New(kMetric, "description", "status"); +constexpr char kOkStatus[] = "ok"; +const int kArbitraryIntResult = 37; + +template +tsl::StatusOr success(T t) { + return t; +} +absl::StatusOr success() { return kArbitraryIntResult; } +template +tsl::StatusOr filtered(T t) { + return tsl::StatusOr(tensorflow::CompileToHloGraphAnalysisFailedError()); +} +absl::StatusOr filtered() { return filtered(kArbitraryIntResult); } +absl::StatusOr failed() { + return absl::StatusOr(absl::InternalError("fail")); +} + +TEST(TestUtil, MatchesOk) { ASSERT_THAT(success(), IsOkOrFiltered()); } + +TEST(TestUtil, DoesntMatchesFailure) { + ASSERT_THAT(failed(), Not(IsOkOrFiltered())); +} + +TEST(TestUtil, MatchesFiltered) { ASSERT_THAT(filtered(), IsOkOrFiltered()); } + +TEST(TestUtil, IncrementsOk) { + CellReader reader(kMetric); + counter->GetCell(kOkStatus)->IncrementBy(1); + + ASSERT_THAT(success(), IncrementedOrFiltered(reader.Delta(kOkStatus), 1)); +} + +TEST(TestUtil, FilteredDoesntIncrementsOk) { + CellReader reader(kMetric); + + ASSERT_THAT(filtered(), IncrementedOrFiltered(reader.Delta(kOkStatus), 1)); +} + +TEST(TestUtil, FailureDoesntMatchIncrement) { + CellReader reader(kMetric); + + ASSERT_THAT(failed(), Not(IncrementedOrFiltered(reader.Delta(kOkStatus), 1))); +} + +tensorflow::XlaCompilationResult CreateXlaComputationResult( + const char* hlo_name) { + auto result = tensorflow::XlaCompilationResult(); + xla::HloModuleProto hlo; + hlo.set_name(hlo_name); + result.computation = std::make_shared(hlo); + return result; +} + +TEST(TestUtil, ComputationContainsOk) { + constexpr char arbitrary_hlo[] = "arbitrary_hlo"; + auto result = CreateXlaComputationResult(arbitrary_hlo); + + ASSERT_THAT(success(result), ComputationProtoContains(arbitrary_hlo)); +} + +TEST(TestUtil, ComputationDoesNotContain) { + constexpr char arbitrary_hlo[] = "arbitrary_hlo"; + constexpr char bad_hlo[] = "bad_hlo"; + auto result = CreateXlaComputationResult(arbitrary_hlo); + + ASSERT_THAT(success(result), Not(ComputationProtoContains(bad_hlo))); +} + +TEST(TestUtil, ComputationDoesNotContainFiltered) { + constexpr char arbitrary_hlo[] = "arbitrary_hlo"; + constexpr char bad_hlo[] = "bad_hlo"; + auto result = CreateXlaComputationResult(arbitrary_hlo); + + ASSERT_THAT(filtered(result), ComputationProtoContains(bad_hlo)); +} + +TEST(TestUtil, MlirModuleHas) { + constexpr char arbirary_mlir[] = "arbirary_mlir"; + + ASSERT_THAT(success(arbirary_mlir), HasMlirModuleWith(arbirary_mlir)); +} + +TEST(TestUtil, MlirModuleDoesNotHave) { + constexpr char arbirary_mlir[] = "arbirary_mlir"; + constexpr char bad_mlir[] = "bad_mlir"; + + ASSERT_THAT(success(arbirary_mlir), Not(HasMlirModuleWith(bad_mlir))); +} + +TEST(TestUtil, MlirModuleDoesNotHaveFiltered) { + constexpr char arbirary_mlir[] = "arbirary_mlir"; + constexpr char bad_mlir[] = "bad_mlir"; + + ASSERT_THAT(filtered(arbirary_mlir), HasMlirModuleWith(bad_mlir)); +} + +} // namespace diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/testdata/dead_const.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/testdata/dead_const.mlir new file mode 100644 index 0000000000000000000000000000000000000000..dd2ff6263a663e5881972ee898ef5bbba8765053 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/testdata/dead_const.mlir @@ -0,0 +1,6 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () { + %s0 = "tf.Const"() {value = dense<[4, 1]> : tensor<2xi32>} : () -> tensor<2xi32> + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..f555eeb671fc9b0c1e17e2437de0fdcbfb984c79 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/BUILD @@ -0,0 +1,79 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tf2xla/internal:__subpackages__", + ], + licenses = ["notice"], +) + +cc_library( + name = "dialect_detection_utils", + srcs = [ + "dialect_detection_utils.cc", + ], + hdrs = [ + "dialect_detection_utils.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:framework", + "//tensorflow/core/transforms/toposort:Pass", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +tf_cc_test( + name = "dialect_detection_utils_test", + srcs = ["dialect_detection_utils_test.cc"], + deps = [ + ":dialect_detection_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@stablehlo//:chlo_ops", + ], +) + +cc_library( + name = "test_metadata_config", + testonly = True, + srcs = ["test_metadata_config.cc"], + hdrs = ["test_metadata_config.h"], + visibility = [ + "//tensorflow/compiler/mlir/tf2xla/api:__subpackages__", + ], + deps = [ + "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:xla_tpu_device", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_tpu_backend_registration", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/core:framework", + "//tensorflow/core:test_main", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tpu/kernels/xla:host_compute_ops", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@xla//xla:shape_util", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo:hlo_dialect_registration", + "@xla//xla/tsl/platform:errors", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..6085cce90c54743c233cc41f5f3e74dcce6377a3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.cc @@ -0,0 +1,45 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h" + +#include +#include + +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +bool IsInBridgeAcceptableDialects(mlir::Operation* op) { + const std::set kBuiltinNamespaces = {"func", "return", + "builtin"}; + const std::set kBridgeAcceptableNamespaces = {"tf", "tf_device"}; + bool isInDefaulNamespaces = + kBuiltinNamespaces.find(op->getDialect()->getNamespace().str()) != + kBuiltinNamespaces.end(); + bool isInBridgeAcceptableNamespaces = + kBridgeAcceptableNamespaces.find( + op->getDialect()->getNamespace().str()) != + kBridgeAcceptableNamespaces.end(); + return isInDefaulNamespaces || isInBridgeAcceptableNamespaces; +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..fb8224d3506c92d391eb6f0083fc02cce159c7d6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h @@ -0,0 +1,33 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_DIALECT_DETECTION_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_DIALECT_DETECTION_UTILS_H_ + +#include "mlir/IR/Operation.h" // from @llvm-project + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Returns true if the op has a valid namespace during clustering & tf dialect +// to executor components of the Bridge. +bool IsInBridgeAcceptableDialects(mlir::Operation* op); + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_DIALECT_DETECTION_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..09a661c66ae9988a4c0a3c266f1000edd7cb20f7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils_test.cc @@ -0,0 +1,76 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/dialect_detection_utils.h" + +#include +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +namespace { + +using mlir::MLIRContext; +using mlir::OpBuilder; +using mlir::Operation; +using mlir::OperationState; +using mlir::UnknownLoc; +using mlir::chlo::ChloDialect; +using mlir::TF::TensorFlowDialect; +using tensorflow::tf2xla::internal::IsInBridgeAcceptableDialects; + +class SharedUtilsTest : public ::testing::Test {}; + +TEST_F(SharedUtilsTest, IsInFunctionalDialectPasses) { + MLIRContext context; + context.loadDialect(); + OpBuilder opBuilder(&context); + OperationState state(UnknownLoc::get(opBuilder.getContext()), + /*OperationName=*/"tf.Const"); + mlir::Operation* op = Operation::create(state); + + bool result = IsInBridgeAcceptableDialects(op); + + EXPECT_TRUE(result); + op->destroy(); +} + +TEST_F(SharedUtilsTest, IsInFunctionalDialectFails) { + MLIRContext context; + context.loadDialect(); + OpBuilder opBuilder(&context); + OperationState state(UnknownLoc::get(opBuilder.getContext()), + /*OperationName=*/"chlo.broadcast_add"); + Operation* op = Operation::create(state); + + bool result = IsInBridgeAcceptableDialects(op); + + EXPECT_FALSE(result); + op->destroy(); +} + +} // namespace +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.cc new file mode 100644 index 0000000000000000000000000000000000000000..6fff92e7dcf0954b6c79cde34f90f2502e7064af --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.cc @@ -0,0 +1,107 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h" + +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/mlir_hlo/mhlo/IR/register.h" +#include "xla/shape.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { +namespace { + +constexpr char kEntryFuncName[] = "main"; + +absl::Status SetupArguments(mlir::ModuleOp module, + std::vector& arg_shapes, + tpu::TPUCompileMetadataProto& metadata_proto) { + auto main_fn = module.lookupSymbol(kEntryFuncName); + if (!main_fn) { + return absl::InternalError("Could not find main function in MLIR Module."); + } + + mlir::FunctionType func_type = main_fn.getFunctionType(); + for (auto input_type : func_type.getInputs()) { + tensorflow::TensorShape tensor_shape; + xla::Shape xla_shape = xla::TypeToShape(input_type); + TF_RETURN_IF_ERROR(tensorflow::TensorShape::BuildTensorShape( + xla_shape.dimensions(), &tensor_shape)); + arg_shapes.emplace_back(tensor_shape); + + DataType dtype; + TF_RETURN_IF_ERROR(ConvertToDataType(input_type, &dtype)); + + auto metadata_arg = metadata_proto.add_args(); + metadata_arg->set_kind(tpu::TPUCompileMetadataProto::Arg::PARAMETER); + metadata_arg->set_dtype(dtype); + } + + return absl::OkStatus(); +} + +absl::Status SetupReturnValues(mlir::ModuleOp module, + tpu::TPUCompileMetadataProto& metadata_proto) { + auto main_fn = module.lookupSymbol(kEntryFuncName); + if (!main_fn) { + return absl::InternalError("Could not find main function in MLIR Module."); + } + + int func_results = main_fn.getFunctionType().getNumResults(); + for (int i = 0; i < func_results; i++) { + metadata_proto.add_retvals(); + } + + return absl::OkStatus(); +} + +} // namespace + +absl::Status ConfigureMetadata(absl::string_view mlir_module_str, + std::vector& arg_shapes, + tpu::TPUCompileMetadataProto& metadata_proto) { + mlir::DialectRegistry registry; + mlir::RegisterCommonToolingDialects(registry); + mlir::MLIRContext context(registry); + mlir::OwningOpRef mlir_module; + + TF_RETURN_IF_ERROR( + DeserializeMlirModule(mlir_module_str, &context, &mlir_module)); + TF_RETURN_IF_ERROR(SetupReturnValues(*mlir_module, metadata_proto)); + TF_RETURN_IF_ERROR(SetupArguments(*mlir_module, arg_shapes, metadata_proto)); + + return absl::OkStatus(); +} + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h new file mode 100644 index 0000000000000000000000000000000000000000..a2a2b758ecbe9e2750886b55d295667a2be6b5a4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/internal/utils/test_metadata_config.h @@ -0,0 +1,41 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_TEST_METADATA_CONFIG_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_TEST_METADATA_CONFIG_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace tf2xla { +namespace internal { + +// Fills in arg_shapes and metadata_proto with appropriate values based on the +// input mlir module. +absl::Status ConfigureMetadata(absl::string_view mlir_module_str, + std::vector& arg_shapes, + tpu::TPUCompileMetadataProto& metadata_proto); + +} // namespace internal +} // namespace tf2xla +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_INTERNAL_UTILS_TEST_METADATA_CONFIG_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.cc new file mode 100644 index 0000000000000000000000000000000000000000..826e82856c75f7ca73825c8ae9dc108c5685abe3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.cc @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.h" + +#include + +#include "tensorflow/compiler/jit/flags.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace tensorflow { + +MlirBridgeRolloutPolicy GetMlirBridgeRolloutPolicy( + const tensorflow::Graph& graph, + const FunctionLibraryDefinition* function_library, + std::optional config_proto, + bool is_supported_by_replicated_brige, bool is_v1_compat, + bool record_stats) { + switch (GetMlirBridgeRolloutState(config_proto)) { + case ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_ENABLED: + return MlirBridgeRolloutPolicy::kEnabledByUser; + case ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_DISABLED: + return MlirBridgeRolloutPolicy::kDisabledByUser; + default: + // User did not explicitly enable or disable the bridge. For now, disable + // the bridge. + return MlirBridgeRolloutPolicy::kDisabledAfterGraphAnalysis; + } +} + +void LogGraphFeatures(const Graph& graph, + const FunctionLibraryDefinition* function_library, + std::optional config_proto, + bool is_v1_compat) {} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.h new file mode 100644 index 0000000000000000000000000000000000000000..71f0edd14c03d58651841d6cb96c1f301d736525 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/mlir_bridge_rollout_policy.h @@ -0,0 +1,79 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_MLIR_BRIDGE_ROLLOUT_POLICY_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_MLIR_BRIDGE_ROLLOUT_POLICY_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace tensorflow { + +enum class MlirBridgeRolloutPolicy { + // The MLIR bridge is explicitly disabled by the user and must not be run. + kDisabledByUser = 0, + // The MLIR bridge is explicitly enabled by the user and must be run. If the + // MLIR bridge errors, the fallback path should NOT be used. + kEnabledByUser, + // The bridge was not explicitly enabled or disabled by the user. Based on the + // features in the model, the MLIR bridge should not be run. + kDisabledAfterGraphAnalysis, + // The bridge was not explicitly enabled or disabled by the user. Based on the + // features in the model, the MLIR bridge should be run. If the MLIR Bridge + // errors, the fallback path should be used whenever possible. + kEnabledAfterGraphAnalysis, +}; + +// Analyzes the user requested policy as well as the contents of the graph and +// returns true when the MLIR Bridge should be run. +// +// If the user explicitly requests the bridge be enabled or disabled, this +// function will respect the request. If the user does not explicitly request +// enabled or disabled, it will decide whether or not to run the bridge. +// +// The config_proto param is a required input for all TF1 graphs but it is +// redundant for TF2 graphs. +// If getting rollout policy involves graph analysis, `record_stats` is used +// to decide whether to emit metrics on unsupported features of the graph. +MlirBridgeRolloutPolicy GetMlirBridgeRolloutPolicy( + const tensorflow::Graph& graph, + const FunctionLibraryDefinition* function_library, + std::optional config_proto, + bool is_supported_by_replicated_brige, bool is_v1_compat, + bool record_stats); + +static inline MlirBridgeRolloutPolicy GetMlirBridge2ndPhaseRolloutPolicy( + mlir::ModuleOp module) { + return MlirBridgeRolloutPolicy::kDisabledAfterGraphAnalysis; +} + +// Explicit Interface for when we want to log features vs test the validity of +// the graph for MLIR bridge processing. Note that right now the logging +// which is done in the logic used by GraphHasFeaturesUnsupportedByMlirBridge +// has diverged and logs supported features as well. Parameters are the same +// as for GetMlirBridgeRolloutPolicy with the exception of +// record_stats, which isn't needed because this interface will always record. +void LogGraphFeatures(const Graph& graph, + const FunctionLibraryDefinition* function_library, + std::optional config_proto, + bool is_v1_compat); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_MLIR_BRIDGE_ROLLOUT_POLICY_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..d821d6a3663da3151ae142df693ea2543cf9177b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/BUILD @@ -0,0 +1,33 @@ +load("//tensorflow:tensorflow.default.bzl", "filegroup") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "@llvm-project//mlir:run_lit.sh", + size_override = { + "legalize-tf-binary-elementwise.mlir": "medium", + "legalize-tf-include-tf2xla-fallback.mlir": "medium", + "legalize-tf-prefer-tf2xla.mlir": "medium", + "legalize-tf.mlir": "medium", + }, + test_file_exts = [ + "mlir", + "hlotxt", + ], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-opt", + "@llvm-project//llvm:FileCheck", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/adjust-layout.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/adjust-layout.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4543f45b1f6d86c6ec86a0e262eea3c117432863 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/adjust-layout.mlir @@ -0,0 +1,14 @@ +// RUN: tf-opt -pass-pipeline='builtin.module(func.func(infeed-ops-xla-adjust-layout))' %s | env FILECHECK_OPTS="" FileCheck %s + +func.func @infeed_dequeue_tuple() -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>) { + // CHECK: [[TOKEN:%.*]] = mhlo.create_token : !mhlo.token + %0 = "mhlo.create_token"() : () -> !mhlo.token + + // CHECK: [[INFEED:%.*]]:3 = "mhlo.infeed"([[TOKEN]]) <{ + // CHECK-SAME{LITERAL}: infeed_config = "", layout = [[1, 3, 2, 0], [1, 2, 0]] + // CHECK-SAME: }> : (!mhlo.token) -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>, !mhlo.token) + %1:3 = "mhlo.infeed"(%0) {infeed_config = ""} : (!mhlo.token) -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>, !mhlo.token) + + // CHECK: return [[INFEED]]#0, [[INFEED]]#1 + func.return %1#0, %1#1 : tensor<1x8x4x4xi32>, tensor<1x100x1xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-BatchMatMulV2.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-BatchMatMulV2.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8dc5e8c4d08315f569a9f3cca93feeeb9d4fa7ab --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-BatchMatMulV2.mlir @@ -0,0 +1,89 @@ +// RUN: tf-opt -xla-legalize-tf %s | FileCheck %s + +//===----------------------------------------------------------------------===// +// tf.BatchMatMulV2 op legalizations. +//===----------------------------------------------------------------------===// + +func.func @batchmatmulv2_basic(%arg0: tensor<1x4x2xf32>, %arg1: tensor<3x2x4xf32>) -> tensor<3x4x4xf32> { +// CHECK-LABEL: func @batchmatmulv2_basic +// CHECK-SAME: ([[LHS:%.*]]: tensor<1x4x2xf32>, [[RHS:%.*]]: tensor<3x2x4xf32>) -> tensor<3x4x4xf32> +// CHECK: [[LHSSHAPE:%.*]] = shape.shape_of [[LHS]] : tensor<1x4x2xf32> +// CHECK: [[RHSSHAPE:%.*]] = shape.shape_of [[RHS]] : tensor<3x2x4xf32> +// CHECK: [[CM2:%.*]] = arith.constant -2 : index +// CHECK: [[LHSHEAD:%.*]], [[LHSTAIL:%.*]] = "shape.split_at"([[LHSSHAPE]], [[CM2]]) +// CHECK: [[RHSHEAD:%.*]], [[RHSTAIL:%.*]] = "shape.split_at"([[RHSSHAPE]], [[CM2]]) +// CHECK: [[BCASTHEAD:%.*]] = shape.broadcast [[LHSHEAD]], [[RHSHEAD]] +// CHECK: [[LHSBCASTSHAPE:%.*]] = shape.concat [[BCASTHEAD]], [[LHSTAIL]] +// CHECK: [[LHSSHAPEEXTENTS:%.*]] = shape.to_extent_tensor [[LHSBCASTSHAPE]] +// CHECK: [[LHSBCAST:%.*]] = "mhlo.dynamic_broadcast_in_dim"([[LHS]], [[LHSSHAPEEXTENTS]]) <{broadcast_dimensions = dense<[0, 1, 2]> : tensor<3xi64>}> : (tensor<1x4x2xf32>, tensor<3xindex>) -> tensor<3x4x2xf32> +// CHECK: [[RHSBCASTSHAPE:%.*]] = shape.concat [[BCASTHEAD]], [[RHSTAIL]] +// CHECK: [[RESULT:%.*]] = "mhlo.dot_general"([[LHSBCAST]], [[RHS]]) +// CHECK: return [[RESULT]] : tensor<3x4x4xf32> +// CHECK: } + + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, device = ""} : (tensor<1x4x2xf32>, tensor<3x2x4xf32>) -> tensor<3x4x4xf32> + func.return %0 : tensor<3x4x4xf32> +} + +func.func @batchmatmulv2_lhs_batch(%arg0: tensor<3x4x2xf32>, %arg1: tensor<2x4xf32>) -> tensor<3x4x4xf32> { +// CHECK-LABEL: func @batchmatmulv2_lhs_batch +// CHECK: "mhlo.dynamic_broadcast_in_dim"({{.*}}, {{.*}}) <{broadcast_dimensions = dense<[1, 2]> : tensor<2xi64>}> +// CHECK: "mhlo.dot_general"({{.*}}, {{.*}}) <{ +// CHECK-SAME: lhs_batching_dimensions = [0] +// CHECK-SAME: rhs_batching_dimensions = [0] +// CHECK-SAME: lhs_contracting_dimensions = [2] +// CHECK-SAME: rhs_contracting_dimensions = [1] + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, device = ""} : (tensor<3x4x2xf32>, tensor<2x4xf32>) -> tensor<3x4x4xf32> + func.return %0 : tensor<3x4x4xf32> +} + +func.func @batchmatmulv2_rhs_batch(%arg0: tensor<4x2xf32>, %arg1: tensor<3x2x4xf32>) -> tensor<3x4x4xf32> { +// CHECK-LABEL: func @batchmatmulv2_rhs_batch +// CHECK: "mhlo.dynamic_broadcast_in_dim"({{.*}}, {{.*}}) <{broadcast_dimensions = dense<[1, 2]> : tensor<2xi64>}> +// CHECK: "mhlo.dot_general"({{.*}}, {{.*}}) <{ +// CHECK-SAME: lhs_batching_dimensions = [0] +// CHECK-SAME: rhs_batching_dimensions = [0] +// CHECK-SAME: lhs_contracting_dimensions = [2] +// CHECK-SAME: rhs_contracting_dimensions = [1] + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, device = ""} : (tensor<4x2xf32>, tensor<3x2x4xf32>) -> tensor<3x4x4xf32> + func.return %0 : tensor<3x4x4xf32> +} + +func.func @batchmatmulv2_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { +// CHECK-LABEL: func @batchmatmulv2_dynamic +// CHECK: "mhlo.dot_general"({{.*}}, {{.*}}) <{ +// CHECK-SAME: lhs_batching_dimensions = [0] +// CHECK-SAME: rhs_batching_dimensions = [0] +// CHECK-SAME: lhs_contracting_dimensions = [2] +// CHECK-SAME: rhs_contracting_dimensions = [1] + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, device = ""} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @batchmatmulv2_adj_real(%arg0: tensor<2x5xf32>, %arg1: tensor<4x2xf32>) -> tensor<5x4xf32> { +// CHECK-LABEL: func @batchmatmulv2_adj_real +// CHECK: "mhlo.dot_general"({{.*}}, {{.*}}) <{ +// CHECK-NOT: lhs_batching_dimensions +// CHECK-NOT: rhs_batching_dimensions +// CHECK-SAME: lhs_contracting_dimensions = [0] +// CHECK-SAME: rhs_contracting_dimensions = [1] + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {adj_x = true, adj_y = true, device = ""} : (tensor<2x5xf32>, tensor<4x2xf32>) -> tensor<5x4xf32> + func.return %0 : tensor<5x4xf32> +} + +func.func @batchmatmulv2_adj_complex(%arg0: tensor<2x5xcomplex>, %arg1: tensor<4x2xcomplex>) -> tensor<5x4xcomplex> { +// CHECK-LABEL: func @batchmatmulv2_adj_complex( +// CHECK-SAME: [[LHS:%.*]]: tensor<2x5xcomplex>, [[RHS:%.*]]: tensor<4x2xcomplex>) -> tensor<5x4xcomplex> { +// CHECK: [[LHSRE:%.*]] = mhlo.real [[LHS]] +// CHECK: [[LHSIM:%.*]] = mhlo.imag [[LHS]] +// CHECK: [[LHSIMNEG:%.*]] = mhlo.negate [[LHSIM]] +// CHECK: [[LHSCONJ:%.*]] = mhlo.complex [[LHSRE]], [[LHSIMNEG]] +// CHECK: [[RHSRE:%.*]] = mhlo.real [[RHS]] +// CHECK: [[RHSIM:%.*]] = mhlo.imag [[RHS]] +// CHECK: [[RHSIMNEG:%.*]] = mhlo.negate [[RHSIM]] +// CHECK: [[RHSCONJ:%.*]] = mhlo.complex [[RHSRE]], [[RHSIMNEG]] +// CHECK: shape.shape_of [[LHSCONJ]] +// CHECK: shape.shape_of [[RHSCONJ]] + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {adj_x = true, adj_y = true, device = ""} : (tensor<2x5xcomplex>, tensor<4x2xcomplex>) -> tensor<5x4xcomplex> + func.return %0 : tensor<5x4xcomplex> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-binary-elementwise.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-binary-elementwise.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5b16fe5ec86920fcbac5a3c754f4f9edb4784539 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-binary-elementwise.mlir @@ -0,0 +1,381 @@ +// Note that binary elementwise tests are run with chlo legalization enabled +// (unlike the rest), since this is the primary use case for such ops and +// verification of shapes and broadcasts is desired. +// RUN: tf-opt "-xla-legalize-tf=legalize-chlo=true" -canonicalize %s | FileCheck %s +// RUN: tf-opt "-xla-legalize-tf=legalize-chlo=false" %s | FileCheck --check-prefix CHLO %s + +//===----------------------------------------------------------------------===// +// Binary op legalizations. +// Most of these expand from the same pattern. Full semantics are +// verified for tf.Add and pattern application only for the rest. +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: func @add +func.func @add(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK-NEXT: %[[SUM0:.*]] = mhlo.add %arg0, %arg0 : tensor<2xi32> + // CHECK-NEXT: return %[[SUM0]] : tensor<2xi32> + %1 = "tf.AddV2"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %1: tensor<2xi32> +} + +// CHECK-LABEL: func @broadcast_add +// TODO(laurenzo): Change this to a (5 + 2x1) shaped add to make the check +// patterns unambiguous and more interesting (once broadcastable trait is +// fixed upstream). +func.func @broadcast_add(%arg0: tensor<1xi32>, %arg1: tensor<1x2xi32>) -> tensor<1x2xi32> { + // CHECK-NEXT: %[[LHS_BCAST:.+]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> + // CHECK-NEXT: mhlo.add %[[LHS_BCAST]], %arg1 + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<1xi32>, tensor<1x2xi32>) -> tensor<1x2xi32> + func.return %0: tensor<1x2xi32> +} + +// CHECK-LABEL: func @broadcast_multi_dim_add +// TODO(laurenzo): Change this to a (4x1x1 + 1x4x4x4) shaped add once upstream +// broadcastable bug is fixed (helps make the CHECK matching unambiguous) +func.func @broadcast_multi_dim_add(%arg0: tensor<4x1x1xi32>, %arg1: tensor<4x4x4x4xi32>) -> tensor<4x4x4x4xi32> { + // CHECK-NEXT: %[[LHS_BCAST:.+]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<[1, 2, 3]> : tensor<3xi64>}> + // CHECK-NEXT: mhlo.add %[[LHS_BCAST]], %arg1 + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<4x1x1xi32>, tensor<4x4x4x4xi32>) -> tensor<4x4x4x4xi32> + func.return %0: tensor<4x4x4x4xi32> +} + +// CHECK-LABEL: func @add_dynamic +func.func @add_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: %[[CSTR_LHS_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK-DAG: %[[CSTR_RHS_SHAPE:.+]] = shape.shape_of %arg1 + // CHECK-NEXT: %[[WITNESS:.+]] = shape.cstr_broadcastable %[[CSTR_LHS_SHAPE]], %[[CSTR_RHS_SHAPE]] + // CHECK-NEXT: shape.assuming %[[WITNESS:.+]] + // CHECK-DAG: %[[LHS_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK-DAG: %[[RHS_SHAPE:.+]] = shape.shape_of %arg1 + // CHECK-NEXT: %[[RESULT_EXTENTS:.+]] = shape.broadcast %[[LHS_SHAPE]], %[[RHS_SHAPE]] : tensor<1xindex>, tensor<2xindex> -> tensor<2xindex> + // CHECK-NEXT: %[[LHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg0, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> + // CHECK-NEXT: %[[RHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>}> + // CHECK-NEXT: %[[RESULT:.+]] = mhlo.add %[[LHS_BCAST]], %[[RHS_BCAST]] : tensor + // CHECK-NEXT: shape.assuming_yield %[[RESULT]] + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// CHECK-LABEL: func @broadcast_add_unranked +// CHLO-LABEL: func @broadcast_add_unranked +func.func @broadcast_add_unranked(%arg0: tensor<1xi32>, %arg1: tensor<*xi32>) -> tensor<*xi32> { + // CHECK: tf.Add + // CHLO: chlo.broadcast_add %arg0, %arg1 + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor<1xi32>, tensor<*xi32>) -> tensor<*xi32> + func.return %0: tensor<*xi32> +} + +// CHECK-LABEL: func @div +func.func @div(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK-NEXT: %0 = mhlo.divide %arg0, %arg0 : tensor<2xi32> + // CHECK-NEXT: return %0 : tensor<2xi32> + %0 = "tf.Div"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> +} + +// CHECK-LABEL: func @shift_left +func.func @shift_left(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32> { + // CHECK: mhlo.shift_left %arg0, %arg1 : tensor<4xi32> + %0 = "tf.LeftShift"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0 : tensor<4xi32> +} + +// CHECK-LABEL: func @div_unranked +func.func @div_unranked(%arg0: tensor<*xi32>, %arg1: tensor) -> tensor { + // CHECK-NEXT: tf.Div + %0 = "tf.Div"(%arg0, %arg1) : (tensor<*xi32>, tensor) -> tensor + func.return %0: tensor +} + +// CHECK-LABEL: func @maximum +func.func @maximum(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NEXT: mhlo.maximum %arg0, %arg1 : tensor<4xf32> + %0 = "tf.Maximum"(%arg0, %arg1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// CHECK-LABEL: func @minimum +func.func @minimum(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NEXT: mhlo.minimum %arg0, %arg1 : tensor<4xf32> + %0 = "tf.Minimum"(%arg0, %arg1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// CHECK-LABEL: func @mod +// CHLO-LABEL: func @mod +func.func @mod(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NEXT: mhlo.remainder %arg0, %arg1 : tensor<4xf32> + // CHLO: chlo.broadcast_remainder + %0 = "tf.Mod"(%arg0, %arg1) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> +} + +// CHECK-LABEL: func @mul +func.func @mul(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK-NEXT: %0 = mhlo.multiply %arg0, %arg0 : tensor<2xi32> + // CHECK-NEXT: return %0 : tensor<2xi32> + %0 = "tf.Mul"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> +} + +// CHECK-LABEL: func @real_div +func.func @real_div(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK-NEXT: %0 = mhlo.divide %arg0, %arg0 : tensor<2xi32> + %0 = "tf.RealDiv"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> +} + +// CHECK-LABEL: func @sub +func.func @sub(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK-NEXT: %0 = mhlo.subtract %arg0, %arg0 : tensor<2xi32> + // CHECK-NEXT: return %0 : tensor<2xi32> + %0 = "tf.Sub"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> +} + +// CHECK-LABEL: func @shift_right +func.func @shift_right(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32> { + // CHECK: mhlo.shift_right_arithmetic %arg0, %arg1 : tensor<4xi32> + %0 = "tf.RightShift"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0 : tensor<4xi32> +} + +// CHECK-LABEL: func @shift_right_unsigned +func.func @shift_right_unsigned(%arg0: tensor<4xui8>, %arg1: tensor<4xui8>) -> tensor<4xui8> { + // CHECK: mhlo.shift_right_logical %arg0, %arg1 : tensor<4xui8> + %0 = "tf.RightShift"(%arg0, %arg1) : (tensor<4xui8>, tensor<4xui8>) -> tensor<4xui8> + func.return %0 : tensor<4xui8> +} + +// CHECK-LABEL: func @broadcast_shift_right_unsigned +func.func @broadcast_shift_right_unsigned(%arg0: tensor<4xui8>, %arg1: tensor<2x4xui8>) -> tensor<2x4xui8> { + // CHECK: %[[BROADCAST:.*]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> : (tensor<4xui8>) -> tensor<2x4xui8> + // CHECK: mhlo.shift_right_logical %[[BROADCAST]], %arg1 : tensor<2x4xui8> + %0 = "tf.RightShift"(%arg0, %arg1) : (tensor<4xui8>, tensor<2x4xui8>) -> tensor<2x4xui8> + func.return %0 : tensor<2x4xui8> +} + +// CHECK-LABEL: func @and +func.func @and(%arg0: tensor<2xi1>, %arg1: tensor<2xi1>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.and + %0 = "tf.LogicalAnd"(%arg0, %arg1) : (tensor<2xi1>, tensor<2xi1>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @and_unranked +func.func @and_unranked(%arg0: tensor<*xi1>, %arg1: tensor<*xi1>) -> tensor<*xi1> { + // CHECK: tf.LogicalAnd + %0 = "tf.LogicalAnd"(%arg0, %arg1) : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @or +func.func @or(%arg0: tensor<2xi1>, %arg1: tensor<2xi1>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.or + %0 = "tf.LogicalOr"(%arg0, %arg1) : (tensor<2xi1>, tensor<2xi1>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @bitwise_or +func.func @bitwise_or(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32> { + // CHECK-NEXT: mhlo.or + %0 = "tf.BitwiseOr"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0: tensor<4xi32> +} + +// CHECK-LABEL: func @bitwise_or_unsigned +func.func @bitwise_or_unsigned(%arg0: tensor<4xui32>, %arg1: tensor<4xui32>) -> tensor<4xui32> { + // CHECK-NEXT: mhlo.or + %0 = "tf.BitwiseOr"(%arg0, %arg1) : (tensor<4xui32>, tensor<4xui32>) -> tensor<4xui32> + func.return %0: tensor<4xui32> +} + +// CHECK-LABEL: func @bitwise_xor +func.func @bitwise_xor(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32> { + // CHECK-NEXT: mhlo.xor + %0 = "tf.BitwiseXor"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0: tensor<4xi32> +} + +// CHECK-LABEL: func @bitwise_xor_unsigned +func.func @bitwise_xor_unsigned(%arg0: tensor<4xui32>, %arg1: tensor<4xui32>) -> tensor<4xui32> { + // CHECK-NEXT: mhlo.xor + %0 = "tf.BitwiseXor"(%arg0, %arg1) : (tensor<4xui32>, tensor<4xui32>) -> tensor<4xui32> + func.return %0: tensor<4xui32> +} + +// CHECK-LABEL: func @bitwise_and +func.func @bitwise_and(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32> { + // CHECK-NEXT: mhlo.and + %0 = "tf.BitwiseAnd"(%arg0, %arg1) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + func.return %0: tensor<4xi32> +} + +// CHECK-LABEL: func @bitwise_and_unsigned +func.func @bitwise_and_unsigned(%arg0: tensor<4xui32>, %arg1: tensor<4xui32>) -> tensor<4xui32> { + // CHECK-NEXT: mhlo.and + %0 = "tf.BitwiseAnd"(%arg0, %arg1) : (tensor<4xui32>, tensor<4xui32>) -> tensor<4xui32> + func.return %0: tensor<4xui32> +} + +// CHECK-LABEL: func @pow +func.func @pow(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NEXT: mhlo.power + %0 = "tf.Pow"(%arg0, %arg0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0: tensor<2xf32> +} + +//===----------------------------------------------------------------------===// +// Equality op legalizations. +// tf.Equal and tf.NotEqual expand from the same pattern. Full semantics are +// verified for tf.Equal and pattern application only for tf.NotEqual +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: func @equal +func.func @equal(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare EQ, %arg0, %arg1 + %0 = "tf.Equal"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @equal_dynamic +func.func @equal_dynamic(%arg0: tensor, %arg1: tensor<1xi32>) -> tensor { + // TODO(jpienaar): Uncomment below when fallout from https://reviews.llvm.org/D83194 fixed. + // NOT-CHECK-DAG: %[[LHS_SHAPE:.+]] = shape.shape_of %arg0 + // NOT-CHECK-DAG: %[[RHS_SHAPE:.+]] = shape.const_shape [1] + // NOT-CHECK-NEXT: %[[WITNESS:.+]] = shape.cstr_broadcastable %[[LHS_SHAPE]], %[[RHS_SHAPE]] + // NOT-CHECK-NEXT: shape.assuming %[[WITNESS]] -> (tensor) { + // NOT-CHECK-DAG: %[[LHS_SHAPE1:.+]] = shape.shape_of %arg0 + // NOT-CHECK-NEXT: %[[RESULT_SHAPE:.+]] = shape.broadcast %[[LHS_SHAPE1]], %[[RHS_SHAPE]] : tensor, tensor -> tensor + // NOT-CHECK-NEXT: %[[RESULT_EXTENTS:.+]] = tensor.cast %[[RESULT_SHAPE]] : tensor to tensor<1xindex> + // NOT-CHECK-DAG: %[[LHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg0, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> + // NOT-CHECK-DAG: %[[RHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> + // NOT-CHECK-NEXT: %[[RESULT:.+]] = mhlo.compare EQ, %[[LHS_BCAST]], %[[RHS_BCAST]] + // NOT-CHECK-NEXT: shape.assuming_yield %[[RESULT]] + %0 = "tf.Equal"(%arg0, %arg1) : (tensor, tensor<1xi32>) -> tensor + func.return %0: tensor +} + +// CHECK-LABEL: func @equal_broadcast +func.func @equal_broadcast(%arg0: tensor<1xi32>, %arg1: tensor<1x2xi32>) -> tensor<1x2xi1> { + // CHECK-DAG: %[[LHS_BCAST:.+]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> + // CHECK-NEXT: mhlo.compare EQ, %[[LHS_BCAST]], %arg1 + %0 = "tf.Equal"(%arg0, %arg1) : (tensor<1xi32>, tensor<1x2xi32>) -> tensor<1x2xi1> + func.return %0: tensor<1x2xi1> +} + +// CHECK-LABEL: func @equal_broadcast_no_incompatible_shapes_error +func.func @equal_broadcast_no_incompatible_shapes_error(%arg0: tensor<2xi32>, %arg1: tensor<1x2xi32>) -> tensor<1x2xi1> { + // CHECK-NEXT: "tf.Equal"(%arg0, %arg1) <{incompatible_shape_error = true}> + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor<2xi32>, tensor<1x2xi32>) -> tensor<1x2xi1> + func.return %0: tensor<1x2xi1> +} + +// CHECK-LABEL: func @equal_incompatible_shape_broadcastable +func.func @equal_incompatible_shape_broadcastable(%arg0: tensor, %arg1: tensor<1xi32>) -> tensor { + // CHECK-NEXT: "tf.Equal"(%arg0, %arg1) <{incompatible_shape_error = true}> + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor, tensor<1xi32>) -> tensor + func.return %0: tensor +} + +// CHECK-LABEL: func @equal_incompatible_shape_dynamic +func.func @equal_incompatible_shape_dynamic(%arg0: tensor<2xi32>, %arg1: tensor) -> tensor<*xi1> { + // CHECK-NEXT: "tf.Equal"(%arg0, %arg1) <{incompatible_shape_error = false}> + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor<2xi32>, tensor) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @equal_incompatible_shape_both_dynamic +func.func @equal_incompatible_shape_both_dynamic(%arg0: tensor, %arg1: tensor) -> tensor<*xi1> { + // CHECK-NEXT: "tf.Equal"(%arg0, %arg1) <{incompatible_shape_error = false}> + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor, tensor) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @equal_unranked +func.func @equal_unranked(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> tensor<*xi1> { + // CHECK: "tf.Equal" + // CHLO: chlo.broadcast_compare %arg0, %arg1 {comparison_direction = #chlo} + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @equal_unsupported_type +func.func @equal_unsupported_type(%arg0: tensor<*x!tf_type.string>, %arg1: tensor<*x!tf_type.string>) -> tensor<*xi1> { + // CHECK: "tf.Equal" + %0 = "tf.Equal"(%arg0, %arg1) { incompatible_shape_error = false } : (tensor<*x!tf_type.string>, tensor<*x!tf_type.string>) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @notequal +func.func @notequal(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare NE, %arg0, %arg1 + %0 = "tf.NotEqual"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +//===----------------------------------------------------------------------===// +// Compare op legalizations. +// These expand from the same pattern. Full semantics are checked for +// tf.Greater. Others just check that the pattern applied. +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: func @greater +func.func @greater(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK: mhlo.compare GT, %arg0, %arg1 + %0 = "tf.Greater"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @broadcast_greater +func.func @broadcast_greater(%arg0: tensor<1xi32>, %arg1: tensor<1x2xi32>) -> tensor<1x2xi1> { + // CHECK-NEXT: %[[LHS_BCAST:.+]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> + // CHECK-NEXT: mhlo.compare GT, %[[LHS_BCAST]], %arg1 + %0 = "tf.Greater"(%arg0, %arg1) : (tensor<1xi32>, tensor<1x2xi32>) -> tensor<1x2xi1> + func.return %0: tensor<1x2xi1> +} + +// CHECK-LABEL: func @greater_dynamic +func.func @greater_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: %[[LHS_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK-DAG: %[[RHS_SHAPE:.+]] = shape.shape_of %arg1 + // CHECK-NEXT: %[[WITNESS:.+]] = shape.cstr_broadcastable %[[LHS_SHAPE]], %[[RHS_SHAPE]] + // CHECK-NEXT: shape.assuming %[[WITNESS]] + // CHECK-DAG: %[[LHS_SHAPE1:.+]] = shape.shape_of %arg0 + // CHECK-DAG: %[[RHS_SHAPE1:.+]] = shape.shape_of %arg1 + // CHECK-NEXT: %[[RESULT_EXTENTS:.+]] = shape.broadcast %[[LHS_SHAPE1]], %[[RHS_SHAPE1]] : tensor<1xindex>, tensor<1xindex> -> tensor<1xindex> + // CHECK-DAG: %[[LHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg0, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> + // CHECK-DAG: %[[RHS_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[RESULT_EXTENTS]]) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> + // CHECK-NEXT: mhlo.compare GT, %[[LHS_BCAST]], %[[RHS_BCAST]] + %0 = "tf.Greater"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// CHECK-LABEL: func @greater_uranked +func.func @greater_uranked(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> tensor<*xi1> { + // CHECK: "tf.Greater" + // CHLO: chlo.broadcast_compare %arg0, %arg1 {comparison_direction = #chlo} + %0 = "tf.Greater"(%arg0, %arg1) : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi1> + func.return %0: tensor<*xi1> +} + +// CHECK-LABEL: func @greater_equal +func.func @greater_equal(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare GE, %arg0, %arg1 + %0 = "tf.GreaterEqual"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @less +func.func @less(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare LT, %arg0, %arg1 + %0 = "tf.Less"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} + +// CHECK-LABEL: func @less_equal +func.func @less_equal(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare LE, %arg0, %arg1 + %0 = "tf.LessEqual"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-collective.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-collective.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7dacaeee3647750242c1b7bee4cdb7d5e1b514af --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-collective.mlir @@ -0,0 +1,285 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -xla-legalize-tf-collective -xla-legalize-tf %s | FileCheck %s + + +// CHECK: module attributes +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_key = 0 +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_size = 2 +// CHECK-LABEL: func @all_reduce_cross_replica +func.func @all_reduce_cross_replica(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK{LITERAL}: replica_groups = dense<[[0], [1]]> : tensor<2x1xi64> + // CHECK-NOT: channel_handle + // CHECK: mhlo.add + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Add", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK: module attributes +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_key = 0 +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_size = 2 +// CHECK-LABEL: func @all_reduce_cross_replica_and_partition +func.func @all_reduce_cross_replica_and_partition(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0], [1]]> : tensor<2x1xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Add", mode = "CrossReplicaAndPartition"} : (tensor, tensor<2x1xi32>) -> tensor + // CHECK: "mhlo.all_reduce" + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0], [1]]> : tensor<2x1xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + %1 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Add", mode = "CrossReplicaAndPartition"} : (tensor, tensor<2x1xi32>) -> tensor + %2 = "tf.Add"(%0, %1) : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// CHECK-LABEL: func @xla_all_reduce_add +func.func @xla_all_reduce_add(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK: mhlo.add + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Add", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @xla_all_reduce_max +func.func @xla_all_reduce_max(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK: mhlo.maximum + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Max", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @xla_all_reduce_mean +func.func @xla_all_reduce_mean(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: %[[GROUP_SIZE:.*]] = mhlo.constant dense<1.000000e+00> + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK: mhlo.add + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[REDUCE]], %[[GROUP_SIZE]] + // CHECK-NEXT: return %[[RESULT]] + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Mean", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @xla_all_reduce_min +func.func @xla_all_reduce_min(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK: mhlo.minimum + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Min", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @xla_all_reduce_mul +func.func @xla_all_reduce_mul(%input: tensor) -> tensor { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // CHECK: "mhlo.all_reduce" + // CHECK: mhlo.mul + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Mul", mode = "CrossReplica"} : (tensor, tensor<2x1xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @xla_all_reduce_tuple(%input: tuple, tensor>) -> tuple, tensor> { + %group_assignment = "tf.Const"() { value = dense<[[0],[1]]> : tensor<2x1xi32> } : () -> tensor<2x1xi32> + // expected-error@+1 {{'tf.XlaAllReduce' op operand #0 must be tensor of bfloat16 or 16-bit float or 32-bit float or 32-bit integer or 32-bit unsigned integer values, but got 'tuple, tensor>'}} + %0 = "tf.XlaAllReduce"(%input, %group_assignment) {reduce_op = "Add", mode = "CrossReplica"} : (tuple, tensor>, tensor<2x1xi32>) -> tuple, tensor> + func.return %0 : tuple, tensor> +} + + +// ----- + +// CHECK: module attributes +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_key = 1 +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_size = 2 +// CHECK-LABEL: func @collective_reduce_v2 +func.func @collective_reduce_v2(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: "mhlo.all_reduce" + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + // CHECK: "mhlo.all_reduce" + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + %1 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + %2 = "tf.Add"(%0, %1) : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// CHECK-LABEL: func @collective_reduce_v2_add_id +func.func @collective_reduce_v2_add_id(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + // CHECK: return %[[REDUCE]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_max_id +func.func @collective_reduce_v2_max_id(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.maximum + // CHECK: mhlo.return + // CHECK: return %[[REDUCE]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Max", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_min_id +func.func @collective_reduce_v2_min_id(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.minimum + // CHECK: mhlo.return + // CHECK: return %[[REDUCE]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Min", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_mul_id +func.func @collective_reduce_v2_mul_id(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.mul + // CHECK: mhlo.return + // CHECK: return %[[REDUCE]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Mul", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_add_div +func.func @collective_reduce_v2_add_div(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[GROUP_SIZE:.*]] = mhlo.constant dense<2.000000e+00> + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.add + // CHECK: mhlo.return + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[REDUCE]], %[[GROUP_SIZE]] + // CHECK-NEXT: return %[[RESULT]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Div"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_max_div +func.func @collective_reduce_v2_max_div(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[GROUP_SIZE:.*]] = mhlo.constant dense<2.000000e+00> + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.maximum + // CHECK: mhlo.return + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[REDUCE]], %[[GROUP_SIZE]] + // CHECK-NEXT: return %[[RESULT]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Max", final_op = "Div"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_min_div +func.func @collective_reduce_v2_min_div(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[GROUP_SIZE:.*]] = mhlo.constant dense<2.000000e+00> + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.minimum + // CHECK: mhlo.return + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[REDUCE]], %[[GROUP_SIZE]] + // CHECK-NEXT: return %[[RESULT]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Min", final_op = "Div"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @collective_reduce_v2_mul_div +func.func @collective_reduce_v2_mul_div(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // CHECK: %[[GROUP_SIZE:.*]] = mhlo.constant dense<2.000000e+00> + // CHECK: %[[REDUCE:.*]] = "mhlo.all_reduce" + // CHECK-SAME{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK: mhlo.mul + // CHECK: mhlo.return + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[REDUCE]], %[[GROUP_SIZE]] + // CHECK-NEXT: return %[[RESULT]] + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Mul", final_op = "Div"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + + +// ----- + +// CHECK: module attributes +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_key = 0 +// CHECK-SAME{LITERAL}: tf2xla.collective_info.group_size = 2 +// CHECK-LABEL: func @collective_assign_group_v2 +func.func @collective_assign_group_v2(%input: tensor) -> tensor { + %rank = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + %key_base = "tf.Const"() { value = dense<10> : tensor } : () -> tensor + %group_assignment = "tf.Const"() { value = dense<[[0, 1]]> : tensor<1x2xi32> } : () -> tensor<1x2xi32> + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + %group_size, %group_key = "tf.CollectiveAssignGroupV2"(%group_assignment, %rank, %key_base) {} : (tensor<1x2xi32>, tensor, tensor) -> (tensor, tensor) + // CHECK-NOT: "tf.CollectiveAssignGroupV2" + // CHECK: "mhlo.all_reduce" + // CHECK: mhlo.add + // CHECK{LITERAL}: replica_groups = dense<[[0, 1]]> : tensor<1x2xi64> + // CHECK-NOT: "tf.CollectiveAssignGroupV2" + %0 = "tf.CollectiveReduceV2"(%input, %group_size, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @inconsistent_collective_info(%input: tensor) -> tensor { + %group_key = "tf.Const"() { value = dense<11> : tensor } : () -> tensor + %group_size1 = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %group_size2 = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %instance_key = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + // expected-error@below {{op module already contains an attribute tf2xla.collective_info.group_size=2, overwritting to a new value 1 is not allowed.}} + %0 = "tf.CollectiveReduceV2"(%input, %group_size1, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + %1 = "tf.CollectiveReduceV2"(%input, %group_size2, %group_key, %instance_key) {merge_op = "Add", final_op = "Id"} : (tensor, tensor, tensor, tensor) -> tensor + %2 = "tf.Add"(%0, %1) : (tensor, tensor) -> tensor + func.return %2 : tensor +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-communication.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-communication.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fdf2c07d31456f0faa13623db3707fd0b1cba619 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-communication.mlir @@ -0,0 +1,916 @@ +// RUN: tf-opt -split-input-file -verify-diagnostics -xla-legalize-tf-communication %s | FileCheck %s + +// Test legalization of `tf._XlaHostComputeMlir` expands into individual +// `mhlo.send` per operand and `mhlo.recv` per result. Channel Id's are uniquely +// assigned per mhlo communcation op, and frontend attributes (modified keys) +// and op shardings are added. Sink tokens are created +// if there are more than one operand or more than one result. +// +// The following op sharding assigns full tensors to TPU core 0. +// Proto debug string: +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// Serialized string: +// "\08\01\1A\01\01\22\01\00" + +// CHECK-LABEL: func @host_compute +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @host_compute(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[SEND_ARG0_TOKEN:%.*]] = "mhlo.send"([[ARG0]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "host_compute_channel_send_dtoh_0"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (tensor, !mhlo.token) -> !mhlo.token + + // CHECK: [[SEND_ARG1_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "host_compute_channel_send_dtoh_1"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (tensor, !mhlo.token) -> !mhlo.token + + // CHECK: [[SEND_SINK_TOKEN:%.*]] = mhlo.after_all [[SEND_ARG0_TOKEN]], [[SEND_ARG1_TOKEN]] + + // CHECK: [[RECV_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND_SINK_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "host_compute_channel_recv_htod_0"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (!mhlo.token) -> (tensor, !mhlo.token) + + // CHECK: [[RECV_RETVAL1_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND_SINK_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "host_compute_channel_recv_htod_1"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (!mhlo.token) -> (tensor, !mhlo.token) + + // CHECK: [[RECV_SINK_TOKEN:%.*]] = mhlo.after_all [[RECV_RETVAL0_TUPLE]]#1, [[RECV_RETVAL1_TUPLE]]#1 + %0:2 = "tf._XlaHostComputeMlir"(%arg0, %arg1) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : (tensor, tensor) -> (tensor, tensor) + + // CHECK: return [[RECV_RETVAL0_TUPLE]]#0, [[RECV_RETVAL1_TUPLE]]#0 : tensor, tensor + func.return %0#0, %0#1 : tensor, tensor +} + +// ----- + +// Tests `tf._XlaHostComputeMlir` with no operands simply forwards the input +// token to its generated `mhlo.recv`. + +// CHECK-LABEL: func @host_compute_no_operands_one_result +func.func @host_compute_no_operands_one_result() { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK-NOT: "mhlo.send" + // CHECK-NOT: "mhlo.after_all" + // CHECK: "mhlo.recv"([[INIT_TOKEN]]) + %0 = "tf._XlaHostComputeMlir"() {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : () -> tensor + func.return +} + +// ----- + +// Tests `tf._XlaHostComputeMlir` with no results simply forwards its token from +// the generated `mhlo.send`. + +// CHECK-LABEL: func @host_compute_one_operand_no_results +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @host_compute_one_operand_no_results(%arg0: tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send"([[ARG0]], [[INIT_TOKEN]]) + // CHECK-NOT: "mhlo.after_all" + "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : (tensor) -> () + + // CHECK: "mhlo.recv"([[SEND_TOKEN]]) + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + func.return +} + +// ----- + +// Tests `tf._XlaHostComputeMlir` with one operand and one result does not +// create any `mhlo.after_all` ops. + +// CHECK-LABEL: func @host_compute_single_operand_result +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @host_compute_single_operand_result(%arg0: tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send"([[ARG0]], [[INIT_TOKEN]]) + // CHECK-NOT: "mhlo.after_all" + // CHECK: "mhlo.recv"([[SEND_TOKEN]]) + // CHECK-NOT: "mhlo.after_all" + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : (tensor) -> tensor + func.return +} + +// ----- + +// Test legalization of `tf.XlaSendToHost` expands into a `mhlo.send` op. + +// CHECK-LABEL: func @send_to_host +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @send_to_host(%arg0: tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: "mhlo.send"([[ARG0]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_key_dtoh_0"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (tensor, !mhlo.token) -> !mhlo.token + "tf.XlaSendToHost"(%arg0) {key = "send_key"} : (tensor) -> () + func.return +} + +// ----- + +// Test legalization of `tf.XlaRecvFromHost` expands into a `mhlo.recv` op. + +// CHECK-LABEL: func @recv_from_host +func.func @recv_from_host() -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: is_host_transfer = true + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_key_htod_0"} + // CHECK-SAME: mhlo.sharding = "\08\01\1A\01\01\22\01\00" + // CHECK-SAME: (!mhlo.token) -> (tensor, !mhlo.token) + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + // CHECK: return [[RECV_TUPLE]]#0 : tensor + func.return %0 : tensor +} + +// ----- + +// Test legalization of multiple TF/XLA communication ops are sequenced with +// their generated tokens. Channel Id's are also uniquely assigned. + +// CHECK-LABEL: func @multiple_consecutive_ops +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @multiple_consecutive_ops(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[SEND0_ARG0_TOKEN:%.*]] = "mhlo.send"([[ARG0]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send0_dtoh_0"} + + // CHECK: [[RECV0_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND0_ARG0_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv0_htod_0"} + + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "recv0", send_key = "send0", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: [[SEND1_ARG0_TOKEN:%.*]] = "mhlo.send"([[RECV0_RETVAL0_TUPLE]]#0, [[RECV0_RETVAL0_TUPLE]]#1) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send1_dtoh_0"} + + // CHECK: [[RECV1_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND1_ARG0_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv1_htod_0"} + + %1 = "tf._XlaHostComputeMlir"(%0) {recv_key = "recv1", send_key = "send1", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: return [[RECV1_RETVAL0_TUPLE]]#0 : tensor + func.return %1 : tensor +} + +// ----- + +// Test private function with TF/XLA communication op being called by another +// function gets rewritten with an extra token argument and an extra token +// result, and the caller passes in a token. The top level function not called +// (or public) will be updated to create a token. + +// CHECK: func @main([[MAIN_ARG0:%.*]]: tensor) -> tensor +func.func @main(%arg0: tensor) -> tensor { + // CHECK: [[MAIN_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[CALL:%.*]]:2 = call @callee([[MAIN_ARG0]], [[MAIN_TOKEN]]) + // CHECK-SAME: (tensor, !mhlo.token) -> (tensor, !mhlo.token) + %0 = func.call @callee(%arg0) : (tensor) -> tensor + + // CHECK: return [[CALL]]#0 + func.return %0 : tensor +} + +// CHECK: func private @callee([[CALLEE_ARG0:%.*]]: tensor, [[CALLEE_ARG1:%.*]]: !mhlo.token) -> (tensor, !mhlo.token) +func.func private @callee(%arg0: tensor) -> tensor { + // CHECK-NOT: mhlo.create_token + + // CHECK: [[SEND_ARG0_TOKEN:%.*]] = "mhlo.send"([[CALLEE_ARG0]], [[CALLEE_ARG1]]) + // CHECK: [[RECV_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND_ARG0_TOKEN]]) + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "recv", send_key = "send", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: return [[RECV_RETVAL0_TUPLE]]#0, [[RECV_RETVAL0_TUPLE]]#1 + func.return %0 : tensor +} + +// ----- + +// Test public function with TF/XLA communication op being called by another +// function. The original public function will be modified to create a token, +// while the function is cloned and rewritten with an extra token argument and +// an extra token result. All callers to the original function are updated to +// point to the cloned function and the function the caller is in is updated to +// pass a token or create a token. + +// CHECK: func @main([[MAIN_ARG0:%.*]]: tensor) -> tensor +func.func @main(%arg0: tensor) -> tensor { + // CHECK: [[MAIN_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[CALL:%.*]]:2 = call [[CALLEE_CLONE:@.*]]([[MAIN_ARG0]], [[MAIN_TOKEN]]) + // CHECK-SAME: (tensor, !mhlo.token) -> (tensor, !mhlo.token) + %0 = func.call @callee(%arg0) : (tensor) -> tensor + + // CHECK: return [[CALL]]#0 : tensor + func.return %0 : tensor +} + +// CHECK: func @callee([[CALLEE_ARG0:%.*]]: tensor) -> tensor +func.func @callee(%arg0: tensor) -> tensor { + // CHECK: [[CALLEE_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[SEND_ARG0_TOKEN:%.*]] = "mhlo.send"([[CALLEE_ARG0]], [[CALLEE_TOKEN]]) + // CHECK: [[RECV_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[SEND_ARG0_TOKEN]]) + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "recv", send_key = "send", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: return [[RECV_RETVAL0_TUPLE]]#0 + func.return %0 : tensor +} + +// CHECK: func private [[CALLEE_CLONE]]([[CALLEE_CLONE_ARG0:%.*]]: tensor, [[CALLEE_CLONE_ARG1:%.*]]: !mhlo.token) -> (tensor, !mhlo.token) +// CHECK-NOT: mhlo.create_token + +// CHECK: [[CLONE_SEND_ARG0_TOKEN:%.*]] = "mhlo.send"([[CALLEE_CLONE_ARG0]], [[CALLEE_CLONE_ARG1]]) +// CHECK: [[CLONE_RECV_RETVAL0_TUPLE:%.*]]:2 = "mhlo.recv"([[CLONE_SEND_ARG0_TOKEN]]) + +// CHECK: return [[CLONE_RECV_RETVAL0_TUPLE]]#0, [[CLONE_RECV_RETVAL0_TUPLE]]#1 + +// ----- + +// Tests generated tokens are passed into a function call that also has TF/XLA +// communication ops. + +// CHECK: func @main([[MAIN_ARG0:%.*]]: tensor) +func.func @main(%arg0: tensor) { + // CHECK: [[MAIN_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[MAIN_SEND0_TOKEN:%.*]] = "mhlo.send"([[MAIN_ARG0]], [[MAIN_TOKEN]]) + "tf.XlaSendToHost"(%arg0) {key = "send0"} : (tensor) -> () + + // CHECK: [[CALL_TOKEN:%.*]] = call @callee([[MAIN_SEND0_TOKEN]]) + // CHECK-SAME: (!mhlo.token) -> !mhlo.token + func.call @callee() : () -> () + + // CHECK: [[MAIN_SEND2_TOKEN:%.*]] = "mhlo.send"([[MAIN_ARG0]], [[CALL_TOKEN]]) + "tf.XlaSendToHost"(%arg0) {key = "send2"} : (tensor) -> () + func.return +} + +// CHECK: func private @callee([[CALLEE_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +func.func private @callee() { + // CHECK-NOT: mhlo.create_token + + // CHECK: [[ZERO:%.*]] = mhlo.constant dense<0> + %0 = mhlo.constant dense<0> : tensor + + // CHECK: [[CALLEE_SEND_TOKEN:%.*]] = "mhlo.send"([[ZERO]], [[CALLEE_ARG0]]) + "tf.XlaSendToHost"(%0) {key = "send1"} : (tensor) -> () + + // CHECK: return [[CALLEE_SEND_TOKEN]] + func.return +} + +// ----- + +// Test only the top level function generates a token. + +// CHECK: func private @callee0() +func.func private @callee0() { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: call @callee1([[INIT_TOKEN]]) + func.call @callee1() : () -> () + func.return +} + +// CHECK: func private @callee1([[CALLEE1_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +func.func private @callee1() { + // CHECK-NOT: mhlo.create_token + + // CHECK: [[CALL_2:%.*]] = call @callee2([[CALLEE1_ARG0]]) + func.call @callee2() : () -> () + + // CHECK: return [[CALL_2]] + func.return +} + +// CHECK: func private @callee2([[CALLEE2_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +func.func private @callee2() { + // CHECK-NOT: mhlo.create_token + + // CHECK: [[RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[CALLEE2_ARG0]]) + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + + // CHECK: return [[RECV_TUPLE]]#1 + func.return +} + +// ----- + +// Test cloned function rewrite also checks transitive function calls to +// TF/XLA communication ops. + +// CHECK: func @callee3() +func.func @callee3() { + // CHECK: [[CALLEE3_INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: call @callee4{{.+}}([[CALLEE3_INIT_TOKEN]]) + func.call @callee4() : () -> () + func.return +} + +// CHECK: func @callee4() +func.func @callee4() { + // CHECK: [[CALLEE4_INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[CALL_5:%.*]] = call @callee5([[CALLEE4_INIT_TOKEN]]) + func.call @callee5() : () -> () + + // CHECK: return + func.return +} + +// CHECK: func private @callee5([[CALLEE5_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +func.func private @callee5() { + // CHECK-NOT: mhlo.create_token + + // CHECK: [[RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[CALLEE5_ARG0]]) + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + + // CHECK: return [[RECV_TUPLE]]#1 + func.return +} + +// CHECK: func private @callee4{{.+}}([[CALLEE4_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +// CHECK-NOT: mhlo.create_token +// CHECK: [[CALL_5:%.*]] = call @callee5([[CALLEE4_ARG0]]) +// CHECK: return [[CALL_5]] + +// ----- + +// Tests `mhlo.if` with branches populated with TF/XLA communication ops. + +// CHECK-LABEL: func @if_both_branches +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func.func @if_both_branches(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[IF:%.*]]:2 = "mhlo.if"([[ARG0]]) + %0 = "mhlo.if"(%arg0) ({ + // CHECK: [[TRUE_SEND_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_if_true_dtoh_0"} + + // CHECK: [[TRUE_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[TRUE_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_if_true_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_if_true", send_key = "send_if_true", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[TRUE_RECV_TUPLE]]#0, [[TRUE_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + }, { + // CHECK: [[FALSE_SEND_TOKEN:%.*]] = "mhlo.send"([[ARG2]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_if_false_dtoh_0"} + + // CHECK: [[FALSE_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[FALSE_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_if_false_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg2) {recv_key = "recv_if_false", send_key = "send_if_false", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[FALSE_RECV_TUPLE]]#0, [[FALSE_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + + // CHECK: (tensor) -> (tensor, !mhlo.token) + }) : (tensor) -> tensor + + // CHECK: return [[IF]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with only the `true` branch populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @if_true_branch +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func.func @if_true_branch(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[IF:%.*]]:2 = "mhlo.if"([[ARG0]]) + %0 = "mhlo.if"(%arg0) ({ + // CHECK: [[TRUE_SEND_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_if_true_dtoh_0"} + + // CHECK: [[TRUE_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[TRUE_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_if_true_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_if_true", send_key = "send_if_true", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[TRUE_RECV_TUPLE]]#0, [[TRUE_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + }, { + // CHECK: mhlo.return [[ARG2]], [[INIT_TOKEN]] + "mhlo.return"(%arg2) : (tensor) -> () + + // CHECK: (tensor) -> (tensor, !mhlo.token) + }) : (tensor) -> tensor + + // CHECK: return [[IF]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with only the `false` branch populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @if_false_branch +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func.func @if_false_branch(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[IF:%.*]]:2 = "mhlo.if"([[ARG0]]) + %0 = "mhlo.if"(%arg0) ({ + // CHECK: mhlo.return [[ARG1]], [[INIT_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + }, { + // CHECK: [[FALSE_SEND_TOKEN:%.*]] = "mhlo.send"([[ARG2]], [[INIT_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_if_false_dtoh_0"} + + // CHECK: [[FALSE_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[FALSE_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_if_false_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg2) {recv_key = "recv_if_false", send_key = "send_if_false", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[FALSE_RECV_TUPLE]]#0, [[FALSE_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + + // CHECK: (tensor) -> (tensor, !mhlo.token) + }) : (tensor) -> tensor + + // CHECK: return [[IF]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with tuple arg from a `mhlo.tuple` only used by `mhlo.if` is +// replaced. + +// CHECK-LABEL: func @if_replace_tuple_arg +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func.func @if_replace_tuple_arg(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK-NOT: mhlo.tuple [[ARG1]], [[ARG2]] + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[IF:%.*]] = "mhlo.if"([[ARG0]]) + %1 = "mhlo.if"(%arg0) ({ + "tf.XlaSendToHost"(%arg1) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%arg1) : (tensor) -> () + }, { + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + func.return %1 : tensor +} + +// ----- + +// Tests `mhlo.if` with tuple arg not from a `mhlo.tuple` is unpacked. + +// CHECK-LABEL: func @if_unpack_tuple_arg +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tuple, tensor>) +func.func @if_unpack_tuple_arg(%arg0: tensor, %arg1: tuple, tensor>) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK-DAG: [[IF_ARG_ELEMENT0:%.*]] = mhlo.get_tuple_element [[ARG1]][0] + // CHECK-DAG: [[IF_ARG_ELEMENT1:%.*]] = mhlo.get_tuple_element [[ARG1]][1] + %0 = "mhlo.get_tuple_element"(%arg1) {index = 0 : i32} : (tuple, tensor>) -> tensor + %1 = "mhlo.get_tuple_element"(%arg1) {index = 1 : i32} : (tuple, tensor>) -> tensor + + // CHECK: [[IF:%.*]] = "mhlo.if"([[ARG0]]) + %2 = "mhlo.if"(%arg0) ({ + "tf.XlaSendToHost"(%0) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%0) : (tensor) -> () + }, { + "mhlo.return"(%0) : (tensor) -> () + }) : (tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// Tests `mhlo.if` tuple result is extended with a `mhlo.token`. + +// CHECK-LABEL: func @if_extend_tuple_result +func.func @if_extend_tuple_result(%arg0: tensor, %arg1: tuple, tensor>) -> tuple, tensor> { + %0 = "mhlo.get_tuple_element"(%arg1) {index = 0 : i32} : (tuple, tensor>) -> tensor + %1 = "mhlo.get_tuple_element"(%arg1) {index = 1 : i32} : (tuple, tensor>) -> tensor + // CHECK: [[IF:%.*]]:3 = "mhlo.if" + %2:2 = "mhlo.if"(%arg0) ({ + "tf.XlaSendToHost"(%0) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%0, %1) : (tensor, tensor) -> () + }, { + "mhlo.return"(%0, %1) : (tensor, tensor) -> () + // CHECK: (tensor) -> (tensor, tensor, !mhlo.token) + }) : (tensor) -> (tensor, tensor) + + // CHECK: [[IF_SUBTUPLE_RESULT:%.*]] = mhlo.tuple [[IF]]#0, [[IF]]#1 + %3 = "mhlo.tuple"(%2#0, %2#1) : (tensor, tensor) -> tuple, tensor> + // CHECK: return [[IF_SUBTUPLE_RESULT]] + func.return %3 : tuple, tensor> +} + +// ----- + +// Tests nested `mhlo.if` containing TF/XLA communication ops. + +// CHECK-LABEL: func @if_nested +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @if_nested(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[IF:%.*]]:2 = "mhlo.if"([[ARG0]]) + %0 = "mhlo.if"(%arg0) ({ + %1 = mhlo.constant dense : tensor + + // CHECK: [[INNER_IF:%.*]]:2 = "mhlo.if"({{%.*}}) + %2 = "mhlo.if"(%1) ({ + + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + "tf.XlaSendToHost"(%arg1) {key = "send_key"} : (tensor) -> () + + // CHECK: mhlo.return [[ARG1]], [[SEND_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + + // CHECK-NEXT: }, { + }, { + + // CHECK: mhlo.return [[ARG1]], [[INIT_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + // CHECK-NEXT: (tensor) -> (tensor, !mhlo.token) + }) : (tensor) -> tensor + + // CHECK: mhlo.return [[ARG1]], [[INNER_IF]]#1 + "mhlo.return"(%arg1) : (tensor) -> () + + // CHECK-NEXT: }, { + }, { + + // CHECK: mhlo.return [[ARG1]], [[INIT_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + // CHECK-NEXT: (tensor) -> (tensor, !mhlo.token) + }) : (tensor) -> tensor + // CHECK-NEXT: return [[IF]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` containing a function call to TF/XLA communication ops. + +// CHECK-LABEL: func @if_function_call +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @if_function_call(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK: "mhlo.if" + %0 = "mhlo.if"(%arg0) ({ + // CHECK: [[CALL_TOKEN:%.*]] = func.call @callee([[ARG1]], [[INIT_TOKEN]]) + func.call @callee(%arg1) : (tensor) -> () + + // CHECK: mhlo.return [[ARG1]], [[CALL_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + }, { + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func private @callee +// CHECK-SAME: ([[CALLEE_ARG0:%.*]]: tensor, [[CALLEE_ARG1:%.*]]: !mhlo.token) -> !mhlo.token +func.func private @callee(%arg0: tensor) { + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send" + "tf.XlaSendToHost"(%arg0) {key = "send_key"} : (tensor) -> () + + // CHECK: return [[SEND_TOKEN]] + func.return +} + +// ----- + +// Tests `mhlo.if` containing multiple TF/XLA communication ops. + +// CHECK-LABEL: func @if_region_multiple_ops +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @if_region_multiple_ops(%arg0: tensor, %arg1: tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK: "mhlo.if" + %0 = "mhlo.if"(%arg0) ({ + // CHECK: [[SEND0_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + "tf.XlaSendToHost"(%arg1) {key = "send_key0"} : (tensor) -> () + + // CHECK: [[SEND1_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[SEND0_TOKEN]]) + "tf.XlaSendToHost"(%arg1) {key = "send_key1"} : (tensor) -> () + + // CHECK: mhlo.return [[ARG1]], [[SEND1_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + }, { + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + func.return +} + +// ----- + +// Tests `mhlo.if` containing TF/XLA communication ops followed by other TF/XLA +// communication ops. + +// CHECK-LABEL: func @if_followed_by_communication_op +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func.func @if_followed_by_communication_op(%arg0: tensor, %arg1: tensor) { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK-NEXT: [[IF:%.*]]:2 = "mhlo.if" + %0 = "mhlo.if"(%arg0) ({ + // CHECK-NEXT: [[SEND0_TOKEN:%.*]] = "mhlo.send"([[ARG1]], [[INIT_TOKEN]]) + "tf.XlaSendToHost"(%arg1) {key = "send_key0"} : (tensor) -> () + // CHECK-NEXT: mhlo.return [[ARG1]], [[SEND0_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + // CHECK-NEXT: }, { + }, { + // CHECK-NEXT: mhlo.return [[ARG1]], [[INIT_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + // CHECK-NEXT: }) + }) : (tensor) -> tensor + + // CHECK-NEXT: "mhlo.send"({{.*}}, [[IF]]#1) + "tf.XlaSendToHost"(%arg1) {key = "send_key1"} : (tensor) -> () + func.return +} + +// ----- + +// Tests `mhlo.while` with cond and body populated with TF/XLA communication +// ops. + +// CHECK-LABEL: func @while_cond_body +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @while_cond_body(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + + // CHECK: [[WHILE:%.*]]:2 = mhlo.while([[ITER_ARG_VALUE:.*]] = [[ARG0]], [[ITER_ARG_TOKEN:.*]] = [[INIT_TOKEN]]) + %0 = "mhlo.while"(%arg0) ({ + ^bb0(%arg1: tensor): + // CHECK: [[COND_SEND_TOKEN:%.*]] = "mhlo.send"([[ITER_ARG_VALUE]], [[ITER_ARG_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_while_cond_dtoh_0"} + + // CHECK: [[COND_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[COND_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_while_cond_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_cond", send_key = "send_while_cond", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: [[COND_COMPARE:%.*]] = mhlo.compare LT, [[COND_RECV_TUPLE]]#0, [[COND_RECV_TUPLE]]#0 + %2 = "mhlo.compare"(%1, %1) {comparison_direction = #mhlo} : (tensor, tensor) -> tensor + + // CHECK: mhlo.return [[COND_COMPARE]] + "mhlo.return"(%2) : (tensor) -> () + }, { + ^bb0(%arg1: tensor): + // CHECK: [[BODY_SEND_TOKEN:%.*]] = "mhlo.send"([[ITER_ARG_VALUE]], [[ITER_ARG_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_while_body_dtoh_0"} + + // CHECK: [[BODY_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[BODY_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_while_body_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_body", send_key = "send_while_body", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[BODY_RECV_TUPLE]]#0, [[BODY_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + }) : (tensor) -> tensor + + // CHECK: return [[WHILE]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` with only the `cond` region populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @while_cond +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @while_cond(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK: [[WHILE:%.*]]:2 = mhlo.while([[ITER_ARG_VALUE:.*]] = [[ARG0]], [[ITER_ARG_TOKEN:.*]] = [[INIT_TOKEN]]) + %0 = "mhlo.while"(%arg0) ({ + ^bb0(%arg1: tensor): + + // CHECK: [[COND_SEND_TOKEN:%.*]] = "mhlo.send"([[ITER_ARG_VALUE]], [[ITER_ARG_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_while_cond_dtoh_0"} + + // CHECK: [[COND_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[COND_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_while_cond_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_cond", send_key = "send_while_cond", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: [[COND_COMPARE:%.*]] = mhlo.compare LT, [[COND_RECV_TUPLE]]#0, [[COND_RECV_TUPLE]]#0 + %2 = "mhlo.compare"(%1, %1) {comparison_direction = #mhlo} : (tensor, tensor) -> tensor + + // CHECK: mhlo.return [[COND_COMPARE]] + "mhlo.return"(%2) : (tensor) -> () + }, { + ^bb0(%arg1: tensor): + // CHECK: mhlo.return [[ITER_ARG_VALUE]], [[ITER_ARG_TOKEN]] + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + + // CHECK: return [[WHILE]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` with only the `body` region populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @while_body +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func.func @while_body(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = mhlo.create_token + // CHECK: [[WHILE:%.*]]:2 = mhlo.while([[ITER_ARG_VALUE:.*]] = [[ARG0]], [[ITER_ARG_TOKEN:.*]] = [[INIT_TOKEN]]) + %0 = "mhlo.while"(%arg0) ({ + ^bb0(%arg1: tensor): + + // CHECK: [[COND_COMPARE:%.*]] = mhlo.compare LT, [[ITER_ARG_VALUE]], [[ITER_ARG_VALUE]] + %2 = "mhlo.compare"(%arg1, %arg1) {comparison_direction = #mhlo} : (tensor, tensor) -> tensor + + // CHECK: mhlo.return [[COND_COMPARE]] + "mhlo.return"(%2) : (tensor) -> () + }, { + ^bb0(%arg1: tensor): + + // CHECK: [[BODY_SEND_TOKEN:%.*]] = "mhlo.send"([[ITER_ARG_VALUE]], [[ITER_ARG_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 2 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "send_while_body_dtoh_0"} + + // CHECK: [[BODY_RECV_TUPLE:%.*]]:2 = "mhlo.recv"([[BODY_SEND_TOKEN]]) + // CHECK-SAME: channel_handle = #mhlo.channel_handle + // CHECK-SAME: handle = + // CHECK-SAME: type = 3 + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_handler_name = "tf_rendezvous", _xla_host_transfer_rendezvous = "recv_while_body_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_body", send_key = "send_while_body", host_mlir_module = ""} : (tensor) -> tensor + + // CHECK: mhlo.return [[BODY_RECV_TUPLE]]#0, [[BODY_RECV_TUPLE]]#1 + "mhlo.return"(%1) : (tensor) -> () + }) : (tensor) -> tensor + + // CHECK: return [[WHILE]]#0 + func.return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` containing TF/XLA communication ops followed by other +// TF/XLA communication ops. + +// CHECK-LABEL: func @while_followed_by_communication_op +func.func @while_followed_by_communication_op(%arg0: tensor) { + // CHECK: [[WHILE:%.*]]:2 = mhlo.while + %0 = "mhlo.while"(%arg0) ({ + ^bb0(%arg1: tensor): + "tf.XlaSendToHost"(%arg1) {key = "send_key0"} : (tensor) -> () + %1 = "mhlo.compare"(%arg1, %arg1) {comparison_direction = #mhlo} : (tensor, tensor) -> tensor + "mhlo.return"(%1) : (tensor) -> () + }, { + ^bb0(%arg1: tensor): + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + + // CHECK: "mhlo.send"({{.*}}, [[WHILE]]#1) + "tf.XlaSendToHost"(%arg0) {key = "send_key1"} : (tensor) -> () + func.return +} + +// ----- + +// Tests unsupported parent of TF/XLA communication op. + +func.func @unsupported_ancestor(%arg0: tensor, %arg1: tensor) { + %0 = "mhlo.reduce"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor, %arg3: tensor): + %1 = mhlo.add %arg2, %arg3 : tensor + // expected-error@+1 {{expects ancestor(s) to be of ['mhlo.if', 'func.func']}} + "tf._XlaHostComputeMlir"() {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : () -> () + "mhlo.return"(%1) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor, tensor) -> tensor + func.return +} + +// ----- + +// Tests transitive unsupported parent of TF/XLA communication op. + +func.func @unsupported_ancestor(%arg0: tensor, %arg1: tensor) { + %0 = "mhlo.reduce"(%arg0, %arg1) ({ + ^bb0(%arg2: tensor, %arg3: tensor): + %1 = mhlo.add %arg2, %arg3 : tensor + // expected-error@+1 {{expects ancestor(s) to be of ['mhlo.if', 'func.func']}} + func.call @callee() : () -> () + "mhlo.return"(%1) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor, tensor) -> tensor + func.return +} + +func.func private @callee() { + "tf._XlaHostComputeMlir"() {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = ""} : () -> () + func.return +} + +// ----- + +// Tests function with more than one block that is to be rewritten emits an +// error instead. + +// expected-error@+1 {{'func.func' ops with more than one block are not supported}} +func.func @multi_block_func() { + cf.br ^bb1 +^bb1: + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf_type.shape<>} : () -> tensor + func.return +} + +// ----- + +// CHECK-LABEL: func @host_compute_manual_sharding +func.func @host_compute_manual_sharding(%arg0: tensor) { + // CHECK: "mhlo.send" + // CHECK-SAME: mhlo.sharding = "\08\04" + // CHECK: "mhlo.recv" + // CHECK-SAME: mhlo.sharding = "\08\04" + %0 = "tf._XlaHostComputeMlir"(%arg0) {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", host_mlir_module = "", manual_sharding = true} : (tensor) -> tensor + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-include-tf2xla-fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-include-tf2xla-fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b28a99c52a05f45b54ad2d17af9414c94da81141 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-include-tf2xla-fallback.mlir @@ -0,0 +1,58 @@ +// RUN: tf-opt "-xla-legalize-tf=use-tf2xla-fallback=true device-type=XLA_CPU_JIT" -verify-diagnostics %s | FileCheck --check-prefix SUPPORTED_FALLBACK_DEVICE %s +// RUN: tf-opt "-xla-legalize-tf=use-tf2xla-fallback=true" %s | FileCheck --check-prefix UNSPECIFIED_FALLBACK_DEVICE %s +// RUN: tf-opt "-xla-legalize-tf=use-tf2xla-fallback=true device-type=INVALID_DEVICE_TYPE" %s | FileCheck --check-prefix UNSUPPORTED_FALLBACK_DEVICE %s + +// We run this test four times: +// 1) Legalize without using TF2XLA fallback (ops cannot be legalized). +// 2) Use fallback with a device that supports all ops (ops can be legalized). +// 3) Use fallback with unspecified device (ops cannot be legalized). +// 4) Use fallback with specified but unsupported device (ops cannot be legalized). +// +// Note: For 3) and 4) we do not use `-verify-diagnostics` because these cases +// produce remarks that don't occur for 1) and 2) and there is no way to check +// the remarks only for 3) and 4) (except using two files). + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + +// CHECK-LABEL: non_max_suppression_v4 +func.func @non_max_suppression_v4(%arg0: tensor<3x4xf32>, %arg1: tensor<3xf32>, %arg2: tensor, %arg3: tensor) -> tensor<2xi32> { + %max_size = mhlo.constant dense<2> : tensor + // NO_FALLBACK: tf.NonMaxSuppressionV4 + // SUPPORTED_FALLBACK_DEVICE-NOT: tf.NonMaxSuppressionV4 + // UNSPECIFIED_FALLBACK_DEVICE: tf.NonMaxSuppressionV4 + // UNSUPPORTED_FALLBACK_DEVICE: tf.NonMaxSuppressionV4 + %0:2 = "tf.NonMaxSuppressionV4"(%arg0, %arg1, %max_size, %arg2, %arg3) {pad_to_max_output_size = true}: (tensor<3x4xf32>, tensor<3xf32>, tensor, tensor, tensor) -> (tensor<2xi32>, tensor) + func.return %0#0 : tensor<2xi32> +} + +// CHECK-LABEL: mirror_pad +func.func @mirror_pad(%arg0: tensor<2x3xcomplex>) -> tensor<4x7xcomplex> { + %0 = mhlo.constant dense<[[1, 1], [2, 2]]> : tensor<2x2xi32> + // NO_FALLBACK: tf.MirrorPad + // SUPPORTED_FALLBACK_DEVICE-NOT: tf.MirrorPad + // UNSPECIFIED_FALLBACK_DEVICE: tf.MirrorPad + // UNSUPPORTED_FALLBACK_DEVICE: tf.MirrorPad + %1 = "tf.MirrorPad"(%arg0, %0) {mode = "SYMMETRIC"} : (tensor<2x3xcomplex>, tensor<2x2xi32>) -> tensor<4x7xcomplex> + func.return %1 : tensor<4x7xcomplex> +} + +// BatchMatMulV2 has native as well as fallback lowering patterns available. +// The fallback pattern uses dot_general without broadcast on operands and then +// transposes the output which is faster. However, the fallback pattern doesn't +// support dynamic shaped operands like the native lowering. Verify that +// fallback lowering is preferred for static shaped operands when available. + +// CHECK-LABEL: batchmatmulv2 +func.func @batchmatmulv2(%arg0: tensor<1x4x2xf32>, %arg1: tensor<3x2x4xf32>) -> tensor<3x4x4xf32> { + // NO_FALLBACK: mhlo.dynamic_broadcast_in_dim + // NO_FALLBACK: mhlo.dot_general + + // SUPPORTED_FALLBACK_DEVICE: mhlo.reduce + // SUPPORTED_FALLBACK_DEVICE: mhlo.dot_general + // SUPPORTED_FALLBACK_DEVICE: mhlo.transpose + + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, grad_x = false, grad_y = false, device = ""} : (tensor<1x4x2xf32>, tensor<3x2x4xf32>) -> tensor<3x4x4xf32> + func.return %0 : tensor<3x4x4xf32> +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-prefer-tf2xla.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-prefer-tf2xla.mlir new file mode 100644 index 0000000000000000000000000000000000000000..601734530dc46117fc53240c6e2acce96b466cf9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-prefer-tf2xla.mlir @@ -0,0 +1,230 @@ +// RUN: tf-opt "-xla-legalize-tf=device-type=XLA_CPU_JIT legalize-chlo=false use-tf2xla-fallback=true prefer-tf2xla=true" %s | FileCheck %s +// RUN: tf-opt "-xla-legalize-tf=device-type=XLA_CPU_JIT legalize-chlo=false prefer-tf2xla=true" %s | FileCheck --check-prefix NOFALLBACK %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + +// CHECK-LABEL: @abs +func.func @abs(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NOT: tf.Abs + %0 = "tf.Abs"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @testBroadcastGradientArgs +func.func @testBroadcastGradientArgs(%s0: tensor<4xi32>, %s1: tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) { + // CHECK: tf.BroadcastGradientArgs + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<4xi32>, tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> +} + +// ----- + +// CHECK-LABEL: @acos +func.func @acos(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NOT: tf.Acos + %0 = "tf.Acos"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// NOFALLBACK-LABEL: @xla_svd +func.func @xla_svd(%arg0: tensor<1x1xf32>) -> (tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32>) { + // NOFALLBACK: XlaSvd + %s, %u, %v = "tf.XlaSvd"(%arg0) {max_iter = 1, epsilon = 1.0E-09 : f32, precision_config = ""} : (tensor<1x1xf32>) -> (tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32>) + func.return %s, %u, %v : tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32> +} + +//===----------------------------------------------------------------------===// +// Random op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @random_uniform_simple +func.func @random_uniform_simple(%arg0: tensor<3xi32>) -> tensor<12x?x64xf32> { + // expected-remark@+1 {{lowering requires operand #0 to be a constant}} + %0 = "tf.RandomUniform"(%arg0) {device = "", seed = 0 : i64, seed2 = 0 : i64} : (tensor<3xi32>) -> tensor<12x?x64xf32> + func.return %0 : tensor<12x?x64xf32> +} + +// ----- +// CHECK-LABEL: func @random_uniform_with_seeds +func.func @random_uniform_with_seeds(%arg0: tensor<4xi32>) -> tensor<32x12x12x64xf32> { + // CHECK: %0 = mhlo.constant dense<[32, 12, 12, 64]> : tensor<4xi32> + // CHECK-NEXT: %1 = mhlo.constant dense<[32, 12, 12, 64]> : tensor<4xi32> + // CHECK-NEXT: %2 = "mhlo.slice"(%1) <{limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<4xi32>) -> tensor<1xi32> + // CHECK-NEXT: %3 = mhlo.reshape %2 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %4 = mhlo.convert %3 : tensor + // CHECK-NEXT: %5 = "mhlo.slice"(%1) <{limit_indices = dense<2> : tensor<1xi64>, start_indices = dense<1> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<4xi32>) -> tensor<1xi32> + // CHECK-NEXT: %6 = mhlo.reshape %5 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %7 = mhlo.convert %6 : tensor + // CHECK-NEXT: %8 = "mhlo.slice"(%1) <{limit_indices = dense<3> : tensor<1xi64>, start_indices = dense<2> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<4xi32>) -> tensor<1xi32> + // CHECK-NEXT: %9 = mhlo.reshape %8 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %10 = mhlo.convert %9 : tensor + // CHECK-NEXT: %11 = "mhlo.slice"(%1) <{limit_indices = dense<4> : tensor<1xi64>, start_indices = dense<3> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<4xi32>) -> tensor<1xi32> + // CHECK-NEXT: %12 = mhlo.reshape %11 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %13 = mhlo.convert %12 : tensor + // CHECK-NEXT: %14 = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-NEXT: %15 = mhlo.constant dense<1.000000e+00> : tensor + // CHECK-NEXT: %16 = mhlo.constant dense<[32, 12, 12, 64]> : tensor<4xi64> + // CHECK-NEXT: %17 = "mhlo.rng"(%14, %15, %16) <{rng_distribution = #mhlo.rng_distribution}> : (tensor, tensor, tensor<4xi64>) -> tensor<32x12x12x64xf32> + %cst = "tf.Const"() {value = dense<[32, 12, 12, 64]> : tensor<4xi32>} : () -> tensor<4xi32> + %0 = "tf.RandomUniform"(%cst) {seed = 87654321 : i64, seed2 = 0 : i64} : (tensor<4xi32>) -> tensor<32x12x12x64xf32> + // CHECK: return %17 : tensor<32x12x12x64xf32> + func.return %0 : tensor<32x12x12x64xf32> +} + +// CHECK-LABEL: slice_variable_start +func.func @slice_variable_start(%arg0: tensor<3x4xi32>, %arg1: tensor<2xi64>) -> tensor<1x4xi32> { + // CHECK: %[[SLICED_START1:.*]] = "mhlo.slice"(%arg1) + // CHECK-DAG-SAME: {limit_indices = dense<1> : tensor<1xi64>, + // CHECK-DAG-SAME: start_indices = dense<0> : tensor<1xi64>, + // CHECK-DAG-SAME: strides = dense<1> : tensor<1xi64>} : (tensor<2xi64>) -> tensor<1xi64> + // CHECK: %[[RESHAPED_START1:.*]] = mhlo.reshape %[[SLICED_START1]] : (tensor<1xi64>) -> tensor + // CHECK: %[[SLICED_START2:.*]] = "mhlo.slice"(%arg1) + // CHECK-DAG-SAME: {limit_indices = dense<2> : tensor<1xi64>, + // CHECK-DAG-SAME: start_indices = dense<1> : tensor<1xi64>, + // CHECK-DAG-SAME: strides = dense<1> : tensor<1xi64>} : (tensor<2xi64>) -> tensor<1xi64> + // CHECK: %[[RESHAPED_START2:.*]] = mhlo.reshape %[[SLICED_START2]] : (tensor<1xi64>) -> tensor + // CHECK: %[[RESULT:.*]] = "mhlo.dynamic_slice"(%arg0, %[[RESHAPED_START1]], %[[RESHAPED_START2]]) <{slice_sizes = dense<[1, 4]> : tensor<2xi64>}> : (tensor<3x4xi32>, tensor, tensor) -> tensor<1x4xi32> + // CHECK: return %[[RESULT]] : tensor<1x4xi32> + %sizes = "tf.Const"() {value = dense<[1, 4]> : tensor<2xi64>} : () -> (tensor<2xi64>) + %0 = "tf.Slice"(%arg0, %arg1, %sizes) : (tensor<3x4xi32>, tensor<2xi64>, tensor<2xi64>) -> tensor<1x4xi32> + func.return %0 : tensor<1x4xi32> +} + +// CHECK-LABEL: slice_variable_start_negsize +func.func @slice_variable_start_negsize(%arg0: tensor<3x4xi32>, %arg1: tensor<2xi32>) -> tensor<1x4xi32> { + // CHECK: %0 = mhlo.constant dense<[1, -1]> : tensor<2xi32> + // CHECK-NEXT: %1 = mhlo.constant dense<[1, -1]> : tensor<2xi32> + // CHECK-NEXT: %2 = "mhlo.slice"(%1) <{limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi32>) -> tensor<1xi32> + // CHECK-NEXT: %3 = mhlo.reshape %2 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %4 = mhlo.constant dense<[1, -1]> : tensor<2xi32> + // CHECK-NEXT: %5 = "mhlo.slice"(%4) <{limit_indices = dense<2> : tensor<1xi64>, start_indices = dense<1> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi32>) -> tensor<1xi32> + // CHECK-NEXT: %6 = mhlo.reshape %5 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %7 = mhlo.constant dense<3> : tensor + // CHECK-NEXT: %8 = "mhlo.set_dimension_size"(%arg0, %7) <{dimension = 0 : i64}> : (tensor<3x4xi32>, tensor) -> tensor<3x4xi32> + // CHECK-NEXT: %9 = mhlo.constant dense<4> : tensor + // CHECK-NEXT: %10 = "mhlo.set_dimension_size"(%8, %9) <{dimension = 1 : i64}> : (tensor<3x4xi32>, tensor) -> tensor<3x4xi32> + // CHECK-NEXT: %11 = mhlo.constant dense<0> : tensor + // CHECK-NEXT: %12 = "mhlo.pad"(%10, %11) <{edge_padding_high = dense<[3, 4]> : tensor<2xi64>, edge_padding_low = dense<0> : tensor<2xi64>, interior_padding = dense<0> : tensor<2xi64>}> : (tensor<3x4xi32>, tensor) -> tensor<6x8xi32> + // CHECK-NEXT: %13 = "mhlo.slice"(%arg1) <{limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi32>) -> tensor<1xi32> + // CHECK-NEXT: %14 = mhlo.reshape %13 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %15 = "mhlo.slice"(%arg1) <{limit_indices = dense<2> : tensor<1xi64>, start_indices = dense<1> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi32>) -> tensor<1xi32> + // CHECK-NEXT: %16 = mhlo.reshape %15 : (tensor<1xi32>) -> tensor + // CHECK-NEXT: %17 = "mhlo.dynamic_slice"(%12, %14, %16) <{slice_sizes = dense<[3, 4]> : tensor<2xi64>}> : (tensor<6x8xi32>, tensor, tensor) -> tensor<3x4xi32> + // CHECK-NEXT: %18 = "mhlo.slice"(%17) <{limit_indices = dense<[1, 4]> : tensor<2xi64>, start_indices = dense<0> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>}> : (tensor<3x4xi32>) -> tensor<1x4xi32> + // CHECK-NEXT: %19 = mhlo.constant dense<4> : tensor + // CHECK-NEXT: %20 = mhlo.subtract %19, %16 : tensor + // CHECK-NEXT: %21 = "mhlo.set_dimension_size"(%18, %20) <{dimension = 1 : i64}> : (tensor<1x4xi32>, tensor) -> tensor<1x?xi32, #mhlo.type_extensions> + // CHECK-NEXT: %cast = tensor.cast %21 : tensor<1x?xi32, #mhlo.type_extensions> to tensor<1x4xi32> + // CHECK-NEXT: return %cast : tensor<1x4xi32> + %sizes = "tf.Const"() {value = dense<[1, -1]> : tensor<2xi32>} : () -> (tensor<2xi32>) + %0 = "tf.Slice"(%arg0, %arg1, %sizes) : (tensor<3x4xi32>, tensor<2xi32>, tensor<2xi32>) -> tensor<1x4xi32> + func.return %0 : tensor<1x4xi32> +} + +//===----------------------------------------------------------------------===// +// StridedSlice op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: simple_strided_slice +func.func @simple_strided_slice(%input: tensor<4x8xf32>) -> tensor<3x2xf32> { + // CHECK: %0 = mhlo.constant dense<[0, 1]> : tensor<2xi32> + // CHECK-NEXT: %1 = mhlo.constant dense<[3, 7]> : tensor<2xi32> + // CHECK-NEXT: %2 = mhlo.constant dense<[1, 3]> : tensor<2xi32> + // CHECK-NEXT: %3 = "mhlo.slice"(%arg0) + %begin = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi32>} : () -> (tensor<2xi32>) + %end = "tf.Const"() {value = dense<[3, 7]> : tensor<2xi32>} : () -> (tensor<2xi32>) + %strides = "tf.Const"() {value = dense<[1, 3]> : tensor<2xi32>} : () -> (tensor<2xi32>) + + %output = "tf.StridedSlice"(%input, %begin, %end, %strides) + : (tensor<4x8xf32>, tensor<2xi32>, tensor<2xi32>, tensor<2xi32>) -> tensor<3x2xf32> + func.return %output : tensor<3x2xf32> + // CHECK: return %4 : tensor<3x2xf32> +} + +// ----- + +// CHECK-LABEL: strided_slice_with_uknown_begin +func.func @strided_slice_with_uknown_begin(%input: tensor<4x8xf32>, %begin: tensor<1xi32>) -> tensor<4x8xf32> { + %end = "tf.Const"() {value = dense<[3]> : tensor<1xi32>} : () -> (tensor<1xi32>) + %strides = "tf.Const"() {value = dense<[1]> : tensor<1xi32>} : () -> (tensor<1xi32>) + + // CHECK: %[[DYNAMIC_SLICE:.*]] = "mhlo.dynamic_slice" + %output = "tf.StridedSlice"(%input, %begin, %end, %strides) + : (tensor<4x8xf32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<4x8xf32> + func.return %output : tensor<4x8xf32> +} + +//===----------------------------------------------------------------------===// +// Fused op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: fused_conv2d +func.func @fused_conv2d(%input: tensor<1x300x300x40xi8>, + %filter: tensor<3x3x40x40xi8>, + %bias: tensor<40xf32>, + %act: tensor<0xi8>) -> tensor<1x300x300x40xi8> { + + // CHECK: %[[v0:.*]] = mhlo.constant dense<1.000000e+00> : tensor + // CHECK-NEXT: %[[v1:.*]] = mhlo.constant dense<2.000000e+00> : tensor + // CHECK-NEXT: %[[v2:.*]] = mhlo.constant dense<2.000000e+00> : tensor + // CHECK-NEXT: %[[v3:.*]] = mhlo.constant dense<-1.280000e+02> : tensor + // CHECK-NEXT: %[[v4:.*]] = "mhlo.broadcast_in_dim"(%3) <{broadcast_dimensions = dense<> : tensor<0xi64>}> : (tensor) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %[[v5:.*]] = mhlo.convert %arg0 : (tensor<1x300x300x40xi8>) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %[[v6:.*]] = mhlo.convert %arg1 : (tensor<3x3x40x40xi8>) -> tensor<3x3x40x40xf32> + // CHECK: %[[v7:.*]] = mhlo.convolution(%[[v5]], %[[v6]]) + // CHECK-SAME{LITERAL}: dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f] + // CHECK-SAME{LITERAL}: window = {stride = [1, 1], pad = [[1, 1], [1, 1]], lhs_dilate = [1, 1], rhs_dilate = [1, 1], reverse = [0, 0]} + // CHECK-SAME: batch_group_count = 1 + // CHECK-SAME: feature_group_count = 1 + // CHECK-NEXT: %[[v8:.*]] = mhlo.convert %7 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %[[v9:.*]] = mhlo.constant dense<1.000000e+00> : tensor + // CHECK-NEXT: %[[v10:.*]] = "mhlo.broadcast_in_dim"(%9) <{broadcast_dimensions = dense<> : tensor<0xi64>}> : (tensor) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %11 = mhlo.multiply %8, %10 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %12 = mhlo.convert %arg2 : tensor<40xf32> + // CHECK-NEXT: %13 = "mhlo.broadcast_in_dim"(%12) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<40xf32>) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %14 = mhlo.add %11, %13 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %15 = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-NEXT: %16 = "mhlo.broadcast_in_dim"(%15) <{broadcast_dimensions = dense<> : tensor<0xi64>}> : (tensor) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %17 = mhlo.maximum %14, %16 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %18 = mhlo.constant dense<1.270000e+02> : tensor + // CHECK-NEXT: %19 = "mhlo.broadcast_in_dim"(%18) <{broadcast_dimensions = dense<> : tensor<0xi64>}> : (tensor) -> tensor<1x300x300x40xf32> + // CHECK-NEXT: %20 = mhlo.clamp %4, %17, %19 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %21 = mhlo.round_nearest_even %20 : tensor<1x300x300x40xf32> + // CHECK-NEXT: %22 = mhlo.convert %21 : (tensor<1x300x300x40xf32>) -> tensor<1x300x300x40xi8> + // CHECK-NEXT: return %22 : tensor<1x300x300x40xi8> + %input_scale = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %side_input_scale = "tf.Const"() {value = dense<2.0> : tensor} : () -> tensor + %conv2d = "tf._FusedConv2D"(%input, %filter, %bias, %act, %input_scale, %side_input_scale) { + data_format = "NHWC", dilations = [1, 1, 1, 1], epsilon = 9.99999974E-5 : f32, explicit_paddings = [], filter_format = "HWIO", fused_ops = ["BiasAdd", "Relu"], leakyrelu_alpha = 2.000000e-01 : f32, num_args = 2 : i64, operandSegmentSizes = array, padding = "SAME", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true + } : (tensor<1x300x300x40xi8>, tensor<3x3x40x40xi8>, tensor<40xf32>, tensor<0xi8>, tensor, tensor) -> tensor<1x300x300x40xi8> + func.return %conv2d : tensor<1x300x300x40xi8> +} + +//===----------------------------------------------------------------------===// +// tf.UniqueV2 legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @doesnt_legalize_uniquev2 +func.func @doesnt_legalize_uniquev2(%arg0: tensor<3xf32>) -> (tensor, tensor<3xi32>) { + //CHECK: %0 = mhlo.constant dense<0> : tensor<1xi32> + //CHECK-NEXT: %y, %idx = "tf.UniqueV2"(%arg0, %0) {device = ""} : (tensor<3xf32>, tensor<1xi32>) -> (tensor, tensor<3xi32>) + //CHECK-NEXT: return %y, %idx : tensor, tensor<3xi32> + + %cst = "tf.Const"() {device = "", value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %y, %idx = "tf.UniqueV2"(%arg0, %cst) {device = ""} : (tensor<3xf32>, tensor<1xi32>) -> (tensor, tensor<3xi32>) + func.return %y, %idx : tensor, tensor<3xi32> +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-quant.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-quant.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a15f13b96225e84e41975e80f00d4830df673d44 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-quant.mlir @@ -0,0 +1,562 @@ +// RUN: tf-opt -convert-tf-quant-types -quant-convert-tf-quant-ops-to-mhlo -canonicalize "-xla-legalize-tf=legalize-chlo=false" -split-input-file %s | env FILECHECK_OPTS="" FileCheck %s + + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedDotHybrid legalization +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: func @quantized_matmul_fn +func.func @quantized_matmul_fn(%input: tensor) -> tensor { + %weight = "tf.Const"() { value = #tf_type : tensor<2x2x!tf_type.qint8> } : () -> tensor<2x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + // CHECK: %[[CONST:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL}: value = dense<[[1, 2], [3, 4]]> : tensor<2x2xi8> + // CHECK-SAME: tensor<2x2x!quant.uniform> + // CHECK: "mhlo.dot"(%arg0, %[[CONST]]) : (tensor, tensor<2x2x!quant.uniform>) -> tensor + + %0 = "tf.UniformQuantizedDotHybrid"(%input, %weight, %weight_scales, %weight_zps) {rhs_quantization_axis = -1 : i64, rhs_quantization_min_val = -128 : i64, rhs_quantization_max_val = 127 : i64} : (tensor, tensor<2x2x!tf_type.qint8>, tensor, tensor) -> tensor + func.return %0 : tensor +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedConvolutionHybrid legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantized_convolution_hybrid +func.func @uniform_quantized_convolution_hybrid(%input: tensor<1x6x6x3xf32>) -> tensor<1x4x1x2xf32> { + %weight = "tf.Const"() {value = #tf_type : tensor<2x3x3x2x!tf_type.qint8>} : () -> tensor<2x3x3x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + // CHECK: %[[CONST:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL} value = dense<127> : tensor<2x3x3x2xi8> + // CHECK-SAME: tensor<2x3x3x2x!quant.uniform> + // CHECK: mhlo.convolution(%arg0, %[[CONST]]) + // CHECK-SAME{LITERAL}: dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f] + // CHECK-SAME{LITERAL}: window = {stride = [1, 2], pad = [[0, 0], [0, 0]], lhs_dilate = [1, 1], rhs_dilate = [2, 2]} + // CHECK-SAME{LITERAL}: batch_group_count = 1 : i64, feature_group_count = 1 : i64 + // CHECK-SAME: (tensor<1x6x6x3xf32>, tensor<2x3x3x2x!quant.uniform>) -> tensor<1x4x1x2xf32> + + %0 = "tf.UniformQuantizedConvolutionHybrid"(%input, %weight, %weight_scales, %weight_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64 + } : (tensor<1x6x6x3xf32>, tensor<2x3x3x2x!tf_type.qint8>, tensor, tensor) -> tensor<1x4x1x2xf32> + func.return %0 : tensor<1x4x1x2xf32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantized_convolution_hybrid_same +func.func @uniform_quantized_convolution_hybrid_same(%input: tensor<1x2x2x3xf32>) -> tensor<1x2x1x2xf32> { + %weight = "tf.Const"() {value = #tf_type : tensor<2x3x3x2x!tf_type.qint8>} : () -> tensor<2x3x3x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + // CHECK: %[[CONST:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL} value = dense<127> : tensor<2x3x3x2xi8> + // CHECK-SAME: tensor<2x3x3x2x!quant.uniform> + // CHECK: mhlo.convolution(%arg0, %[[CONST]]) + // CHECK-SAME{LITERAL}: dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f] + // CHECK-SAME{LITERAL}: window = {stride = [1, 2], pad = [[1, 1], [1, 2]], lhs_dilate = [1, 1], rhs_dilate = [2, 2]} + // CHECK-SAME{LITERAL}: batch_group_count = 1 : i64, feature_group_count = 1 : i64 + // CHECK-SAME: (tensor<1x2x2x3xf32>, tensor<2x3x3x2x!quant.uniform>) -> tensor<1x2x1x2xf32> + + %0 = "tf.UniformQuantizedConvolutionHybrid"(%input, %weight, %weight_scales, %weight_zps) { + window_strides = [1, 2], + padding = "SAME", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64 + } : (tensor<1x2x2x3xf32>, tensor<2x3x3x2x!tf_type.qint8>, tensor, tensor) -> tensor<1x2x1x2xf32> + func.return %0 : tensor<1x2x1x2xf32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantized_convolution_hybrid_explicit +func.func @uniform_quantized_convolution_hybrid_explicit(%input: tensor<1x2x2x3xf32>) -> tensor<1x3x3x2xf32> { + %weight = "tf.Const"() {value = #tf_type : tensor<2x3x3x2x!tf_type.qint8>} : () -> tensor<2x3x3x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + // CHECK: %[[CONST:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL} value = dense<127> : tensor<2x3x3x2xi8> + // CHECK-SAME: tensor<2x3x3x2x!quant.uniform> + // CHECK: mhlo.convolution(%arg0, %[[CONST]]) + // CHECK-SAME{LITERAL}: dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f] + // CHECK-SAME{LITERAL}: window = {stride = [1, 2], pad = [[1, 2], [3, 4]], lhs_dilate = [1, 1], rhs_dilate = [2, 2]} + // CHECK-SAME{LITERAL}: batch_group_count = 1 : i64, feature_group_count = 1 : i64 + // CHECK-SAME: (tensor<1x2x2x3xf32>, tensor<2x3x3x2x!quant.uniform>) -> tensor<1x3x3x2xf32> + + %0 = "tf.UniformQuantizedConvolutionHybrid"(%input, %weight, %weight_scales, %weight_zps) { + window_strides = [1, 2], + padding = "EXPLICIT", + explicit_padding = [1, 2, 3, 4], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64 + } : (tensor<1x2x2x3xf32>, tensor<2x3x3x2x!tf_type.qint8>, tensor, tensor) -> tensor<1x3x3x2xf32> + func.return %0 : tensor<1x3x3x2xf32> +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantize and tf.UniformDequantize legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantize_and_dequantize +func.func @uniform_quantize_and_dequantize(%arg0 : tensor<2xf32>) -> tensor<2xf32> { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + // CHECK: %[[QUANTIZE:.*]] = mhlo.uniform_quantize %arg0 : (tensor<2xf32>) -> tensor<2x!quant.uniform> + // CHECK: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[QUANTIZE]] : (tensor<2x!quant.uniform>) -> tensor<2xi8> + // CHECK: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<2xi8>) -> tensor<2x!quant.uniform> + // CHECK: %[[DEQUANTIZE:.*]] = mhlo.uniform_dequantize %[[CONVERT_2]] : (tensor<2x!quant.uniform>) -> tensor<2xf32> + // CHECK: return %[[DEQUANTIZE]] : tensor<2xf32> + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2xf32>, tensor, tensor) -> tensor<2x!tf_type.qint8> + %1 = "tf.UniformDequantize"(%0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x!tf_type.qint8>, tensor, tensor) -> tensor<2xf32> + func.return %1 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantize_and_dequantize_per_axis +func.func @uniform_quantize_and_dequantize_per_axis(%arg0 : tensor<2x2xf32>) -> tensor<2x2xf32> { + %scales = "tf.Const"() { value = dense<[1.0, 2.0]> : tensor<2xf32> } : () -> tensor<2xf32> + %zps = "tf.Const"() { value = dense<[3, 4]> : tensor<2xi32> } : () -> tensor<2xi32> + + // CHECK: %[[QUANTIZE:.*]] = mhlo.uniform_quantize %arg0 : (tensor<2x2xf32>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[QUANTIZE]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xi8> + // CHECK: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<2x2xi8>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[DEQUANTIZE:.*]] = mhlo.uniform_dequantize %[[CONVERT_2]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xf32> + // CHECK: return %[[DEQUANTIZE]] : tensor<2x2xf32> + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = 0 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2xf32>, tensor<2xf32>, tensor<2xi32>) -> tensor<2x2x!tf_type.qint8> + %1 = "tf.UniformDequantize"(%0, %scales, %zps) { + quantization_axis = 0 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<2xi32>) -> tensor<2x2xf32> + func.return %1 : tensor<2x2xf32> +} + +//===----------------------------------------------------------------------===// +// tf.UniformRequantize legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantize_requantize_and_dequantize +func.func @uniform_quantize_requantize_and_dequantize(%arg0 : tensor<4xf32>) -> tensor<4xf32> { + %scales_0 = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps_0 = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + %scales_1 = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %zps_1 = "tf.Const"() { value = dense<5> : tensor } : () -> tensor + + // CHECK: %[[QUANTIZE:.*]] = mhlo.uniform_quantize %arg0 : (tensor<4xf32>) -> tensor<4x!quant.uniform> + // CHECK: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[QUANTIZE]] : (tensor<4x!quant.uniform>) -> tensor<4xi8> + // CHECK: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<4xi8>) -> tensor<4x!quant.uniform> + // CHECK: %[[REQUANTIZE:.*]] = mhlo.uniform_quantize %[[CONVERT_2]] : (tensor<4x!quant.uniform>) -> tensor<4x!quant.uniform> + // CHECK: %[[CONVERT_3:.*]] = mhlo.bitcast_convert %[[REQUANTIZE]] : (tensor<4x!quant.uniform>) -> tensor<4xi8> + // CHECK: %[[CONVERT_4:.*]] = mhlo.bitcast_convert %[[CONVERT_3]] : (tensor<4xi8>) -> tensor<4x!quant.uniform> + // CHECK: %[[DEQUANTIZE:.*]] = mhlo.uniform_dequantize %[[CONVERT_4]] : (tensor<4x!quant.uniform>) -> tensor<4xf32> + // CHECK: return %[[DEQUANTIZE]] : tensor<4xf32> + + %0 = "tf.UniformQuantize"(%arg0, %scales_0, %zps_0) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<4xf32>, tensor, tensor) -> tensor<4x!tf_type.qint8> + %1 = "tf.UniformRequantize"(%0, %scales_0, %zps_0, %scales_1, %zps_1) { + input_quantization_axis = -1 : i64, input_quantization_min_val = -128 : i64, input_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, output_quantization_min_val = -128 : i64, output_quantization_max_val = 127 : i64 + } : (tensor<4x!tf_type.qint8>, tensor, tensor, tensor, tensor) -> tensor<4x!tf_type.qint8> + %2 = "tf.UniformDequantize"(%1, %scales_1, %zps_1) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<4x!tf_type.qint8>, tensor, tensor) -> tensor<4xf32> + func.return %2 : tensor<4xf32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantize_requantize_and_dequantize_per_axis +func.func @uniform_quantize_requantize_and_dequantize_per_axis(%arg0 : tensor<2x2xf32>) -> tensor<2x2xf32> { + %scales_0 = "tf.Const"() { value = dense<[1.0, 2.0]> : tensor<2xf32> } : () -> tensor<2xf32> + %zps_0 = "tf.Const"() { value = dense<[3, 4]> : tensor<2xi32> } : () -> tensor<2xi32> + %scales_1 = "tf.Const"() { value = dense<[3.0, 4.0]> : tensor<2xf32> } : () -> tensor<2xf32> + %zps_1 = "tf.Const"() { value = dense<[5, 6]> : tensor<2xi32> } : () -> tensor<2xi32> + + // CHECK: %[[QUANTIZE:.*]] = mhlo.uniform_quantize %arg0 : (tensor<2x2xf32>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[QUANTIZE]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xi8> + // CHECK: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<2x2xi8>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[REQUANTIZE:.*]] = mhlo.uniform_quantize %[[CONVERT_2]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[CONVERT_3:.*]] = mhlo.bitcast_convert %[[REQUANTIZE]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xi8> + // CHECK: %[[CONVERT_4:.*]] = mhlo.bitcast_convert %[[CONVERT_3]] : (tensor<2x2xi8>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[DEQUANTIZE:.*]] = mhlo.uniform_dequantize %[[CONVERT_4]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xf32> + // CHECK: return %[[DEQUANTIZE]] : tensor<2x2xf32> + + %0 = "tf.UniformQuantize"(%arg0, %scales_0, %zps_0) { + quantization_axis = 0 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2xf32>, tensor<2xf32>, tensor<2xi32>) -> tensor<2x2x!tf_type.qint8> + %1 = "tf.UniformRequantize"(%0, %scales_0, %zps_0, %scales_1, %zps_1) { + input_quantization_axis = 0 : i64, input_quantization_min_val = -128 : i64, input_quantization_max_val = 127 : i64, + output_quantization_axis = 0 : i64, output_quantization_min_val = -128 : i64, output_quantization_max_val = 127 : i64 + } : (tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<2xi32>, tensor<2xf32>, tensor<2xi32>) -> tensor<2x2x!tf_type.qint8> + %2 = "tf.UniformDequantize"(%1, %scales_1, %zps_1) { + quantization_axis = 0 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2x!tf_type.qint8>, tensor<2xf32>, tensor<2xi32>) -> tensor<2x2xf32> + func.return %2 : tensor<2x2xf32> +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedDot legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantized_dot +func.func @uniform_quantized_dot(%input: tensor) -> tensor { + %input_scales = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %input_zps = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + + %weight = "tf.Const"() { value = #tf_type : tensor<2x2x!tf_type.qint8> } : () -> tensor<2x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + %output_scales = "tf.Const"() { value = dense<3.0> : tensor } : () -> tensor + %output_zps = "tf.Const"() { value = dense<5> : tensor } : () -> tensor + + // CHECK-DAG: %[[RHS:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL}: <{value = dense<[[1, 2], [3, 4]]> : tensor<2x2xi8>}> : () -> tensor<2x2x!quant.uniform> + // CHECK-DAG: %[[LHS:.*]] = mhlo.uniform_quantize %arg0 : (tensor) -> tensor> + // CHECK-DAG: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[LHS]] : (tensor>) -> tensor + // CHECK-DAG: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor) -> tensor> + // CHECK: "mhlo.dot"(%[[CONVERT_2]], %[[RHS]]) : (tensor>, tensor<2x2x!quant.uniform>) + // CHECK-SAME: -> tensor> + + %0 = "tf.UniformQuantize"(%input, %input_scales, %input_zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + %1 = "tf.UniformQuantizedDot"( + %0, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor, tensor<2x2x!tf_type.qint8>, + tensor, tensor, + tensor, tensor, + tensor, tensor) -> tensor + func.return %1 : tensor +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedConvolution legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantized_convolution +func.func @uniform_quantized_convolution(%input: tensor<1x6x6x3xf32>) -> tensor<1x4x1x2x!tf_type.qint32> { + %input_scales = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %input_zps = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + + %weight = "tf.Const"() {value = #tf_type : tensor<2x3x3x2x!tf_type.qint8>} : () -> tensor<2x3x3x2x!tf_type.qint8> + %weight_scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %weight_zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + + %output_scales = "tf.Const"() { value = dense<3.0> : tensor } : () -> tensor + %output_zps = "tf.Const"() { value = dense<5> : tensor } : () -> tensor + + // CHECK-DAG: %[[RHS:.*]] = mhlo.constant() + // CHECK-SAME{LITERAL}: <{value = dense<127> : tensor<2x3x3x2xi8>}> : () -> tensor<2x3x3x2x!quant.uniform> + // CHECK-DAG: %[[LHS:.*]] = mhlo.uniform_quantize %arg0 : (tensor<1x6x6x3xf32>) -> tensor<1x6x6x3x!quant.uniform> + // CHECK-DAG: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[LHS]] : (tensor<1x6x6x3x!quant.uniform>) -> tensor<1x6x6x3xi8> + // CHECK-DAG: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<1x6x6x3xi8>) -> tensor<1x6x6x3x!quant.uniform> + // CHECK: mhlo.convolution(%[[CONVERT_2]], %[[RHS]]) + // CHECK-SAME{LITERAL}: dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f] + // CHECK-SAME{LITERAL}: window = {stride = [1, 2], pad = [[0, 0], [0, 0]], lhs_dilate = [1, 1], rhs_dilate = [2, 2]} + // CHECK-SAME{LITERAL}: batch_group_count = 1 : i64, feature_group_count = 1 : i64 + // CHECK-SAME: (tensor<1x6x6x3x!quant.uniform>, tensor<2x3x3x2x!quant.uniform>) -> tensor<1x4x1x2x!quant.uniform> + + %0 = "tf.UniformQuantize"(%input, %input_scales, %input_zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<1x6x6x3xf32>, tensor, tensor) -> tensor<1x6x6x3x!tf_type.qint8> + %1 = "tf.UniformQuantizedConvolution"( + %0, %weight, + %input_scales, %input_zps, + %weight_scales, %weight_zps, + %output_scales, %output_zps) { + window_strides = [1, 2], + padding = "VALID", + explicit_padding = [], + lhs_dilation = [1, 1], + rhs_dilation = [2, 2], + batch_group_count = 1 : i64, + feature_group_count = 1 : i64, + dimension_numbers = "\10\03\1A\02\01\02 \02(\032\02\00\01@\03J\02\01\02", + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -128 : i64, + lhs_quantization_max_val = 127 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -128 : i64, + rhs_quantization_max_val = 127 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<1x6x6x3x!tf_type.qint8>, tensor<2x3x3x2x!tf_type.qint8>, + tensor, tensor, + tensor, tensor, + tensor, tensor) -> tensor<1x4x1x2x!tf_type.qint32> + func.return %1 : tensor<1x4x1x2x!tf_type.qint32> +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedAdd legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantized_add +func.func @uniform_quantized_add(%arg0: tensor<3x2x!tf_type.qint32>) -> tensor<3x2x!tf_type.qint32> { + %input_scales = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %input_zps = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + // tensor_proto that points to dense<127> of type !tf_type.qint32. + %bias = "tf.Const"() { value = #tf_type : tensor<2x!tf_type.qint32> } : () -> tensor<2x!tf_type.qint32> + %bias_scales = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %bias_zps = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + + %output_scales = "tf.Const"() { value = dense<2.0> : tensor } : () -> tensor + %output_zps = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + + // CHECK-DAG: %[[LHS:.*]] = mhlo.bitcast_convert %arg0 : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK-DAG: %[[RHS:.*]] = mhlo.constant() <{value = dense<127> : tensor<2xi32>}> : () -> tensor<2x!quant.uniform> + // CHECK: %[[RES:.*]] = chlo.broadcast_add %[[LHS]], %[[RHS]] {broadcast_dimensions = array} : + // CHECK-SAME: (tensor<3x2x!quant.uniform>, tensor<2x!quant.uniform>) + // CHECK-SAME: -> tensor<3x2x!quant.uniform> + // CHECK: %[[RES_INT:.*]] = mhlo.bitcast_convert %[[RES]] : (tensor<3x2x!quant.uniform>) -> tensor<3x2xi32> + // CHECK: return %[[RES_INT]] : tensor<3x2xi32> + + %0 = "tf.UniformQuantizedAdd"( + %arg0, %bias, + %input_scales, %input_zps, + %bias_scales, %bias_zps, + %output_scales, %output_zps) { + lhs_quantization_axis = -1 : i64, + lhs_quantization_min_val = -2147483648 : i64, + lhs_quantization_max_val = 2147483647 : i64, + rhs_quantization_axis = -1 : i64, + rhs_quantization_min_val = -2147483648 : i64, + rhs_quantization_max_val = 2147483647 : i64, + output_quantization_axis = -1 : i64, + output_quantization_min_val = -2147483648 : i64, + output_quantization_max_val = 2147483647 : i64} : ( + tensor<3x2x!tf_type.qint32>, tensor<2x!tf_type.qint32>, + tensor, tensor, + tensor, tensor, + tensor, tensor) -> tensor<3x2x!tf_type.qint32> + func.return %0 : tensor<3x2x!tf_type.qint32> +} + +//===----------------------------------------------------------------------===// +// tf.UniformQuantizedClipByValue legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @uniform_quantized_clip_by_value +func.func @uniform_quantized_clip_by_value(%input: tensor<3x2xf32>) -> tensor<3x2x!tf_type.qint32> { + %scales = "tf.Const"() { value = dense<2.0> : tensor<2xf32> } : () -> tensor<2xf32> + %zps = "tf.Const"() { value = dense<4> : tensor<2xi32> } : () -> tensor<2xi32> + + // tensor_proto that points to dense<127> of type !tf_type.qint32. + // CHECK-DAG: %[[MIN_MAX:.*]] = mhlo.constant() <{value = dense<127> : tensor<3x2xi32>}> : () -> tensor<3x2x!quant.uniform> + %min = "tf.Const"() { value = #tf_type : tensor<3x2x!tf_type.qint32> } : () -> tensor<3x2x!tf_type.qint32> + %max = "tf.Const"() { value = #tf_type : tensor<3x2x!tf_type.qint32> } : () -> tensor<3x2x!tf_type.qint32> + + // CHECK-DAG: %[[OPERAND:.*]] = mhlo.uniform_quantize %arg0 : (tensor<3x2xf32>) -> tensor<3x2x!quant.uniform> + %0 = "tf.UniformQuantize"(%input, %scales, %zps) { + quantization_axis = 1 : i64, quantization_min_val = -2147483648 : i64, quantization_max_val = 2147483647 : i64 + } : (tensor<3x2xf32>, tensor<2xf32>, tensor<2xi32>) -> tensor<3x2x!tf_type.qint32> + + // CHECK-DAG: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[OPERAND]] : (tensor<3x2x!quant.uniform>) -> tensor<3x2xi32> + // CHECK-DAG: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK: %[[MIN_CLIPPED:.*]] = chlo.broadcast_maximum %[[CONVERT_2]], %[[MIN_MAX]] {broadcast_dimensions = array} : + // CHECK-SAME: (tensor<3x2x!quant.uniform>, tensor<3x2x!quant.uniform>) + // CHECK-SAME: -> tensor<3x2x!quant.uniform> + // CHECK: %[[MAX_CLIPPED:.*]] = chlo.broadcast_minimum %[[MIN_CLIPPED]], %[[MIN_MAX]] {broadcast_dimensions = array} : + // CHECK-SAME: (tensor<3x2x!quant.uniform>, tensor<3x2x!quant.uniform>) + // CHECK-SAME: -> tensor<3x2x!quant.uniform> + // CHECK: %[[RESULT:.*]] = mhlo.bitcast_convert %[[MAX_CLIPPED]] : (tensor<3x2x!quant.uniform>) -> tensor<3x2xi32> + // CHECK: return %[[RESULT]] : tensor<3x2xi32> + %1 = "tf.UniformQuantizedClipByValue"(%0, %min, %max, %scales, %zps) { + quantization_axis = 1 : i64, + quantization_min_val = -2147483648 : i64, + quantization_max_val = 2147483647 : i64 + } : (tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<2xf32>, tensor<2xi32>) -> tensor<3x2x!tf_type.qint32> + func.return %1 : tensor<3x2x!tf_type.qint32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantized_clip_by_value_min_not_const +func.func @uniform_quantized_clip_by_value_min_not_const(%input: tensor<3x2x!tf_type.qint32>, %min: tensor<3x2x!tf_type.qint32>) -> tensor<3x2x!tf_type.qint32> { + %scales = "tf.Const"() { value = dense<2.0> : tensor<2xf32> } : () -> tensor<2xf32> + %zps = "tf.Const"() { value = dense<4> : tensor<2xi32> } : () -> tensor<2xi32> + // tensor_proto that points to dense<127> of type !tf_type.qint32. + %max = "tf.Const"() { value = #tf_type : tensor<3x2x!tf_type.qint32> } : () -> tensor<3x2x!tf_type.qint32> + + // CHECK-DAG: %[[INPUT:.*]] = mhlo.bitcast_convert %arg0 : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK-DAG: %[[MIN:.*]] = mhlo.bitcast_convert %arg1 : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK: chlo.broadcast_maximum %[[INPUT]], %[[MIN]] + %res = "tf.UniformQuantizedClipByValue"(%input, %min, %max, %scales, %zps) { + quantization_axis = 1 : i64, + quantization_min_val = -2147483648 : i64, + quantization_max_val = 2147483647 : i64 + } : (tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<2xf32>, tensor<2xi32>) -> tensor<3x2x!tf_type.qint32> + func.return %res : tensor<3x2x!tf_type.qint32> +} + +// ----- + +// CHECK-LABEL: func @uniform_quantized_clip_by_value_max_not_const +func.func @uniform_quantized_clip_by_value_max_not_const(%input: tensor<3x2x!tf_type.qint32>, %max: tensor<3x2x!tf_type.qint32>) -> tensor<3x2x!tf_type.qint32> { + %scales = "tf.Const"() { value = dense<2.0> : tensor<2xf32> } : () -> tensor<2xf32> + %zps = "tf.Const"() { value = dense<4> : tensor<2xi32> } : () -> tensor<2xi32> + // tensor_proto that points to dense<127> of type !tf_type.qint32. + %min = "tf.Const"() { value = #tf_type : tensor<3x2x!tf_type.qint32> } : () -> tensor<3x2x!tf_type.qint32> + + // CHECK-DAG: %[[INPUT:.*]] = mhlo.bitcast_convert %arg0 : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK-DAG: %[[MAX:.*]] = mhlo.bitcast_convert %arg1 : (tensor<3x2xi32>) -> tensor<3x2x!quant.uniform> + // CHECK-DAG: %[[INPUT_1:.*]] = chlo.broadcast_maximum + // CHECK: chlo.broadcast_minimum %[[INPUT_1]], %[[MAX]] + %res = "tf.UniformQuantizedClipByValue"(%input, %min, %max, %scales, %zps) { + quantization_axis = 1 : i64, + quantization_min_val = -2147483648 : i64, + quantization_max_val = 2147483647 : i64 + } : (tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<3x2x!tf_type.qint32>, tensor<2xf32>, tensor<2xi32>) -> tensor<3x2x!tf_type.qint32> + func.return %res : tensor<3x2x!tf_type.qint32> +} + +//===----------------------------------------------------------------------===// +// quant.uniform type handling with control flow ops +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @while_region_with_quant +func.func @while_region_with_quant(%arg0: tensor, %arg1: tensor) -> tensor { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps = "tf.Const"() { value = dense<3> : tensor } : () -> tensor + %one = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + + // CHECK: %[[QUANT0:.*]] = mhlo.uniform_quantize %[[ARG:.*]] : (tensor) -> tensor> + // CHECK: %[[CONVERT_1:.*]] = mhlo.bitcast_convert %[[QUANT0]] : (tensor>) -> tensor + // CHECK: mhlo.while() + // CHECK: cond + // CHECK: %[[CHECK_RES:.*]] = chlo.broadcast_compare + // CHECK: mhlo.return %[[CHECK_RES]] : tensor + // CHECK: %[[CONVERT_2:.*]] = mhlo.bitcast_convert %[[CONVERT_1]] : (tensor) -> tensor> + // CHECK: %[[RET:.*]] = mhlo.uniform_dequantize %[[CONVERT_2]] : (tensor>) -> tensor + // CHECK: return %[[RET]] : tensor + + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor, tensor, tensor) -> tensor + %1 = "tf.WhileRegion"(%0) ({ + ^bb0(%carg0: tensor): + %check = "tf.Equal"(%arg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%check) : (tensor) -> () + }, { + ^bb0(%barg0: tensor): + %id = "tf.Identity"(%barg0) : (tensor) -> tensor + "tf.Yield"(%id) : (tensor) -> () + }) {is_stateless = false} : (tensor) -> tensor + %2 = "tf.UniformDequantize"(%1, %scales, %zps) {quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64} : (tensor, tensor, tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// CHECK-LABEL: func @while_region_with_quant_two_args +func.func @while_region_with_quant_two_args(%arg0: tensor<2x2xf32>, %arg1: tensor) -> (tensor<2x?xf32>, tensor) { + %scales = "tf.Const"() { value = dense<1.0> : tensor } : () -> tensor + %zps2 = "tf.Const"() { value = dense<2> : tensor } : () -> tensor + %zps4 = "tf.Const"() { value = dense<4> : tensor } : () -> tensor + %one = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + + // CHECK: %[[QUANT0:.*]] = mhlo.uniform_quantize %[[ARG:.*]] : (tensor<2x2xf32>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[INT0:.*]] = mhlo.bitcast_convert %[[QUANT0]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xi8> + %0 = "tf.UniformQuantize"(%arg0, %scales, %zps2) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2xf32>, tensor, tensor) -> tensor<2x2x!tf_type.qint8> + // CHECK: %[[QUANT1:.*]] = mhlo.uniform_quantize %[[ARG:.*]] : (tensor<2x2xf32>) -> tensor<2x2x!quant.uniform> + // CHECK: %[[INT1:.*]] = mhlo.bitcast_convert %[[QUANT1]] : (tensor<2x2x!quant.uniform>) -> tensor<2x2xi8> + %1 = "tf.UniformQuantize"(%arg0, %scales, %zps4) { + quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64 + } : (tensor<2x2xf32>, tensor, tensor) -> tensor<2x2x!tf_type.qint8> + + // CHECK: %[[WHILE_RESULT:.*]]:2 = mhlo.while(%[[ARG0:.*]] = %[[INT0]], %[[ARG1:.*]] = %[[INT1]]) + // CHECK-SAME: tensor<2x2xi8>, tensor<2x2xi8> + + // CHECK: cond + + // CHECK: do + // CHECK: mhlo.return %[[ARG0]], %[[ARG1]] : tensor<2x?xi8>, tensor + + %2:2 = "tf.WhileRegion"(%0, %1) ({ + ^bb0(%carg0: tensor<2x2x!tf_type.qint8>, %carg1: tensor<2x2x!tf_type.qint8>): + %check = "tf.Equal"(%arg1, %one) : (tensor, tensor) -> tensor + "tf.Yield"(%check) : (tensor) -> () + }, { + ^bb0(%barg0: tensor<2x?x!tf_type.qint8>, %barg1: tensor): + %id = "tf.Identity"(%barg0) : (tensor<2x?x!tf_type.qint8>) -> tensor<2x?x!tf_type.qint8> + "tf.Yield"(%id, %barg1) : (tensor<2x?x!tf_type.qint8>, tensor) -> () + }) {is_stateless = false} : (tensor<2x2x!tf_type.qint8>, tensor<2x2x!tf_type.qint8>) -> (tensor<2x?x!tf_type.qint8>, tensor) + + // %[[RESULT0:.*]] = mhlo.uniform_dequantize %[[WHILE_RESULT]]#0 + %3 = "tf.UniformDequantize"(%2#0, %scales, %zps2) {quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64} : (tensor<2x?x!tf_type.qint8>, tensor, tensor) -> tensor<2x?xf32> + + // %[[RESULT1:.*]] = mhlo.uniform_dequantize %[[WHILE_RESULT]]#0 + %4 = "tf.UniformDequantize"(%2#1, %scales, %zps4) {quantization_axis = -1 : i64, quantization_min_val = -128 : i64, quantization_max_val = 127 : i64} : (tensor, tensor, tensor) -> tensor + + // return %[[RESULT0]], %[[RESULT1]] + func.return %3, %4 : tensor<2x?xf32>, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-with-tf2xla-hlo-importer.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-with-tf2xla-hlo-importer.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ddd38150efc4d5b1c56cc7185a7d4c61afb3d37a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf-with-tf2xla-hlo-importer.mlir @@ -0,0 +1,710 @@ +// RUN: tf-opt "-xla-legalize-tf=device-type=XLA_CPU_JIT prefer-tf2xla=true use-tf2xla-fallback=true" %s -verify-diagnostics -mlir-disable-threading | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + // CHECK-LABEL: binary_op + func.func @binary_op(%arg0: tensor<2xf32>, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.atan2 + %0 = "tf.Atan2"(%arg0, %arg1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: multiple_return_values + func.func @multiple_return_values(%arg0: tensor<3xi64>) -> tensor { + %0:3 = "tf.Unpack"(%arg0) {axis = 0 : i64} : (tensor<3xi64>) -> (tensor, tensor, tensor) + // CHECK: return %1 : tensor + func.return %0#0 : tensor + } + + // CHECK-LABEL: constant_parameter + func.func @constant_parameter(%arg0: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.Const"() {value = dense<1.42> : tensor<2xf32>} : () -> tensor<2xf32> + // CHECK: mhlo.atan2 %arg0, %2 + %1 = "tf.Atan2"(%arg0, %0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: uses_translated_return_type + func.func @uses_translated_return_type(%arg0: tensor<3xf32>) -> tensor { + // CHECK: tensor.cast %{{[0-9]+}} : tensor> to tensor + %y, %idx = "tf.Unique"(%arg0) {device = ""} : (tensor<3xf32>) -> (tensor, tensor<3xi32>) + return %y : tensor + } + + // CHECK-LABEL: @abs + func.func @abs(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NOT: tf.Abs + %0 = "tf.Abs"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: func @testBroadcastGradientArgs + func.func @testBroadcastGradientArgs(%s0: tensor<4xi32>, %s1: tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) { + // CHECK: tf.BroadcastGradientArgs + %r0, %r1 = "tf.BroadcastGradientArgs"(%s0, %s1) : (tensor<4xi32>, tensor<4xi32>) -> (tensor<1xi32>, tensor<0xi32>) + func.return %r0, %r1 : tensor<1xi32>, tensor<0xi32> + } + + // CHECK-LABEL: @acos + func.func @acos(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NOT: tf.Acos + %0 = "tf.Acos"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + + // CHECK-LABEL: unknown_op + func.func @unknown_op(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: tf.CustomTestOp + %0 = "tf.CustomTestOp"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: add_v2 + func.func @add_v2(%arg0: tensor<2xi32>) -> tensor<2xi32> { + // CHECK: mhlo.add %arg0, %arg0 : tensor<2xi32> + %0 = "tf.AddV2"(%arg0, %arg0) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> + } + + // CHECK-LABEL: not_allowlisted_op + func.func @not_allowlisted_op(%arg0: tensor<3xi32>, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: tf.TensorListReserve + %0 = "tf.TensorListReserve"(%arg0, %arg1) : (tensor<3xi32>, tensor) -> tensor>> + // CHECK: tf.TensorListGetItem + %1 = "tf.TensorListGetItem"(%0, %arg2, %arg0) : (tensor>>, tensor, tensor<3xi32>) -> tensor + func.return %1 : tensor + } + + // CHECK-LABEL: unranked_operand + func.func @unranked_operand(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: tf.Atan2 + // expected-remark@+1 {{lowering requires bounded tensor operands}} + %0 = "tf.Atan2"(%arg0, %arg0) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + + func.return %0 : tensor<*xf32> + } + + // CHECK-LABEL: dynamic_operand + func.func @dynamic_operand(%arg0: tensor) -> tensor { + // CHECK: tf.Atan2 + // expected-remark@+1 {{lowering requires bounded tensor operands}} + %0 = "tf.Atan2"(%arg0, %arg0) : (tensor, tensor) -> tensor + + func.return %0 : tensor + } + + // CHECK-LABEL: tuple_type + func.func @tuple_type(%arg0: tuple, tensor>) -> tensor { + // Verifies that the pass can handle operands of non-tensor type like tuple + // from non TensorFlow ops. + %0 = "mhlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor>) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: unsupported_dtype + func.func @unsupported_dtype(%arg0: tensor<2x!tf_type.variant>) -> tensor<2x!tf_type.variant> { + // CHECK: tf.AddN + // expected-remark@+1 {{skipping legalization due to unsupported type 'tensor<2x!tf_type.variant>'}} + %0 = "tf.AddN"(%arg0, %arg0) : (tensor<2x!tf_type.variant>, tensor<2x!tf_type.variant>) -> tensor<2x!tf_type.variant> + + func.return %0 : tensor<2x!tf_type.variant> + } + + // CHECK-LABEL: multiple_dialect_ops + func.func @multiple_dialect_ops(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.negate + %0 = "mhlo.negate"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + // CHECK: mhlo.atan2 + %1 = "tf.Atan2"(%arg0, %0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + + func.return %1 : tensor<2xf32> + } + + // CHECK-LABEL: binary_op_broadcast + func.func @binary_op_broadcast(%arg0: tensor<4x1xf32>, %arg1: tensor<4x1x4xf32>) -> tensor<4x4x4xf32> { + // CHECK: %[[BROADCAST0:.*]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<[1, 2]> : tensor<2xi64>}> : (tensor<4x1xf32>) -> tensor<4x4x1xf32> + // CHECK: %[[RESHAPE0:.*]] = mhlo.reshape %[[BROADCAST0]] : (tensor<4x4x1xf32>) -> tensor<4x4xf32> + // CHECK: %[[UPDATED_ARG0:.*]] = "mhlo.broadcast_in_dim"(%[[RESHAPE0]]) <{broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>}> : (tensor<4x4xf32>) -> tensor<4x4x4xf32> + + // CHECK: %[[RESHAPE1:.*]] = mhlo.reshape %arg1 : (tensor<4x1x4xf32>) -> tensor<4x4xf32> + // CHECK: %[[UPDATED_ARG1:.*]] = "mhlo.broadcast_in_dim"(%[[RESHAPE1]]) <{broadcast_dimensions = dense<[0, 2]> : tensor<2xi64>}> : (tensor<4x4xf32>) -> tensor<4x4x4xf32> + + // CHECK: %[[RESULT:.*]] = mhlo.atan2 %[[UPDATED_ARG0]], %[[UPDATED_ARG1]] : tensor<4x4x4xf32> + // CHECK: return %[[RESULT]] : tensor<4x4x4xf32> + + %0 = "tf.Atan2"(%arg0, %arg1) : (tensor<4x1xf32>, tensor<4x1x4xf32>) -> tensor<4x4x4xf32> + func.return %0: tensor<4x4x4xf32> + } + + // CHECK-LABEL: func @ternary_op + func.func @ternary_op(%arg0: tensor<2xi1>, %arg1: tensor<2xi32>, %arg2: tensor<2xi32>) -> tensor<2xi32> { + // CHECK: mhlo.select %arg0, %arg1, %arg2 + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<2xi1>, tensor<2xi32>, tensor<2xi32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> + } + + // CHECK-LABEL: func @convert + func.func @convert(%arg0: tensor<2xi32>) -> tensor<2xf32> { + // CHECK: mhlo.convert %arg0 : (tensor<2xi32>) -> tensor<2xf32> + %0 = "tf.Cast"(%arg0) {Truncate = false} : (tensor<2xi32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: func @constant + func.func @constant(%arg0: tensor) -> tensor { + // CHECK: %[[ONE:.*]] = mhlo.constant dense<1.000000e+00> : tensor + // CHECK: %[[RESULT:.*]] = mhlo.divide %[[ONE]], %arg0 : tensor + // CHECK: return %[[RESULT]] + + %0 = "tf.Inv"(%arg0) : (tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: func @const_inputs + // CHECK-SAME: (%[[ARG0:.*]]: tensor<2x2xf64>, %[[ARG1:.*]]: tensor, + func.func @const_inputs(%arg0: tensor<2x2xf64>, %arg1: tensor, %arg2: tensor<2xi32>, %arg3: tensor<2xi32>, %arg4: tensor<2xi32>) -> tensor<6x5xf64> { + + // CHECK: "mhlo.pad"(%[[ARG0]], %[[ARG1]]) + // CHECK-SAME-DAG: edge_padding_high = dense<[1, 2]> : tensor<2xi64> + // CHECK-SAME-DAG: edge_padding_low = dense<[2, 1]> : tensor<2xi64> + // CHECK-SAME-DAG: interior_padding = dense<[1, 0]> : tensor<2xi64> + + %0 = mhlo.constant dense<[2, 1]> : tensor<2xi32> + %1 = mhlo.constant dense<[1, 2]> : tensor<2xi32> + %2 = mhlo.constant dense<[1, 0]> : tensor<2xi32> + %3 = "tf.XlaPad"(%arg0, %arg1, %0, %1, %2) : (tensor<2x2xf64>, tensor, tensor<2xi32>, tensor<2xi32>, tensor<2xi32>) -> tensor<6x5xf64> + func.return %3 : tensor<6x5xf64> + } + + func.func @non_const_inputs(%arg0: tensor<2x2xf64>, %arg1: tensor, %arg2: tensor<2xi32>, %arg3: tensor<2xi32>, %arg4: tensor<2xi32>) -> tensor<6x5xf64> { + // expected-remark@+1 {{compilation to HLO failed: INVALID_ARGUMENT: Input 2 to node `tf.XlaPad` with op XlaPad must be a compile-time constant.}} + %0 = "tf.XlaPad"(%arg0, %arg1, %arg2, %arg3, %arg4) : (tensor<2x2xf64>, tensor, tensor<2xi32>, tensor<2xi32>, tensor<2xi32>) -> tensor<6x5xf64> + func.return %0 : tensor<6x5xf64> + } + + // CHECK-LABEL: dynamic_result_type + func.func @dynamic_result_type(%arg0: tensor<2xf32>) -> tensor<*xf32> { + // CHECK: %[[RESULT:.*]] = mhlo.atan2 %arg0, %arg0 : tensor<2xf32> + // CHECK: tensor.cast %[[RESULT]] : tensor<2xf32> to tensor<*xf32> + %0 = "tf.Atan2"(%arg0, %arg0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<*xf32> + + // return %[[RESULT]] + func.return %0 : tensor<*xf32> + } + + func.func @truncated_normal() -> tensor<2x2xf32> { + // CHECK-NOT: tf.TruncatedNormal + %0 = mhlo.constant dense<[2, 2]> : tensor<2xi32> + %1 = "tf.TruncatedNormal"(%0) {T = i32, device = "", dtype = f32, seed = 0 : i64, seed2 = 1950157571 : i64} : (tensor<2xi32>) -> tensor<2x2xf32> + func.return %1 : tensor<2x2xf32> + } + + // CHECK-LABEL: dynamic_update_slice + // CHECK-SAME: (%[[ARG0:.*]]: tensor<3x4xi32>, %[[ARG1:.*]]: tensor<2x2xi32>, %[[ARG2:.*]]: tensor<2xi32> + func.func @dynamic_update_slice(%arg0: tensor<3x4xi32>, %arg1: tensor<2x2xi32>, %arg2: tensor<2xi32>) -> tensor<3x4xi32> { + + // CHECK: %[[SLICE0:.*]] = "mhlo.slice"(%[[ARG2]]) + // CHECK-DAG-SAME: start_indices = dense<0> : tensor<1xi64> + // CHECK-DAG-SAME: limit_indices = dense<1> : tensor<1xi64> + // CHECK-DAG-SAME: strides = dense<1> : tensor<1xi64> + // CHECK-SAME: (tensor<2xi32>) -> tensor<1xi32> + // CHECK: %[[DIM0:.*]] = mhlo.reshape %[[SLICE0]] : (tensor<1xi32>) -> tensor + + // CHECK: %[[SLICE1:.*]] = "mhlo.slice"(%[[ARG2]]) + // CHECK-DAG-SAME: start_indices = dense<1> : tensor<1xi64> + // CHECK-DAG-SAME: limit_indices = dense<2> : tensor<1xi64> + // CHECK-DAG-SAME: strides = dense<1> : tensor<1xi64> + // CHECK-SAME: (tensor<2xi32>) -> tensor<1xi32> + // CHECK: %[[DIM1:.*]] = mhlo.reshape %[[SLICE1]] : (tensor<1xi32>) -> tensor + + // CHECK: mhlo.dynamic_update_slice %[[ARG0]], %[[ARG1]], %[[DIM0]], %[[DIM1]] + + %0 = "tf.XlaDynamicUpdateSlice"(%arg0, %arg1, %arg2) : (tensor<3x4xi32>, tensor<2x2xi32>, tensor<2xi32>) -> tensor<3x4xi32> + func.return %0: tensor<3x4xi32> + } + + // CHECK-LABEL: @sparse_to_dense + // CHECK-SAME: (%[[ARG0:.*]]: tensor<3x2xi32>, %[[ARG1:.*]]: tensor<3xf32>, %[[ARG2:.*]]: tensor) + func.func @sparse_to_dense(%arg0: tensor<3x2xi32>, %arg1: tensor<3xf32>, %arg2: tensor) -> tensor<3x3xf32> { + + // CHECK: %[[DEFAULT:.*]] = "mhlo.broadcast_in_dim"(%[[ARG2]]) <{broadcast_dimensions = dense<> : tensor<0xi64>}> : (tensor) -> tensor<3x3xf32> + + // CHECK: %[[RESULT:.*]] = "mhlo.scatter"(%[[DEFAULT]], %[[ARG0]], %[[ARG1]]) + // CHECK-SAME: indices_are_sorted = false + // CHECK-SAME: scatter_dimension_numbers + // CHECK-SAME: inserted_window_dims = [0, 1] + // CHECK-SAME: scatter_dims_to_operand_dims = [0, 1] + // CHECK-SAME: index_vector_dim = 1 + // CHECK-SAME: unique_indices = false + // CHECK-NEXT: ^bb0(%[[ARG3:.*]]: tensor, %[[ARG4:.*]]: tensor): + // CHECK: mhlo.return %[[ARG4]] : tensor + // CHECK: }) + // CHECK-SAME: (tensor<3x3xf32>, tensor<3x2xi32>, tensor<3xf32>) -> tensor<3x3xf32> + + // return %[[RESULT]] : tensor<3x3xf32> + + %cst = mhlo.constant dense<3> : tensor<2xi32> + %0 = "tf.SparseToDense"(%arg0, %cst, %arg1, %arg2) {validate_indices = true}: (tensor<3x2xi32>, tensor<2xi32>, tensor<3xf32>, tensor) -> tensor<3x3xf32> + func.return %0 : tensor<3x3xf32> + } + + // CHECK-LABEL: reverse_sequence + func.func @reverse_sequence(%arg0: tensor<4x2x3x1x1xi32>, %arg1: tensor<3xi32>) -> tensor<4x2x3x1x1xi32> { + // CHECK-NOT: tf.ReverseSequence + %0 = "tf.ReverseSequence"(%arg0, %arg1) {batch_dim = 2 : i64, seq_dim = 0 : i64}: (tensor<4x2x3x1x1xi32>, tensor<3xi32>) -> tensor<4x2x3x1x1xi32> + func.return %0 : tensor<4x2x3x1x1xi32> + } + + // CHECK-LABEL: mirror_pad + func.func @mirror_pad(%arg0: tensor<2x3xcomplex>) -> tensor<4x7xcomplex> { + %0 = mhlo.constant dense<[[1, 1], [2, 2]]> : tensor<2x2xi32> + // CHECK-NOT: tf.MirrorPad + %1 = "tf.MirrorPad"(%arg0, %0) {mode = "SYMMETRIC"} : (tensor<2x3xcomplex>, tensor<2x2xi32>) -> tensor<4x7xcomplex> + func.return %1 : tensor<4x7xcomplex> + } + + // CHECK-LABEL: bucketize + func.func @bucketize(%arg0: tensor<2x5xf32>) -> tensor<2x5xi32> { + // CHECK-NOT: tf.Bucketize + %0 = "tf.Bucketize"(%arg0) {boundaries = [0.000000e+00 : f32, 3.000000e+00 : f32, 8.000000e+00 : f32, 1.100000e+01 : f32]} : (tensor<2x5xf32>) -> tensor<2x5xi32> + func.return %0 : tensor<2x5xi32> + } + + // CHECK-LABEL: arg_min + func.func @arg_min(%arg0: tensor<6xf64>) -> tensor { + // CHECK-NOT: ArgMin + %0 = mhlo.constant dense<0> : tensor + %1 = "tf.ArgMin"(%arg0, %0) : (tensor<6xf64>, tensor) -> tensor + func.return %1 : tensor + } + + // CHECK-LABEL: non_max_suppression_v4 + func.func @non_max_suppression_v4(%arg0: tensor<3x4xf32>, %arg1: tensor<3xf32>, %arg2: tensor, %arg3: tensor) -> tensor<2xi32> { + %max_size = mhlo.constant dense<2> : tensor + // CHECK-NOT: tf.NonMaxSuppressionV4 + %0:2 = "tf.NonMaxSuppressionV4"(%arg0, %arg1, %max_size, %arg2, %arg3) {pad_to_max_output_size = true}: (tensor<3x4xf32>, tensor<3xf32>, tensor, tensor, tensor) -> (tensor<2xi32>, tensor) + func.return %0#0 : tensor<2xi32> + } + + // CHECK-LABEL: bessel_i0e + func.func @bessel_i0e(%arg0: tensor<3xf16>, %arg1: tensor<3xf32>, %arg2: tensor<3xf64>) -> (tensor<3xf16>, tensor<3xf32>, tensor<3xf64>) { + // CHECK-NOT: tf.BesselI0e + %0 = "tf.BesselI0e"(%arg0) : (tensor<3xf16>) -> (tensor<3xf16>) + %1 = "tf.BesselI0e"(%arg1) : (tensor<3xf32>) -> (tensor<3xf32>) + %2 = "tf.BesselI0e"(%arg2) : (tensor<3xf64>) -> (tensor<3xf64>) + func.return %0, %1, %2 : tensor<3xf16>, tensor<3xf32>, tensor<3xf64> + } + + // CHECK-LABEL: bessel_i1e + func.func @bessel_i1e(%arg0: tensor<3xf16>, %arg1: tensor<3xf32>, %arg2: tensor<3xf64>) -> (tensor<3xf16>, tensor<3xf32>, tensor<3xf64>) { + // CHECK-NOT: tf.BesselI1e + %0 = "tf.BesselI1e"(%arg0) : (tensor<3xf16>) -> (tensor<3xf16>) + %1 = "tf.BesselI1e"(%arg1) : (tensor<3xf32>) -> (tensor<3xf32>) + %2 = "tf.BesselI1e"(%arg2) : (tensor<3xf64>) -> (tensor<3xf64>) + func.return %0, %1, %2 : tensor<3xf16>, tensor<3xf32>, tensor<3xf64> + } + + // CHECK-LABEL: diag + func.func @diag(%arg0: tensor<2xf32>) -> tensor<2x2xf32> { + // CHECK-NOT: tf.Diag + %0 = "tf.Diag"(%arg0) : (tensor<2xf32>) -> tensor<2x2xf32> + func.return %0 : tensor<2x2xf32> + } + + // CHECK-LABEL: random_uniform_int + func.func @random_uniform_int(%arg0: tensor, %arg1: tensor) -> tensor<1000xi32> { + %0 = "tf.Const"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + // CHECK-NOT: tf.RandomUniformInt + %1 = "tf.RandomUniformInt"(%0, %arg0, %arg1) {seed = 0 : i64, seed2 = 0 : i64} : (tensor<1xi32>, tensor, tensor) -> tensor<1000xi32> + func.return %1 : tensor<1000xi32> + } + + // CHECK-LABEL: multinomial + func.func @multinomial(%arg0: tensor<2x4xf32>, %seed: tensor, %seed2: tensor) -> tensor<2x10xi32> { + // CHECK-NOT: tf.Multinomial + %samples = "tf.Const"() { value = dense<10> : tensor } : () -> tensor + %1 = "tf.Multinomial"(%arg0, %samples) {seed = 0, seed2 = 0}: (tensor<2x4xf32>, tensor) -> tensor<2x10xi32> + func.return %1 : tensor<2x10xi32> + } + + // CHECK-LABEL: @set_dynamic_dimension_size + func.func @set_dynamic_dimension_size(%input: tensor<4xf32>, %size: tensor) -> tensor { + %dimension = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // CHECK: mhlo.set_dimension_size + // CHECK-SAME: <{dimension = 0 : i64}> : (tensor<4xf32>, tensor) -> tensor> + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4xf32>, tensor, tensor) -> tensor + func.return %0 : tensor + } + + // CHECK-LABEL: unique + func.func @unique(%arg0: tensor<5xf32>) -> (tensor, tensor) { + // CHECK-NOT: tf.Unique + %0, %1 = "tf.Unique"(%arg0) : (tensor<5xf32>) -> (tensor, tensor) + func.return %0, %1 : tensor , tensor + } + + // CHECK-LABEL: @erfinv + func.func @erfinv(%input: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NOT: tf.Erfinv + %0 = "tf.Erfinv"(%input) : (tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> + } + + // CHECK-LABEL: @ndtri + func.func @ndtri(%input: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NOT: tf.Ndtri + %0 = "tf.Ndtri"(%input) : (tensor<4xf32>) -> tensor<4xf32> + func.return %0 : tensor<4xf32> + } + + // CHECK-LABEL: @fake_param + func.func @fake_param() -> tensor<4xf32> { + // CHECK-NOT: tf.FakeParam + %0 = "tf.FakeParam"() {shape = #tf_type.shape<4>} : () -> tensor<4xf32> + func.return %0 : tensor<4xf32> + } + + // CHECK-LABEL: @parameterized_truncated_normal + func.func @parameterized_truncated_normal(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> tensor<10000000xf32> { + %0 = "tf.Const"() {value = dense<10000000> : tensor<1xi32>} : () -> tensor<1xi32> + // CHECK-NOT: tf.ParameterizedTruncatedNormal + %1 = "tf.ParameterizedTruncatedNormal"(%0, %arg0, %arg1, %arg2, %arg3) {seed = 0 : i64, seed2 = 0 : i64} : (tensor<1xi32>, tensor, tensor, tensor, tensor) -> tensor<10000000xf32> + func.return %1 : tensor<10000000xf32> + } + + // Check XlaSpmdFullToShardShape's conversion from split sharding to manual + // sharding. + // The split sharding is: + // type: OTHER + // tile_assignment_dimensions: 2 + // tile_assignment_dimensions: 1 + // tile_assignment_devices: 0 + // tile_assignment_devices: 1 + // Serialized string: + // "\08\03\1A\02\02\01\22\02\00\01" + // The manual sharding is: + // type: MANUAL + // Serialized string: + // "\08\04" + + // CHECK-LABEL: @xla_spmd_full_to_shard_shape + func.func @xla_spmd_full_to_shard_shape(%arg0: tensor<2x2xi64>) -> (tensor<1x2xi64>) { + // CHECK: %[[SHARDING:.*]] = mhlo.custom_call @Sharding(%arg0) {backend_config = "", mhlo.sharding = "{devices=[2,1]0,1}"} : (tensor<2x2xi64>) -> tensor<2x2xi64> + // CHECK: %[[MANUAL:.*]] = mhlo.custom_call @SPMDFullToShardShape(%[[SHARDING]]) {backend_config = "", mhlo.sharding = "{manual}"} : (tensor<2x2xi64>) -> tensor<1x2xi64> + // CHECK: return %[[MANUAL]] + %0 = "tf.XlaSpmdFullToShardShape"(%arg0) {dim = -1 : i64, manual_sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<2x2xi64>) -> tensor<1x2xi64> + func.return %0 : tensor<1x2xi64> + } + + // Check XlaSpmdShardToFullShape's conversion from manual sharding to split + // sharding. + // The manual sharding is: + // type: MANUAL + // Serialized string: + // "\08\04" + // The split sharding is: + // type: OTHER + // tile_assignment_dimensions: 2 + // tile_assignment_dimensions: 1 + // tile_assignment_devices: 0 + // tile_assignment_devices: 1 + // Serialized string: + // "\08\03\1A\02\02\01\22\02\00\01" + + // CHECK-LABEL: @xla_spmd_shard_to_full_shape + func.func @xla_spmd_shard_to_full_shape(%arg0: tensor<1x2xi64>) -> (tensor<2x2xi64>) { + // CHECK: %[[SHARDING:.*]] = mhlo.custom_call @Sharding(%arg0) {backend_config = "", mhlo.sharding = "{manual}"} : (tensor<1x2xi64>) -> tensor<1x2xi64> + // CHECK: %[[FULL:.*]] = mhlo.custom_call @SPMDShardToFullShape(%[[SHARDING]]) {backend_config = "", mhlo.sharding = "{devices=[2,1]0,1}"} : (tensor<1x2xi64>) -> tensor<2x2xi64> + // CHECK: return %[[FULL]] + %0 = "tf.XlaSpmdShardToFullShape"(%arg0) {dim = -1 : i64, full_shape = #tf_type.shape<2x2>, manual_sharding = "\08\03\1A\02\02\01\22\02\00\01", unspecified_dims = []} : (tensor<1x2xi64>) -> tensor<2x2xi64> + func.return %0 : tensor<2x2xi64> + } + + // CHECK-LABEL: @xla_svd + func.func @xla_svd(%arg0: tensor<1x1xf32>) -> (tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32>) { + // CHECK-NOT: XlaSvd + %s, %u, %v = "tf.XlaSvd"(%arg0) {max_iter = 1, epsilon = 1.0E-09 : f32, precision_config = ""} : (tensor<1x1xf32>) -> (tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32>) + func.return %s, %u, %v : tensor<1xf32>, tensor<1x1xf32>, tensor<1x1xf32> + } + + func.func @identity(%arg0: f32) -> f32 { + func.return %arg0 : f32 + } + + // This test verifies that legalization for ops with symbol reference attribute + // is not attempted even if they are in allow-list. XLA op kernels for these + // ops compile the function to HLO on-demand which won't work in our case as it + // may contain unsupported ops in the fallback nor we provide XlaCompiler to + // the kernel. Using a allowed op Atan2 to protect against future addition of a + // new op with a symbol ref. + + // CHECK-LABEL: @atan2_with_symbol_ref + func.func @atan2_with_symbol_ref(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: tf.Atan2 + // expected-remark@+1 {{ops with symbol references are not supported}} + %0 = "tf.Atan2"(%arg0, %arg0) {_body = @identity} : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + + func.return %0 : tensor<2xf32> + } + + func.func private @branch0(tensor<2xf32>) -> tensor<2xf32> + func.func private @branch1(tensor<2xf32>) -> tensor<2xf32> + + func.func @case_with_symbol_ref(%arg0: tensor, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: tf.Case + // expected-remark@+1 {{ops with symbol references are not supported}} + %0 = "tf.Case"(%arg0, %arg1) {branches = [@branch0, @branch1], is_stateless = false} : (tensor, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + + // CHECK-LABEL: const + func.func @const() -> tensor<2xf32> { + // CHECK: mhlo.const + %cst = "tf.Const"() {value = dense<2.0> : tensor<2xf32>} : () -> tensor<2xf32> + func.return %cst : tensor<2xf32> + } + + // CHECK-LABEL: @bounds_propagation + func.func @bounds_propagation(%input: tensor<4xf32>, %size: tensor) -> tensor { + %dimension = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // CHECK: %[[BOUNDED:.*]] = "mhlo.set_dimension_size" + // CHECK-SAME: <{dimension = 0 : i64}> : (tensor<4xf32>, tensor) -> tensor> + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4xf32>, tensor, tensor) -> tensor + + %axis = "tf.Const"() { value = dense<0> : tensor<1xi32> } : () -> tensor<1xi32> + // CHECK: %[[REVERSED:.*]] = "mhlo.reverse"(%[[BOUNDED]]) + // CHECK-SAME: {dimensions = dense<0> : tensor<1xi64>} + // CHECK-SAME: (tensor>) -> tensor> + %1 = "tf.ReverseV2"(%0, %axis) : (tensor, tensor<1xi32>) -> tensor + + // CHECK: %[[RESULT:.*]] = tensor.cast %[[REVERSED]] : tensor> to tensor + // CHECK: return %[[RESULT]] : tensor + func.return %1 : tensor + } + + // CHECK-LABEL: @bounds_propagation_skip_symbol_ref_ops + func.func @bounds_propagation_skip_symbol_ref_ops(%input: tensor<4xf32>, %size: tensor) -> tensor { + %dimension = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // CHECK: %[[BOUNDED:.*]] = "mhlo.set_dimension_size" + // CHECK-SAME: <{dimension = 0 : i64}> : (tensor<4xf32>, tensor) -> tensor> + %0 = "tf.XlaSetDynamicDimensionSize"(%input, %dimension, %size) : (tensor<4xf32>, tensor, tensor) -> tensor + + // CHECK: %[[ORIGINAL:.*]] = tensor.cast %[[BOUNDED]] : tensor> to tensor + + %axis = "tf.Const"() { value = dense<0> : tensor<1xi32> } : () -> tensor<1xi32> + // CHECK: tf.ReverseV2 + // CHECK-SAME: (tensor, tensor<1xi32>) -> tensor + // expected-remark@+1 {{lowering requires bounded tensor operands}} + %1 = "tf.ReverseV2"(%0, %axis) {_body = @identity} : (tensor, tensor<1xi32>) -> tensor + + func.return %1 : tensor + } + + // CHECK-LABEL: func @set_bound + func.func @set_bound(%arg0: tensor) -> tensor { + %bound = "tf.Const"() {value = dense<16> : tensor} : () -> tensor + + // CHECK: %[[RESULT:.*]] = mhlo.custom_call @SetBound(%arg0) {backend_config = "", mhlo.literal = dense<16> : tensor} + %bounded = "tf.XlaSetBound"(%arg0, %bound) : (tensor, tensor) -> tensor + + // CHECK: return %[[RESULT]] + func.return %bounded : tensor + } + + // CHECK-LABEL: func @greater + func.func @greater(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2xi1> { + // CHECK-NEXT: mhlo.compare GT, %arg0, %arg1 + %0 = "tf.Greater"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2xi1> + func.return %0: tensor<2xi1> + } + + // CHECK-LABEL: batchmatmulv2 + func.func @batchmatmulv2(%arg0: tensor<1x4x2xf32>, %arg1: tensor<3x2x4xf32>) -> tensor<3x4x4xf32> { + // CHECK: mhlo.reduce + // CHECK: mhlo.dot_general + // CHECK: mhlo.transpose + %0 = "tf.BatchMatMulV2"(%arg0, %arg1) {T = f32, adj_x = false, adj_y = false, grad_x = false, grad_y = false, device = ""} : (tensor<1x4x2xf32>, tensor<3x2x4xf32>) -> tensor<3x4x4xf32> + func.return %0 : tensor<3x4x4xf32> + } + + // CHECK-LABEL: approx_topk + func.func @approx_topk(%arg0: tensor>> {tf._user_specified_name = "db"}) -> (tensor<10x10xbf16>) { + %0 = "tf.ReadVariableOp"(%arg0) {device = ""} : (tensor>>) -> tensor<10x500xbf16> + // CHECK: mhlo.compare GT + %values, %indices = "tf.ApproxTopK"(%0) {aggregate_to_topk = true, device = "", is_max_k = true, k = 10 : i64, recall_target = 0.949999988 : f32, reduction_dimension = -1 : i64, reduction_input_size_override = -1 : i64} : (tensor<10x500xbf16>) -> (tensor<10x10xbf16>, tensor<10x10xi32>) + return %values : tensor<10x10xbf16> + } + + // CHECK-LABEL: fusedBatchNormV3_noTraining + func.func @fusedBatchNormV3_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: "mhlo.batch_norm_inference"({{.*}}, %arg1, %arg2, %arg3, %arg4) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> + } + + // CHECK-LABEL: fusedBatchNormV3_training + func.func @fusedBatchNormV3_training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: %[[OUT:.*]], %[[MEAN:.*]], %[[VAR:.*]] = "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> + } + + // CHECK-LABEL: fusedBatchNormGradV3_noTraining + func.func @fusedBatchNormGradV3_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK: %[[scr1:.*]] = mhlo.rsqrt + // CHECK: %[[bcast_arg3:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg3, {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK: %[[sub:.*]] = mhlo.subtract %[[act]], %[[bcast_arg3]] : tensor<8x8x8x8xf32> + // CHECK: %[[mul:.*]] = mhlo.multiply %[[grad]], %[[sub]] : tensor<8x8x8x8xf32> + // CHECK: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cmul:.*]] = mhlo.convert %[[mul]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-NEXT: %[[convert_init:.*]] = mhlo.convert %[[init]] : tensor + // CHECK: %[[red1:.*]] = mhlo.reduce(%[[cmul]] init: %[[convert_init]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK: %[[scr2:.*]] = mhlo.convert %[[red1]] : tensor<8xf32> + + // CHECK: %[[mul2:.*]] = mhlo.multiply %arg2, %[[scr1]] : tensor<8xf32> + // CHECK: %[[bcast_mul2:.+]] = "mhlo.dynamic_broadcast_in_dim"(%[[mul2]], {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK: %[[mul3:.*]] = mhlo.multiply %[[grad]], %[[bcast_mul2]] : tensor<8x8x8x8xf32> + + // CHECK: %[[scale_backprop:.*]] = mhlo.multiply %[[scr1]], %[[scr2]] : tensor<8xf32> + + // CHECK: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK: %[[cgrad:.*]] = mhlo.convert %[[grad]] : tensor<8x8x8x8xf32> + // CHECK: %[[init2:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-NEXT: %[[convert_init2:.*]] = mhlo.convert %[[init2]] : tensor + // CHECK: %[[red2:.*]] = mhlo.reduce(%[[cgrad]] init: %[[convert_init2]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK: %[[offset_backprop:.*]] = mhlo.convert %[[red2]] : tensor<8xf32> + + // CHECK: %[[x_backprop:.*]] = mhlo.convert %[[mul3]] : tensor<8x8x8x8xf32> + // CHECK: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> + } + + // CHECK-LABEL: fusedBatchNormGradV3_Training + func.func @fusedBatchNormGradV3_Training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<0xf32>, tensor<*xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : tensor<8x8x8x8xf32> + // CHECK: return %[[x_backprop]] + // CHECK-SAME: tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<0xf32>, tensor<*xf32>) + func.return %0#0, %0#3, %0#4 : tensor<8x8x8x8xf32>, tensor<0xf32>, tensor<*xf32> + } + + // CHECK-LABEL: @max_pool_grad_valid + // CHECK-SAME: %[[INPUT:.*]]: tensor<10x24x24x64xf32>, %arg1: tensor<10x12x12x64xf32>, %[[GRAD:.*]]: tensor<10x12x12x64xf32> + func.func @max_pool_grad_valid(%orig_input: tensor<10x24x24x64xf32>, %orig_output: tensor<10x12x12x64xf32>, %grad: tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> { + // CHECK: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: %[[RESULT:.*]] = "mhlo.select_and_scatter"(%[[INPUT]], %[[GRAD]], %[[ZERO]]) <{ + // CHECK-SAME: padding = dense<0> : tensor<4x2xi64>, window_dimensions = dense<[1, 2, 2, 1]> : tensor<4xi64>, window_strides = dense<[1, 2, 2, 1]> : tensor<4xi64> + // CHECK-SAME }> ({ + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.compare GE, %[[VALUE_A]], %[[VALUE_B]] : (tensor, tensor) -> tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }, { + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.add %[[VALUE_A]], %[[VALUE_B]] : tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }) : (tensor<10x24x24x64xf32>, tensor<10x12x12x64xf32>, tensor) -> tensor<10x24x24x64xf32> + // CHECK: return %[[RESULT]] : tensor<10x24x24x64xf32> + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + explicit_paddings = [], + ksize = [1, 2, 2, 1], + padding = "VALID", + strides = [1, 2, 2, 1] + } : (tensor<10x24x24x64xf32>, tensor<10x12x12x64xf32>, tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> + func.return %result : tensor<10x24x24x64xf32> + } + + // CHECK-LABEL: @max_pool_grad_same + func.func @max_pool_grad_same(%orig_input: tensor<2x13x25x7xf32>, %orig_output: tensor<2x4x7x7xf32>, %grad: tensor<2x4x7x7xf32>) -> tensor<2x13x25x7xf32> { + // CHECK: padding = dense<{{\[\[}}0, 0], [0, 1], [1, 1], [0, 0]]> : tensor<4x2xi64> + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + explicit_paddings = [], + ksize = [1, 2, 3, 1], + padding = "SAME", + strides = [1, 4, 4, 1] + } : (tensor<2x13x25x7xf32>, tensor<2x4x7x7xf32>, tensor<2x4x7x7xf32>) -> tensor<2x13x25x7xf32> + func.return %result : tensor<2x13x25x7xf32> + } + + //===--------------------------------------------------------------------===// + // tf.XlaReduceScatter legalization + //===--------------------------------------------------------------------===// + // CHECK-LABEL: func @xla_reduce_scatter + func.func @xla_reduce_scatter(%arg0: tensor<128x128xf32>) -> tensor<64x128xf32> { + %cst = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {value = dense<[[0, 4], [1, 5], [2, 6], [3, 7]]> : tensor<4x2xi32>} : () -> tensor<4x2xi32> + // CHECK: "mhlo.reduce_scatter"(%arg0) + // CHECK{LITERAL}: replica_groups = dense<[[0, 4], [1, 5], [2, 6], [3, 7]]> + // CHECK-SAME: scatter_dimension = 0 + // + %1 = "tf.XlaReduceScatter"(%arg0, %cst_0, %cst) {reduce_op = "Add"} : (tensor<128x128xf32>, tensor<4x2xi32>, tensor) -> tensor<64x128xf32> + func.return %1 : tensor<64x128xf32> + } + + // CHECK-LABEL: func @tf_mod + func.func @tf_mod(%arg1: tensor<2x2xf32>) -> tensor<2x2xf32> { + %cst = "tf.Const"() {value = dense<7.000000e+00> : tensor} : () -> tensor + // CHECK: mhlo.broadcast_in_dim + // CHECK: mhlo.remainder + %6 = "tf.Mod"(%arg1, %cst) {_global_shape = [#tf_type.shape<4x8>], device = ""} : (tensor<2x2xf32>, tensor) -> tensor<2x2xf32> + return %6 : tensor<2x2xf32> + } + + // CHECK-LABEL: func @concat_v2 + func.func @concat_v2(%arg0: tensor<3x3xf32>, %arg1: tensor<3x3xf32>) -> tensor<6x3xf32> { + // CHECK: "mhlo.concatenate"({{.*}}) <{dimension = 0 : i64}> : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<6x3xf32> + %axis = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + %1 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<3x3xf32>, tensor<3x3xf32>, tensor) -> tensor<6x3xf32> + func.return %1 : tensor<6x3xf32> + } + + // CHECK-LABEL: func @matrix_diag_part_v3 + // CHECK-SAME: %[[ARG:.*]]: tensor<7x140x128xi32> + func.func @matrix_diag_part_v3(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + // CHECK: mhlo.iota + // CHECK: mhlo.reshape + // CHECK: mhlo.concatenate + // CHECK: mhlo.gather + // CHECK: mhlo.broadcast + // CHECK: mhlo.select + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "RIGHT_LEFT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + func.return %2: tensor<7x22x128xi32> + } + + // CHECK-LABEL: func @xla_call_module + func.func @xla_call_module(%arg0: tensor) -> tensor<*xf32> { + // Equivalent to the following: + // + // module @jit_sin { + // func.func public @main(%arg0: tensor) -> tensor { + // %0 = mhlo.sine %arg0 : tensor + // return %0 : tensor + // } + // } + // CHECK: call @main.1 + %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], function_list = [], disabled_checks = [], has_token_input_output = false, module = "ML\EFR\03MLIRxxx-trunk\00\01\17\05\01\05\01\03\05\03\07\07\t\0B\03K5\07\01\1B\07\0B\13\0B3\0B\0B\0B\0B\0F\0B\13\0B\03\1B\0F\1B\0B\0B\0B\0B\0B\0F\13\0B\0B\0B\0B\03\07\0F\17\07\02\A7\1F\05\0D\03\03\03\07\05\0F\03\0B\0B\1B\0D'\0F)\031\113\05\11\05\13\05\15\05\17\1D\15\17\05\19\17\19\EF\01\05\1B\03\03\1D\0D\05\1F!#%\1D\1D\1D\1F\1D!\1D##\03\03\03+\0D\03-/\1D%\1D'\1D)\1D+)\01\05\11\03\01\03\01\t\04A\05\01\11\01\05\07\03\01\05\03\11\01\t\05\03\05\0B\03\01\01\05\06\13\03\01\03\01\07\04\01\03\03\06\03\01\05\01\00\9A\04-\0F\0B\03!\1B\1D\05\1B\83/\1F\15\1D\15\11\13\15\11\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00sine_v1\00return_v1\00sym_name\00jit_sin\00arg_attrs\00function_type\00res_attrs\00sym_visibility\00jit(sin)/jit(main)/sin\00third_party/py/jax/experimental/jax2tf/tests/back_compat_test.py\00jax.arg_info\00x\00mhlo.sharding\00{replicated}\00jax.result_info\00\00main\00public\00", platforms = ["CPU"], version = 6 : i64} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } + + // Verifies that the following functions are added from xla_call_module. Note this must be at the end of the file. + // CHECK: func.func private @main.1(%arg0: tensor {mhlo.sharding = "{replicated}"}) -> tensor { + // CHECK: %0 = mhlo.sine %arg0 : tensor + // CHECK: return %0 : tensor + // CHECK: } + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a59194ef2773bd9d88a5971bd83b37db9815bb8d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/legalize-tf.mlir @@ -0,0 +1,2769 @@ +// RUN: tf-opt "-xla-legalize-tf=legalize-chlo=false" -split-input-file %s | env FILECHECK_OPTS="" FileCheck %s +// RUN: tf-opt "-xla-legalize-tf=legalize-chlo=true" -split-input-file -verify-diagnostics %s | FileCheck %s --check-prefix CHLO +// This test runs twice: +// 1. Through FILECHECK_OPTS="" FileCheck with chlo legalization disabled since verifying +// that the chlo ops emit produces more useful tests. +// 2. With chlo legalization enabled, verifying diagnostics to pick up any +// issues with the full lowering (can catch some broadcasting corner +// cases which emit with a warning). + +//===----------------------------------------------------------------------===// +// BatchNorm op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// fusedBatchNormV2 is almost identical to fusedBatchNormV3 (and uses the same +// code), so only do a couple of basic checks. + +// CHECK-LABEL: fusedBatchNormV2_noTraining +func.func @fusedBatchNormV2_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: "mhlo.batch_norm_inference"({{.*}}, %arg1, %arg2, %arg3, %arg4) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> + %0:5 = "tf.FusedBatchNormV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV2_training +func.func @fusedBatchNormV2_training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: %[[OUT:.*]], %[[MEAN:.*]], %[[VAR:.*]] = "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:5 = "tf.FusedBatchNormV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK: mhlo.constant + // CHECK: chlo.broadcast_multiply %[[VAR]], {{.*}} : (tensor<8xf32>, tensor) -> tensor<8xf32> + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_noTraining +func.func @fusedBatchNormV3_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: "mhlo.batch_norm_inference"({{.*}}, %arg1, %arg2, %arg3, %arg4) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> tensor<8x8x8x8xf32> + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_noTraining_mixedPrecision +// CHECK-SAME: ([[X:%.*]]: tensor<8x8x8x8xbf16>, [[SCALE:%.*]]: tensor<8xf32>, [[OFFSET:%.*]]: tensor<8xf32>, [[MEAN:%.*]]: tensor<8xf32>, [[VARIANCE:%.*]]: tensor<8xf32>) +func.func @fusedBatchNormV3_noTraining_mixedPrecision(%arg0: tensor<8x8x8x8xbf16>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<*xf32>) { + // CHECK: [[CONVERT_X:%.*]] = mhlo.convert [[X]] : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + // CHECK: [[Y:%.*]] = "mhlo.batch_norm_inference"([[CONVERT_X]], [[SCALE]], [[OFFSET]], [[MEAN]], [[VARIANCE]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<*xf32>) + // CHECK: [[Y_CONVERT:%.*]] = mhlo.convert [[Y]] : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + // CHECK: [[DUMMY:%.*]] = mhlo.constant dense<0.000000e+00> : tensor<0xf32> + // CHECK: [[DUMMY_CAST:%.*]] = tensor.cast [[DUMMY]] : tensor<0xf32> to tensor<*xf32> + // CHECK: return [[Y_CONVERT]], [[MEAN]], [[VARIANCE]], [[MEAN]], [[VARIANCE]], [[DUMMY_CAST]] + func.return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5 : tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<*xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_training +func.func @fusedBatchNormV3_training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: %[[OUT:.*]], %[[MEAN:.*]], %[[VAR:.*]] = "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK: mhlo.constant + // CHECK: chlo.broadcast_multiply %[[VAR]], {{.*}} : (tensor<8xf32>, tensor) -> tensor<8xf32> + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: func @fusedBatchNormV3_training_batchVariance +func.func @fusedBatchNormV3_training_batchVariance(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> tensor<8xf32> { + // CHECK: %[[OUT:.*]], %[[MEAN:.*]], %[[VAR:.*]] = "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK: return %[[VAR]] + func.return %0#4 : tensor<8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_training_exponentialAvgFactor +func.func @fusedBatchNormV3_training_exponentialAvgFactor(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) { + // CHECK: %[[OUT:.*]], %[[MEAN:.*]], %[[VAR:.*]] = "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 0.8 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK: %[[FACTOR:.*]] = mhlo.constant dense<1.00195694> + // CHECK: %[[CORRECTED_VAR:.*]] = chlo.broadcast_multiply %[[VAR]], %[[FACTOR]] + + // CHECK-DAG: %[[ALPHA:.*]] = mhlo.constant dense<0.199999988> + // CHECK-DAG: %[[BETA:.*]] = mhlo.constant dense<8.000000e-01> + + // CHECK: %[[ALPHA_MUL_OLD_MEAN:.*]] = chlo.broadcast_multiply %[[ALPHA]], %arg3 + // CHECK: %[[BETA_MUL_BATCH_MEAN:.*]] = chlo.broadcast_multiply %[[BETA]], %[[MEAN]] + // CHECK: %[[NEW_BATCH_MEAN:.*]] = chlo.broadcast_add %[[ALPHA_MUL_OLD_MEAN]], %[[BETA_MUL_BATCH_MEAN]] + + // CHECK: %[[ALPHA_MUL_OLD_VAR:.*]] = chlo.broadcast_multiply %[[ALPHA]], %arg4 + // CHECK: %[[BETA_MUL_CORRECTED_VAR:.*]] = chlo.broadcast_multiply %[[BETA]], %[[CORRECTED_VAR]] + // CHECK: %[[NEW_BATCH_VAR:.*]] = chlo.broadcast_add %[[ALPHA_MUL_OLD_VAR]], %[[BETA_MUL_CORRECTED_VAR]] + + // CHECK: return %[[NEW_BATCH_MEAN]], %[[NEW_BATCH_VAR]], %[[MEAN]], %[[VAR]] + func.return %0#1, %0#2, %0#3, %0#4 : tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_training_mixedPrecision +func.func @fusedBatchNormV3_training_mixedPrecision(%arg0: tensor<8x8x8x8xbf16>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>) { + // CHECK: mhlo.convert %arg0 : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK: mhlo.convert {{.*}} : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + func.return %0#0 : tensor<8x8x8x8xbf16> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_NCHW +func.func @fusedBatchNormV3_NCHW(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: "mhlo.batch_norm_training"({{.*}}, %arg1, %arg2) <{epsilon = 1.000000e-03 : f32, feature_index = 1 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_NDHWC +func.func @fusedBatchNormV3_NDHWC(%arg0: tensor<8x8x8x8x8xf32>, %arg1: tensor<8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8x8xf32>) { + // CHECK: feature_index = 4 : i64 + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NDHWC", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor<8x8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_noTraining_dynamic_supported +func.func @fusedBatchNormV3_noTraining_dynamic_supported(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor) -> (tensor) { + // CHECK: "mhlo.batch_norm_inference"({{.*}}, %arg1, %arg2, %arg3, %arg4) <{epsilon = 1.000000e-03 : f32, feature_index = 1 : i64}> : (tensor, tensor, tensor, tensor, tensor) -> tensor + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = false} : (tensor, tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0 : tensor +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_training_dynamic_unsupported1 +func.func @fusedBatchNormV3_training_dynamic_unsupported1(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor) -> (tensor) { + // CHECK: tf.FusedBatchNormV3 + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor, tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor, tensor, tensor, tensor) + func.return %0#0 : tensor +} + +// ----- + +// CHECK-LABEL: fusedBatchNormV3_training_dynamic_unsupported2 +func.func @fusedBatchNormV3_training_dynamic_unsupported2(%arg0: tensor, %arg1: tensor<6xf32>, %arg2: tensor<6xf32>, %arg3: tensor<6xf32>, %arg4: tensor<6xf32>) -> (tensor) { + // CHECK: tf.FusedBatchNormV3 + %0:6 = "tf.FusedBatchNormV3"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, exponential_avg_factor = 1.0 : f32, is_training = true} : (tensor, tensor<6xf32>, tensor<6xf32>, tensor<6xf32>, tensor<6xf32>) -> (tensor, tensor<6xf32>, tensor<6xf32>, tensor<6xf32>, tensor<6xf32>, tensor<6xf32>) + func.return %0#0 : tensor +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGrad_noTraining +func.func @fusedBatchNormGrad_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[eps:.*]] = mhlo.constant dense<1.000000e-03> : tensor + + // CHECK-NEXT: %[[add:.*]] = chlo.broadcast_add %arg4, %[[eps]] {broadcast_dimensions = array} : (tensor<8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr1:.*]] = mhlo.rsqrt %[[add]] : tensor<8xf32> + + // CHECK: %[[bcast_arg3:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg3, {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[sub:.*]] = mhlo.subtract %[[act]], %[[bcast_arg3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul:.*]] = mhlo.multiply %[[grad]], %[[sub]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cmul:.*]] = mhlo.convert %[[mul]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red1:.*]] = mhlo.reduce(%[[cmul]] init: %[[init]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr2:.*]] = mhlo.convert %[[red1]] : tensor<8xf32> + + // CHECK-NEXT: %[[mul2:.*]] = mhlo.multiply %arg2, %[[scr1]] : tensor<8xf32> + // CHECK: %[[bcast_mul2:.+]] = "mhlo.dynamic_broadcast_in_dim"(%[[mul2]], {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul3:.*]] = mhlo.multiply %[[grad]], %[[bcast_mul2]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[scale_backprop:.*]] = mhlo.multiply %[[scr1]], %[[scr2]] : tensor<8xf32> + + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cgrad:.*]] = mhlo.convert %[[grad]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init2:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red2:.*]] = mhlo.reduce(%[[cgrad]] init: %[[init2]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[offset_backprop:.*]] = mhlo.convert %[[red2]] : tensor<8xf32> + + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[mul3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGrad"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGrad_Training +func.func @fusedBatchNormGrad_Training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGrad"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV2_noTraining +func.func @fusedBatchNormGradV2_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[eps:.*]] = mhlo.constant dense<1.000000e-03> : tensor + + // CHECK-NEXT: %[[add:.*]] = chlo.broadcast_add %arg4, %[[eps]] {broadcast_dimensions = array} : (tensor<8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr1:.*]] = mhlo.rsqrt %[[add]] : tensor<8xf32> + + // CHECK: %[[bcast_arg3:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg3, {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[sub:.*]] = mhlo.subtract %[[act]], %[[bcast_arg3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul:.*]] = mhlo.multiply %[[grad]], %[[sub]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cmul:.*]] = mhlo.convert %[[mul]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red1:.*]] = mhlo.reduce(%[[cmul]] init: %[[init]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr2:.*]] = mhlo.convert %[[red1]] : tensor<8xf32> + + // CHECK-NEXT: %[[mul2:.*]] = mhlo.multiply %arg2, %[[scr1]] : tensor<8xf32> + // CHECK: %[[bcast_mul2:.+]] = "mhlo.dynamic_broadcast_in_dim"(%[[mul2]], {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul3:.*]] = mhlo.multiply %[[grad]], %[[bcast_mul2]] : tensor<8x8x8x8xf32> + + // CHECK-NEXT: %[[scale_backprop:.*]] = mhlo.multiply %[[scr1]], %[[scr2]] : tensor<8xf32> + + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cgrad:.*]] = mhlo.convert %[[grad]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init2:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red2:.*]] = mhlo.reduce(%[[cgrad]] init: %[[init2]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[offset_backprop:.*]] = mhlo.convert %[[red2]] : tensor<8xf32> + + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[mul3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV2_Training +func.func @fusedBatchNormGradV2_Training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV2_noTraining_mixed_precision +func.func @fusedBatchNormGradV2_noTraining_mixed_precision(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xbf16>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + + // CHECK: %[[x_backprop:.*]] = mhlo.convert {{.*}} : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xbf16> + + %0:5 = "tf.FusedBatchNormGradV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xbf16> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV2_Training_mixed_precision +func.func @fusedBatchNormGradV2_Training_mixed_precision(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xbf16>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xbf16> + + %0:5 = "tf.FusedBatchNormGradV2"(%arg0, %arg1, %arg2, %arg3, %arg4) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xbf16> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_noTraining +func.func @fusedBatchNormGradV3_noTraining(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[eps:.*]] = mhlo.constant dense<1.000000e-03> : tensor + + // CHECK-NEXT: %[[add:.*]] = chlo.broadcast_add %arg4, %[[eps]] {broadcast_dimensions = array} : (tensor<8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr1:.*]] = mhlo.rsqrt %[[add]] : tensor<8xf32> + + // CHECK: %[[bcast_arg3:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg3, {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[sub:.*]] = mhlo.subtract %[[act]], %[[bcast_arg3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul:.*]] = mhlo.multiply %[[grad]], %[[sub]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cmul:.*]] = mhlo.convert %[[mul]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red1:.*]] = mhlo.reduce(%[[cmul]] init: %[[init]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr2:.*]] = mhlo.convert %[[red1]] : tensor<8xf32> + + // CHECK-NEXT: %[[mul2:.*]] = mhlo.multiply %arg2, %[[scr1]] : tensor<8xf32> + // CHECK: %[[bcast_mul2:.+]] = "mhlo.dynamic_broadcast_in_dim"(%[[mul2]], {{.*}}) <{broadcast_dimensions = dense<3> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul3:.*]] = mhlo.multiply %[[grad]], %[[bcast_mul2]] : tensor<8x8x8x8xf32> + + // CHECK-NEXT: %[[scale_backprop:.*]] = mhlo.multiply %[[scr1]], %[[scr2]] : tensor<8xf32> + + // CHECK-NEXT: mhlo.constant dense<[0, 1, 2]> : tensor<3xi64> + // CHECK-NEXT: %[[cgrad:.*]] = mhlo.convert %[[grad]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init2:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red2:.*]] = mhlo.reduce(%[[cgrad]] init: %[[init2]]) applies mhlo.add across dimensions = [0, 1, 2] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[offset_backprop:.*]] = mhlo.convert %[[red2]] : tensor<8xf32> + + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[mul3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_Training +func.func @fusedBatchNormGradV3_Training(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<0xf32>, tensor<*xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : tensor<8x8x8x8xf32> + // CHECK: return %[[x_backprop]] + // CHECK-SAME: tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<0xf32>, tensor<*xf32>) + func.return %0#0, %0#3, %0#4 : tensor<8x8x8x8xf32>, tensor<0xf32>, tensor<*xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_noTraining_mixed_precision +func.func @fusedBatchNormGradV3_noTraining_mixed_precision(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xbf16>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + + // CHECK: %[[x_backprop:.*]] = mhlo.convert {{.*}} : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xbf16> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xbf16> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_Training_mixed_precision +func.func @fusedBatchNormGradV3_Training_mixed_precision(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xbf16>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xbf16>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : (tensor<8x8x8x8xbf16>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[grad_operand:.*]], %[[grad_scale:.*]], %[[grad_offset:.*]] = "mhlo.batch_norm_grad"(%[[act]], %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 3 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[grad_operand]] : (tensor<8x8x8x8xf32>) -> tensor<8x8x8x8xbf16> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xbf16> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NHWC", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xbf16>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xbf16> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_noTraining_NCHW +func.func @fusedBatchNormGradV3_noTraining_NCHW(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK-NEXT: %[[grad:.*]] = mhlo.convert %arg0 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[act:.*]] = mhlo.convert %arg1 : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[eps:.*]] = mhlo.constant dense<1.000000e-03> : tensor + + // CHECK-NEXT: %[[add:.*]] = chlo.broadcast_add %arg4, %[[eps]] {broadcast_dimensions = array} : (tensor<8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr1:.*]] = mhlo.rsqrt %[[add]] : tensor<8xf32> + + // CHECK: %[[bcast_arg3:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg3, {{.*}}) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[sub:.*]] = mhlo.subtract %[[act]], %[[bcast_arg3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul:.*]] = mhlo.multiply %[[grad]], %[[sub]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: mhlo.constant dense<[0, 2, 3]> : tensor<3xi64> + // CHECK-NEXT: %[[cmul:.*]] = mhlo.convert %[[mul]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red1:.*]] = mhlo.reduce(%[[cmul]] init: %[[init]]) applies mhlo.add across dimensions = [0, 2, 3] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[scr2:.*]] = mhlo.convert %[[red1]] : tensor<8xf32> + + // CHECK-NEXT: %[[mul2:.*]] = mhlo.multiply %arg2, %[[scr1]] : tensor<8xf32> + // CHECK: %[[bcast_mul2:.+]] = "mhlo.dynamic_broadcast_in_dim"(%[[mul2]], {{.*}}) <{broadcast_dimensions = dense<1> : tensor<1xi64>}> : (tensor<8xf32>, tensor<4xindex>) -> tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[mul3:.*]] = mhlo.multiply %[[grad]], %[[bcast_mul2]] : tensor<8x8x8x8xf32> + + // CHECK-NEXT: %[[scale_backprop:.*]] = mhlo.multiply %[[scr1]], %[[scr2]] : tensor<8xf32> + + // CHECK-NEXT: mhlo.constant dense<[0, 2, 3]> : tensor<3xi64> + // CHECK-NEXT: %[[cgrad:.*]] = mhlo.convert %[[grad]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: %[[init2:.*]] = mhlo.constant dense<-0.000000e+00> : tensor + // CHECK-NEXT: %[[red2:.*]] = mhlo.reduce(%[[cgrad]] init: %[[init2]]) applies mhlo.add across dimensions = [0, 2, 3] : (tensor<8x8x8x8xf32>, tensor) -> tensor<8xf32> + // CHECK-NEXT: %[[offset_backprop:.*]] = mhlo.convert %[[red2]] : tensor<8xf32> + + // CHECK-NEXT: %[[x_backprop:.*]] = mhlo.convert %[[mul3]] : tensor<8x8x8x8xf32> + // CHECK-NEXT: return %[[x_backprop]] : tensor<8x8x8x8xf32> + + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, is_training = false} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +// ----- + +// CHECK-LABEL: fusedBatchNormGradV3_Training_NCHW +func.func @fusedBatchNormGradV3_Training_NCHW(%arg0: tensor<8x8x8x8xf32>, %arg1: tensor<8x8x8x8xf32>, %arg2: tensor<8xf32>, %arg3: tensor<8xf32>, %arg4: tensor<8xf32>, %arg5: tensor<8xf32>) -> (tensor<8x8x8x8xf32>) { + // CHECK: %{{.*}} = "mhlo.batch_norm_grad"(%{{.*}}, %arg2, %arg3, %arg4, %[[grad]]) <{epsilon = 1.000000e-03 : f32, feature_index = 1 : i64}> : (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8x8x8x8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>) + %0:5 = "tf.FusedBatchNormGradV3"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) {T = "tfdtype$DT_FLOAT", data_format = "NCHW", epsilon = 0.001 : f32, is_training = true} : (tensor<8x8x8x8xf32>, tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) -> (tensor<8x8x8x8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>, tensor<8xf32>) + func.return %0#0 : tensor<8x8x8x8xf32> +} + +//===----------------------------------------------------------------------===// +// Bias op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @biasAdd_default +func.func @biasAdd_default(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // CHECK: %[[ARG0_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK: %[[ARG0_EXTENTS:.+]] = shape.to_extent_tensor %[[ARG0_SHAPE]] + // CHECK: %[[ARG1_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[ARG0_EXTENTS]]) + // CHECK-SAME: {broadcast_dimensions = dense<3> : tensor<1xi64>} + // CHECK: %[[RESULT:.+]] = mhlo.add %arg0, %[[ARG1_BCAST]] + %0 = "tf.BiasAdd"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT"} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} + +// ----- + +// CHECK-LABEL: func @biasAdd_NHWC +func.func @biasAdd_NHWC(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // CHECK: %[[ARG0_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK: %[[ARG0_EXTENTS:.+]] = shape.to_extent_tensor %[[ARG0_SHAPE]] + // CHECK: %[[ARG1_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[ARG0_EXTENTS]]) + // CHECK-SAME: {broadcast_dimensions = dense<3> : tensor<1xi64>} + // CHECK: %[[RESULT:.+]] = mhlo.add %arg0, %[[ARG1_BCAST]] + %0 = "tf.BiasAdd"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", data_format = "NHWC"} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} + +// ----- + +// CHECK-LABEL: func @biasAdd_NCHW +func.func @biasAdd_NCHW(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // CHECK: %[[ARG0_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK: %[[ARG0_EXTENTS:.+]] = shape.to_extent_tensor %[[ARG0_SHAPE]] + // CHECK: %[[ARG1_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[ARG0_EXTENTS]]) + // CHECK-SAME: {broadcast_dimensions = dense<1> : tensor<1xi64>} + // CHECK: %[[RESULT:.+]] = mhlo.add %arg0, %[[ARG1_BCAST]] + %0 = "tf.BiasAdd"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", data_format = "NCHW"} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} + +// ----- + +// CHECK-LABEL: func @biasAdd_dynamic +func.func @biasAdd_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: %[[ARG0_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK: %[[ARG0_EXTENTS:.+]] = shape.to_extent_tensor %[[ARG0_SHAPE]] + // CHECK: %[[ARG1_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[ARG0_EXTENTS]]) + // CHECK-SAME: {broadcast_dimensions = dense<1> : tensor<1xi64>} + // CHECK: %[[RESULT:.+]] = mhlo.add %arg0, %[[ARG1_BCAST]] + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NCHW"} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: func @biasAdd_partial_dynamic +func.func @biasAdd_partial_dynamic(%arg0: tensor, %arg1: tensor<512xi32>) -> tensor { + // CHECK: %[[ARG0_SHAPE:.+]] = shape.shape_of %arg0 + // CHECK: %[[ARG0_EXTENTS:.+]] = shape.to_extent_tensor %[[ARG0_SHAPE]] + // CHECK: %[[ARG1_BCAST:.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, %[[ARG0_EXTENTS]]) + // CHECK-SAME: {broadcast_dimensions = dense<3> : tensor<1xi64>} + // CHECK: %[[RESULT:.+]] = mhlo.add %arg0, %[[ARG1_BCAST]] + // CHECK: %[[CAST:.+]] = tensor.cast %[[RESULT]] : tensor to tensor + // CHECK: return %[[CAST]] : tensor + %0 = "tf.BiasAdd"(%arg0, %arg1) {data_format = "NHWC"} : (tensor, tensor<512xi32>) -> tensor + func.return %0 : tensor +} + + +//===----------------------------------------------------------------------===// +// ClipByValue +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @clip +func.func @clip(%arg0 : tensor, %arg1 : tensor, %arg2 : tensor) -> tensor { + // CHECK: [[VAL:%.+]] = mhlo.clamp %arg1, %arg0, %arg2 + + %0 = "tf.ClipByValue"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + // CHECK: return [[VAL]] + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @clip_dynamic +func.func @clip_dynamic(%arg0 : tensor, %arg1 : tensor, %arg2 : tensor) -> tensor { + // CHECK-DAG: [[CLAMP:%.+]] = mhlo.clamp %arg1, %arg0, %arg2 + %0 = "tf.ClipByValue"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + + // CHECK: return [[CLAMP]] + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @clip_static_broadcast +func.func @clip_static_broadcast(%arg0 : tensor<5xf32>, %arg1 : tensor, %arg2 : tensor) -> tensor<5xf32> { + // CHECK-DAG: [[SHPIDX:%.+]] = mhlo.constant dense<5> + // CHECK-DAG: [[BROADCAST_MIN:%.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, [[SHPIDX]]) <{broadcast_dimensions = dense<> : tensor<0xi64>}> + // CHECK-DAG: [[BROADCAST_MAX:%.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg2, [[SHPIDX]]) <{broadcast_dimensions = dense<> : tensor<0xi64>}> + // CHECK-DAG: [[CLAMP:%.+]] = mhlo.clamp [[BROADCAST_MIN]], %arg0, [[BROADCAST_MAX]] + %0 = "tf.ClipByValue"(%arg0, %arg1, %arg2) : (tensor<5xf32>, tensor, tensor) -> tensor<5xf32> + + // CHECK: return [[CLAMP]] + func.return %0 : tensor<5xf32> +} + + +// CHECK-LABEL: @clip_dynamic_broadcast +func.func @clip_dynamic_broadcast(%arg0 : tensor, %arg1 : tensor, %arg2 : tensor) -> tensor { + // CHECK: [[SHP:%.+]] = shape.shape_of %arg0 + // CHECK: [[SHPIDX:%.+]] = arith.index_cast [[SHP]] : tensor<1xindex> to tensor<1xi32> + // CHECK-DAG: [[BROADCAST_MIN:%.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg1, [[SHPIDX]]) <{broadcast_dimensions = dense<> : tensor<0xi64>}> + // CHECK-DAG: [[BROADCAST_MAX:%.+]] = "mhlo.dynamic_broadcast_in_dim"(%arg2, [[SHPIDX]]) <{broadcast_dimensions = dense<> : tensor<0xi64>}> + // CHECK-DAG: [[CLAMP:%.+]] = mhlo.clamp [[BROADCAST_MIN]], %arg0, [[BROADCAST_MAX]] + %0 = "tf.ClipByValue"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + + // CHECK: return [[CLAMP]] + func.return %0 : tensor +} + +//===----------------------------------------------------------------------===// +// DiagPart +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @diag_part +// CHECK-SAME: %[[ARG:.*]]: tensor<4x3x4x3xf32> +func.func @diag_part(%arg0: tensor<4x3x4x3xf32>) -> tensor<4x3xf32> { + // CHECK: %[[RS:.*]] = mhlo.reshape %[[ARG]] : (tensor<4x3x4x3xf32>) -> tensor<12x12xf32> + // CHECK-DAG: %[[IOTA0:.*]] = "mhlo.iota"() <{iota_dimension = 0 : i64}> : () -> tensor<12x12xi32> + // CHECK-DAG: %[[IOTA1:.*]] = "mhlo.iota"() <{iota_dimension = 1 : i64}> : () -> tensor<12x12xi32> + // CHECK-DAG: %[[COMP:.*]] = mhlo.compare EQ, %[[IOTA0]], %[[IOTA1]] : (tensor<12x12xi32>, tensor<12x12xi32>) -> tensor<12x12xi1> + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-DAG: %[[ZERO_MAT:.*]] = "mhlo.broadcast"(%[[ZERO]]) <{broadcast_sizes = dense<12> : tensor<2xi64>}> : (tensor) -> tensor<12x12xf32> + // CHECK-DAG: %[[SEL:.*]] = mhlo.select %[[COMP]], %[[RS]], %[[ZERO_MAT]] : tensor<12x12xi1>, tensor<12x12xf32> + // CHECK-DAG: %[[RED:.*]] = mhlo.reduce(%[[SEL]] init: %[[ZERO]]) applies mhlo.add across dimensions = [0] : (tensor<12x12xf32>, tensor) -> tensor<12xf32> + // CHECK-DAG: %[[RES:.*]] = mhlo.reshape %[[RED]] : (tensor<12xf32>) -> tensor<4x3xf32> + // CHECK-DAG: return %[[RES]] : tensor<4x3xf32> + %0 = "tf.DiagPart"(%arg0) : (tensor<4x3x4x3xf32>) -> tensor<4x3xf32> + func.return %0: tensor<4x3xf32> +} + +//===----------------------------------------------------------------------===// +// MatrixDiagPart +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @matrix_diag_part +// CHECK-SAME: %[[ARG:.*]]: tensor<7x140x128xi32> +func.func @matrix_diag_part(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + // CHECK-DAG: %[[V0:.*]] = mhlo.constant dense<42> : tensor + // CHECK-DAG: %[[V1:.*]] = mhlo.constant dense<[-10, 11]> : tensor<2xi32> + // CHECK-DAG: %[[V2:.*]] = "mhlo.iota"() <{iota_dimension = 1 : i64}> : () -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V3:.*]] = "mhlo.iota"() <{iota_dimension = 2 : i64}> : () -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V4:.*]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: %[[V5:.*]] = "mhlo.broadcast"(%[[V4]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V6:.*]] = mhlo.constant dense : tensor + // CHECK-DAG: %[[V7:.*]] = "mhlo.broadcast"(%[[V6]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V8:.*]] = mhlo.constant dense : tensor + // CHECK-DAG: %[[V9:.*]] = "mhlo.broadcast"(%[[V8]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V10:.*]] = mhlo.constant dense<11> : tensor + // CHECK-DAG: %[[V11:.*]] = "mhlo.broadcast"(%[[V10]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V12:.*]] = mhlo.constant dense<140> : tensor + // CHECK-DAG: %[[V13:.*]] = "mhlo.broadcast"(%[[V12]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V14:.*]] = mhlo.constant dense<128> : tensor + // CHECK-DAG: %[[V15:.*]] = "mhlo.broadcast"(%[[V14]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V16:.*]] = mhlo.constant dense<128> : tensor + // CHECK-DAG: %[[V17:.*]] = "mhlo.broadcast"(%[[V16]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi32> + // CHECK-DAG: %[[V18:.*]] = mhlo.subtract %[[V11]], %[[V2]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V19:.*]] = mhlo.negate %[[V18]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V20:.*]] = mhlo.minimum %[[V18]], %[[V5]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V21:.*]] = mhlo.add %[[V13]], %[[V20]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V22:.*]] = mhlo.maximum %[[V18]], %[[V5]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V23:.*]] = mhlo.subtract %[[V15]], %[[V22]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V24:.*]] = mhlo.minimum %[[V21]], %[[V23]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V25:.*]] = chlo.broadcast_compare %[[V18]], %[[V5]] {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V26:.*]] = mhlo.subtract %[[V17]], %[[V24]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V27:.*]] = mhlo.select %[[V25]], %[[V26]], %[[V5]] : tensor<1x22x128xi1>, tensor<1x22x128xi32> + // CHECK-DAG: %[[V28:.*]] = mhlo.maximum %[[V18]], %[[V5]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V29:.*]] = mhlo.subtract %[[V28]], %[[V27]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V30:.*]] = mhlo.maximum %[[V19]], %[[V5]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V31:.*]] = mhlo.subtract %[[V30]], %[[V27]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V32:.*]] = mhlo.add %[[V3]], %[[V29]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V33:.*]] = mhlo.add %[[V3]], %[[V31]] : tensor<1x22x128xi32> + // CHECK-DAG: %[[V34:.*]] = chlo.broadcast_compare %[[V32]], %[[V5]] {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V35:.*]] = chlo.broadcast_compare %[[V32]], %[[V15]] {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V36:.*]] = mhlo.and %[[V34]], %[[V35]] : tensor<1x22x128xi1> + // CHECK-DAG: %[[V37:.*]] = chlo.broadcast_compare %[[V33]], %[[V5]] {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V38:.*]] = chlo.broadcast_compare %[[V33]], %[[V13]] {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK-DAG: %[[V39:.*]] = mhlo.and %[[V37]], %[[V38]] : tensor<1x22x128xi1> + // CHECK-DAG: %[[V40:.*]] = mhlo.and %[[V36]], %[[V39]] : tensor<1x22x128xi1> + // CHECK-DAG: %[[V41:.*]] = mhlo.reshape %[[V40]] : (tensor<1x22x128xi1>) -> tensor<22x128xi1> + // CHECK-DAG: %[[V42:.*]] = "mhlo.concatenate"(%[[V33]], %[[V32]]) <{dimension = 0 : i64}> : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<2x22x128xi32> + // CHECK-DAG: %[[V43:.*]] = "mhlo.gather"(%[[ARG]], %[[V42]]) <{dimension_numbers = #mhlo.gather, slice_sizes = dense<[7, 1, 1]> : tensor<3xi64>}> : (tensor<7x140x128xi32>, tensor<2x22x128xi32>) -> tensor<7x22x128xi32> + // CHECK-DAG: %[[V44:.*]] = "mhlo.broadcast"(%[[V41]]) <{broadcast_sizes = dense<7> : tensor<1xi64>}> : (tensor<22x128xi1>) -> tensor<7x22x128xi1> + // CHECK-DAG: %[[V45:.*]] = "mhlo.broadcast"(%[[V0]]) <{broadcast_sizes = dense<[7, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<7x22x128xi32> + // CHECK: %[[V46:.*]] = mhlo.select %[[V44]], %[[V43]], %[[V45]] : tensor<7x22x128xi1>, tensor<7x22x128xi32> + // CHECK: return %[[V46]] : tensor<7x22x128xi32> + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "RIGHT_LEFT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + func.return %2: tensor<7x22x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_zero_dim_complex +func.func @matrix_diag_part_zero_dim_complex(%arg0: tensor<4x0xcomplex>) -> tensor<0xcomplex> { + %cst = "tf.Const"() {value = dense<-3> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {value = dense<(0.000000e+00,0.000000e+00)> : tensor>} : () -> tensor> + %0 = "tf.MatrixDiagPartV3"(%arg0, %cst, %cst_0) {align = "RIGHT_LEFT", device = ""} : (tensor<4x0xcomplex>, tensor, tensor>) -> tensor<0xcomplex> + // CHECK: return %{{[0-9]*}} : tensor<0xcomplex> + return %0 : tensor<0xcomplex> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_single_diagonal +func.func @matrix_diag_part_single_diagonal(%arg0: tensor<7x140x128xi32>) -> tensor<7x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<0> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "RIGHT_LEFT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x128xi32> + // CHECK: %[[result:.*]] = mhlo.reshape {{.*}} : (tensor<7x1x128xi32>) -> tensor<7x128xi32> + // CHECK: return %[[result]] : tensor<7x128xi32> + func.return %2: tensor<7x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_align_ll +func.func @matrix_diag_part_align_ll(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "LEFT_LEFT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + // CHECK: %[[false:.*]] = mhlo.constant dense : tensor + // CHECK: %[[b_false:.*]] = "mhlo.broadcast"(%[[false]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi1> + // CHECK: %{{[0-9]*}} = mhlo.select %[[b_false]], %{{[0-9]*}}, %{{[0-9]*}} : tensor<1x22x128xi1>, tensor<1x22x128xi32> + func.return %2: tensor<7x22x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_align_lr +func.func @matrix_diag_part_align_lr(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "LEFT_RIGHT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + // CHECK: %[[le:.*]] = chlo.broadcast_compare %{{[0-9]*}}, %{{[0-9]*}} {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK: %{{[0-9]*}} = mhlo.select %[[le]], %{{[0-9]*}}, %{{[0-9]*}} : tensor<1x22x128xi1>, tensor<1x22x128xi32> + func.return %2: tensor<7x22x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_align_rl +func.func @matrix_diag_part_align_rl(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "RIGHT_LEFT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + // CHECK: %[[ge:.*]] = chlo.broadcast_compare %{{[0-9]*}}, %{{[0-9]*}} {comparison_direction = #chlo} : (tensor<1x22x128xi32>, tensor<1x22x128xi32>) -> tensor<1x22x128xi1> + // CHECK: %{{[0-9]*}} = mhlo.select %[[ge]], %{{[0-9]*}}, %{{[0-9]*}} : tensor<1x22x128xi1>, tensor<1x22x128xi32> + func.return %2: tensor<7x22x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_align_rr +func.func @matrix_diag_part_align_rr(%arg0: tensor<7x140x128xi32>) -> tensor<7x22x128xi32> { + %0 = mhlo.constant dense<42> : tensor // padding value + %1 = mhlo.constant dense<[-10, 11]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = i32, align = "RIGHT_RIGHT" + } : (tensor<7x140x128xi32>, tensor<2xi32>, tensor) -> tensor<7x22x128xi32> + // CHECK: %[[true:.*]] = mhlo.constant dense : tensor + // CHECK: %[[b_true:.*]] = "mhlo.broadcast"(%[[true]]) <{broadcast_sizes = dense<[1, 22, 128]> : tensor<3xi64>}> : (tensor) -> tensor<1x22x128xi1> + // CHECK: %{{[0-9]*}} = mhlo.select %[[b_true]], %{{[0-9]*}}, %{{[0-9]*}} : tensor<1x22x128xi1>, tensor<1x22x128xi32> + func.return %2: tensor<7x22x128xi32> +} + +// ----- + +// CHECK-LABEL: func @matrix_diag_part_align_7d +// CHECK: (%arg0: tensor<3x5x7x9x11x13x17xf32>) -> tensor<3x5x7x9x11x4x10xf32> +func.func @matrix_diag_part_align_7d(%arg0: tensor<3x5x7x9x11x13x17xf32>) -> tensor<3x5x7x9x11x4x10xf32> { + %0 = mhlo.constant dense<-1.> : tensor // padding value + %1 = mhlo.constant dense<[-6, -3]> : tensor<2xi32> // k + %2 = "tf.MatrixDiagPartV3"(%arg0, %1, %0) { + T = f32, align = "LEFT_RIGHT" + } : (tensor<3x5x7x9x11x13x17xf32>, tensor<2xi32>, tensor) -> tensor<3x5x7x9x11x4x10xf32> + func.return %2: tensor<3x5x7x9x11x4x10xf32> +} + +//===----------------------------------------------------------------------===// +// Conv +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: func @conv2d_NHWC +func.func @conv2d_NHWC(%arg0: tensor<1x4x4x2xf32> {tf_saved_model.index_path = ["input_2"]}, %arg1: tensor<3x3x2x2xf32>, %arg2: tensor<2xf32>, %arg3: tensor<2xf32>, %arg4: tensor<2xf32>, %arg5: tensor<2xf32>, %arg6: tensor<2xf32>, %arg7: tensor<2xf32>) -> (tensor<1x4x4x2xf32> {tf_saved_model.index_path = [""]}) { + // CHECK{LITERAL}: mhlo.convolution(%arg0, %arg1) dim_numbers = [b, 0, 1, f]x[0, 1, i, o]->[b, 0, 1, f], window = {stride = [1, 1], pad = [[1, 1], [1, 1]], rhs_dilate = [1, 1]} {batch_group_count = 1 : i64, feature_group_count = 1 : i64, precision_config = [#mhlo, #mhlo]} : (tensor<1x4x4x2xf32>, tensor<3x3x2x2xf32>) -> tensor<1x4x4x2xf32> + %0 = "tf.Conv2D"(%arg0, %arg1) <{data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 1, 1, 1], use_cudnn_on_gpu = true}> {device = ""} : (tensor<1x4x4x2xf32>, tensor<3x3x2x2xf32>) -> tensor<1x4x4x2xf32> + %1 = "tf.Mul"(%0, %arg6) : (tensor<1x4x4x2xf32>, tensor<2xf32>) -> tensor<1x4x4x2xf32> + %2 = "tf.AddV2"(%1, %arg7) : (tensor<1x4x4x2xf32>, tensor<2xf32>) -> tensor<1x4x4x2xf32> + return %2 : tensor<1x4x4x2xf32> +} + +// ----- + +// CHECK-LABEL: func @conv2d_backprop_input +func.func @conv2d_backprop_input(%arg0: tensor<3x3x8x8xf32>, %arg1: tensor<1x128x192x8xf32>) -> tensor<1x256x384x8xf32> { + %cst = "tf.Const"() <{value = dense<[1, 256, 384, 8]> : tensor<4xi32>}> : () -> tensor<4xi32> + %0 = "tf.Conv2DBackpropInput"(%cst, %arg0, %arg1) <{data_format = "NHWC", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true}> {device = ""} : (tensor<4xi32>, tensor<3x3x8x8xf32>, tensor<1x128x192x8xf32>) -> tensor<1x256x384x8xf32> + return %0 : tensor<1x256x384x8xf32> + } + +//===----------------------------------------------------------------------===// +// Cumulative +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @cumsum +func.func @cumsum(%arg0: tensor<1x4x1xf32>) -> tensor<1x4x1xf32> { + // CHECK: mhlo.reduce_window + // CHECK-SAME{LITERAL}: padding = dense<[[0, 0], [3, 0], [0, 0]]> : tensor<3x2xi64>, window_dimensions = dense<[1, 4, 1]> : tensor<3xi64>, window_strides = dense<1> : tensor<3xi64> + // CHECK: mhlo.add + %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %0 = "tf.Cumsum"(%arg0, %cst) <{exclusive = false, reverse = false}> {device = ""} : (tensor<1x4x1xf32>, tensor) -> tensor<1x4x1xf32> + return %0 : tensor<1x4x1xf32> +} + +// ----- + +// CHECK-LABEL: func @cumprod +func.func @cumprod(%arg0: tensor<1x4x1xf32>) -> tensor<1x4x1xf32> { + // CHECK: mhlo.reduce_window + // CHECK-SAME{LITERAL}: padding = dense<0> : tensor<3x2xi64>, window_dimensions = dense<1> : tensor<3xi64>, window_strides = dense<1> : tensor<3xi64> + // CHECK: mhlo.multiply + %cst = "tf.Const"() <{value = dense<2> : tensor}> : () -> tensor + %0 = "tf.Cumprod"(%arg0, %cst) <{exclusive = false, reverse = false}> {device = ""} : (tensor<1x4x1xf32>, tensor) -> tensor<1x4x1xf32> + return %0 : tensor<1x4x1xf32> +} + +//===----------------------------------------------------------------------===// +// DynamicSlice +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @dynamic_slice_i32 +func.func @dynamic_slice_i32(%arg0: tensor<8x512x384xbf16>, %arg1: tensor<3xi32>) -> tensor<1x512x384xbf16> attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2", outputs = "_retval0"}} { + %cst = "tf.Const"() <{value = dense<[1, 512, 384]> : tensor<3xi32>}> : () -> tensor<3xi32> + // CHECK: "mhlo.dynamic_slice"{{.*}}slice_sizes = dense<[1, 512, 384]> : tensor<3xi64> + %0 = "tf.XlaDynamicSlice"(%arg0, %arg1, %cst) {device = ""} : (tensor<8x512x384xbf16>, tensor<3xi32>, tensor<3xi32>) -> tensor<1x512x384xbf16> + return %0 : tensor<1x512x384xbf16> +} + +// ----- + +// CHECK-LABEL: func @dynamic_slice_i64 +func.func @dynamic_slice_i64(%arg0: tensor<8x512x384xbf16>, %arg1: tensor<3xi32>) -> tensor<1x512x384xbf16> attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2", outputs = "_retval0"}} { + %cst = "tf.Const"() <{value = dense<[1, 512, 384]> : tensor<3xi64>}> : () -> tensor<3xi64> + // CHECK: "mhlo.dynamic_slice"{{.*}}slice_sizes = dense<[1, 512, 384]> : tensor<3xi64> + %0 = "tf.XlaDynamicSlice"(%arg0, %arg1, %cst) {device = ""} : (tensor<8x512x384xbf16>, tensor<3xi32>, tensor<3xi64>) -> tensor<1x512x384xbf16> + return %0 : tensor<1x512x384xbf16> +} + +//===----------------------------------------------------------------------===// +// Erf +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @erf +func.func @erf(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> { + // CHECK: chlo.erf %arg0 : tensor<2x3xf32> + // CHLO: mhlo.erf %arg0 : tensor<2x3xf32> + %0 = "tf.Erf"(%arg0) : (tensor<2x3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> +} + +//===----------------------------------------------------------------------===// +// Erfc +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @erfc +func.func @erfc(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> { + // CHECK: chlo.erfc %arg0 : tensor<2x3xf32> + %0 = "tf.Erfc"(%arg0) : (tensor<2x3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> +} + +//===----------------------------------------------------------------------===// +// Einsum. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @einsum +func.func @einsum(%arg0: tensor<2x3xf32>, %arg1: tensor<3x4xf32>) -> tensor<2x4xf32> { + // CHECK: mhlo.einsum + %0 = "tf.Einsum"(%arg0, %arg1) {equation = "ab,bc->ac"} : (tensor<2x3xf32>, tensor<3x4xf32>) -> tensor<2x4xf32> + func.return %0: tensor<2x4xf32> +} + +// ----- + +// CHECK-LABEL: func @unary_einsum +func.func @unary_einsum(%arg0: tensor<2x3xf32>) -> tensor<2x2xf32> { + // CHECK: mhlo.constant{{.*}}1.000000e+00 + // CHECK: mhlo.einsum{{.*}}",ab->aa" + %0 = "tf.Einsum"(%arg0) {equation = "ab->aa"} : (tensor<2x3xf32>) -> tensor<2x2xf32> + func.return %0: tensor<2x2xf32> +} + +//===----------------------------------------------------------------------===// +// FloorDiv and FloorMod. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @floordiv_broadcast_i32 +func.func @floordiv_broadcast_i32(%arg0: tensor<2x3xi32>, %arg1: tensor<3xi32>) -> tensor<2x3xi32> { + // CHECK-DAG: [[DIV:%.+]] = chlo.broadcast_divide %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[MUL:%.+]] = chlo.broadcast_multiply [[DIV]], %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[MUL]], %arg0 {comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS1:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg0, [[ZEROS1]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS2:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare %arg1, [[ZEROS2]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ONES:%.+]] = mhlo.constant dense<1> + // CHECK-DAG: [[SUB:%.+]] = chlo.broadcast_subtract [[DIV]], [[ONES]] + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[SUB]], [[DIV]] + // CHECK: return [[SELECT]] + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor<2x3xi32>, tensor<3xi32>) -> tensor<2x3xi32> + func.return %0: tensor<2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @floordiv_reverse_broadcast_i32 +func.func @floordiv_reverse_broadcast_i32(%arg0: tensor<3xi32>, %arg1: tensor<2x3xi32>) -> tensor<2x3xi32> { + // CHECK-DAG: [[DIV:%.+]] = chlo.broadcast_divide %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[MUL:%.+]] = chlo.broadcast_multiply [[DIV]] + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[MUL]], %arg0 {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS1:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg0, [[ZEROS1]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS2:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare %arg1, [[ZEROS2]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ONES:%.+]] = mhlo.constant dense<1> + // CHECK-DAG: [[SUB:%.+]] = chlo.broadcast_subtract [[DIV]], [[ONES]] + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[SUB]], [[DIV]] + // CHECK: return [[SELECT]] + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor<3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32> + func.return %0: tensor<2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @floordiv_f32 +func.func @floordiv_f32(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-NEXT: %[[DIV:.*]] = chlo.broadcast_divide %arg0, %arg0 + // CHECK-NEXT: %[[FLOOR:.*]] = mhlo.floor %[[DIV]] + // CHECK-NEXT: return %[[FLOOR]] : tensor<2xf32> + %0 = "tf.FloorDiv"(%arg0, %arg0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0: tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @floordiv_bf16 +func.func @floordiv_bf16(%arg0: tensor<2xbf16>) -> tensor<2xbf16> { + // CHECK-NEXT: mhlo.convert + // CHECK-NEXT: mhlo.convert + // CHECK-NEXT: chlo.broadcast_divide + // CHECK-NEXT: mhlo.floor + // CHECK-NEXT: mhlo.convert + // CHECK-NEXT: return + %0 = "tf.FloorDiv"(%arg0, %arg0) : (tensor<2xbf16>, tensor<2xbf16>) -> tensor<2xbf16> + func.return %0: tensor<2xbf16> +} + +// ----- + +// CHECK-LABEL: func @floordiv_f16_broadcast +func.func @floordiv_f16_broadcast(%arg0: tensor<2x3xf16>, %arg1: tensor<3xf16>) -> tensor<2x3xf16> { + // CHECK-NEXT: chlo.broadcast_divide + // CHECK-NEXT: mhlo.floor + // CHECK-NEXT: return + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor<2x3xf16>, tensor<3xf16>) -> tensor<2x3xf16> + func.return %0: tensor<2x3xf16> +} + +// ----- + +// CHECK-LABEL: func @floordiv_dynamic +func.func @floordiv_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: [[DIV:%.+]] = chlo.broadcast_divide %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[MUL:%.+]] = chlo.broadcast_multiply [[DIV]], %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[MUL]], %arg0 {comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS1:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg0, [[ZEROS1]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZEROS2:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare %arg1, [[ZEROS2]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ONES:%.+]] = mhlo.constant dense<1> + // CHECK-DAG: [[SUB:%.+]] = chlo.broadcast_subtract [[DIV]], [[ONES]] + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[SUB]], [[DIV]] + // CHECK: return [[SELECT]] + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @floordiv_unsigned +func.func @floordiv_unsigned(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: [[DIV:%.+]] = chlo.broadcast_divide %arg0, %arg1 {broadcast_dimensions = array} + // CHECK: return [[DIV]] + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @floordiv_int +func.func @floordiv_int(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: [[DIV:%.+]] = chlo.broadcast_divide %arg0, %arg1 : (tensor, tensor) -> tensor + // CHECK-DAG: [[MUL:%.+]] = chlo.broadcast_multiply [[DIV]], %arg1 : (tensor, tensor) -> tensor + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[MUL]], %arg0 {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[ZEROS1:%.+]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg0, [[ZEROS1]] {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[ZEROS2:%.+]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare %arg1, [[ZEROS2]] {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ONES:%.+]] = mhlo.constant dense<1> : tensor + // CHECK-DAG: [[SUB:%.+]] = chlo.broadcast_subtract [[DIV]], [[ONES]] + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[SUB]], [[DIV]] + // CHECK: return [[SELECT]] + %0 = "tf.FloorDiv"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @floormod_broadcast_numerator +func.func @floormod_broadcast_numerator(%arg0: tensor<3xi32>, %arg1: tensor<2x3xi32>) -> tensor<2x3xi32> { + // CHECK-DAG: [[REM:%.+]] = chlo.broadcast_remainder %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[ZL:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[REM]], [[ZL]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZR:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg1, [[ZR]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare [[REM]], [[ZR]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ADD:%.+]] = chlo.broadcast_add %arg1, [[REM]] + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[ADD]], [[REM]] + // CHECK-NEXT: return [[SELECT]] + %0 = "tf.FloorMod"(%arg0, %arg1) : (tensor<3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32> + func.return %0: tensor<2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @floormod_broadcast_denominator +func.func @floormod_broadcast_denominator(%arg0: tensor<2x3xi32>, %arg1: tensor<3xi32>) -> tensor<2x3xi32> { + // CHECK-DAG: [[REM:%.+]] = chlo.broadcast_remainder %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[ZL:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[REM]], [[ZL]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZR:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg1, [[ZR]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare [[REM]], [[ZR]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ADD:%.+]] = chlo.broadcast_add %arg1, [[REM]] {broadcast_dimensions = array} + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[ADD]], [[REM]] + // CHECK-NEXT: return [[SELECT]] + %0 = "tf.FloorMod"(%arg0, %arg1) : (tensor<2x3xi32>, tensor<3xi32>) -> tensor<2x3xi32> + func.return %0: tensor<2x3xi32> +} + +// ----- + +// CHECK-LABEL: func @floormod_unsigned_broadcast_denominator +func.func @floormod_unsigned_broadcast_denominator(%arg0: tensor<2x3xui32>, %arg1: tensor<3xui32>) -> tensor<2x3xui32> { + // CHECK-DAG: [[REM:%.+]] = chlo.broadcast_remainder %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-NEXT: return [[REM]] + %0 = "tf.FloorMod"(%arg0, %arg1) : (tensor<2x3xui32>, tensor<3xui32>) -> tensor<2x3xui32> + func.return %0: tensor<2x3xui32> +} + +// ----- + +// CHECK-LABEL: func @floormod_dynamic_broadcast_numerator +func.func @floormod_dynamic_broadcast_numerator_(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-DAG: [[REM:%.+]] = chlo.broadcast_remainder %arg0, %arg1 {broadcast_dimensions = array} + // CHECK-DAG: [[ZL:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[REM]], [[ZL]] {comparison_direction = #chlo} + // CHECK-DAG: [[ZR:%.+]] = mhlo.constant dense<0> + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg1, [[ZR]] {comparison_direction = #chlo} + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare [[REM]], [[ZR]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] + // CHECK-DAG: [[ADD:%.+]] = chlo.broadcast_add %arg1, [[REM]] {broadcast_dimensions = array} + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[ADD]], [[REM]] + // CHECK-NEXT: return [[SELECT]] + %0 = "tf.FloorMod"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @floormod_dynamic_broadcast_denominator +func.func @floormod_dynamic_broadcast_denominator_(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK-NOT: tf.FloorMod + // CHECK-DAG: [[REM:%.+]] = chlo.broadcast_remainder %arg0, %arg1 {broadcast_dimensions = array} : (tensor, tensor) -> tensor + // CHECK-DAG: [[ZL:%.+]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-DAG: [[CMP1:%.+]] = chlo.broadcast_compare [[REM]], [[ZL]] {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[ZR:%.+]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-DAG: [[CMP2:%.+]] = chlo.broadcast_compare %arg1, [[ZR]] {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[CMP3:%.+]] = chlo.broadcast_compare [[REM]], [[ZR]] {broadcast_dimensions = array, comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[CMP4:%.+]] = chlo.broadcast_compare [[CMP2]], [[CMP3]] {comparison_direction = #chlo} : (tensor, tensor) -> tensor + // CHECK-DAG: [[AND:%.+]] = chlo.broadcast_and [[CMP1]], [[CMP4]] : (tensor, tensor) -> tensor + // CHECK-DAG: [[ADD:%.+]] = chlo.broadcast_add %arg1, [[REM]] : (tensor, tensor) -> tensor + // CHECK-DAG: [[SELECT:%.+]] = mhlo.select [[AND]], [[ADD]], [[REM]] : tensor, tensor + // CHECK-NEXT: return [[SELECT]] : tensor + %0 = "tf.FloorMod"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0: tensor +} + +//===----------------------------------------------------------------------===// +// OnesLike +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @ones_like +// CHECK-SAME: (%[[ARG:.*]]: tensor<2x?xf32>) +func.func @ones_like(%arg0: tensor<2x?xf32>) -> tensor<2x?xf32> { + // CHECK: %[[RES:.*]] = "chlo.constant_like"(%[[ARG]]) <{value = 1.0{{.*}}}> + // CHECK: return %[[RES]] + %0 = "tf.OnesLike"(%arg0) : (tensor<2x?xf32>) -> tensor<2x?xf32> + func.return %0 : tensor<2x?xf32> +} + +//===----------------------------------------------------------------------===// +// ZerosLike +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @zeros_like +// CHECK-SAME: (%[[ARG:.*]]: tensor<2x?xf32>) +func.func @zeros_like(%arg0: tensor<2x?xf32>) -> tensor<2x?xf32> { + // CHECK: %[[RES:.*]] = "chlo.constant_like"(%[[ARG]]) <{value = 0.0{{.*}}}> + // CHECK: return %[[RES]] + %0 = "tf.ZerosLike"(%arg0) : (tensor<2x?xf32>) -> tensor<2x?xf32> + func.return %0 : tensor<2x?xf32> +} + +//===----------------------------------------------------------------------===// +// BroadcastTo. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @broadcast_to +func.func @broadcast_to(%arg0: tensor<16xf32>) -> tensor<16x16x16x16xf32> { + %cst = "tf.Const"() { value = dense<16> : tensor<4xi32> } : () -> tensor<4xi32> + + // CHECK: [[CST:%.+]] = mhlo.constant + // CHECK: "mhlo.dynamic_broadcast_in_dim"(%arg0, [[CST]]) + // CHECK-SAME: {broadcast_dimensions = dense<3> : tensor<1xi64>} + %0 = "tf.BroadcastTo"(%arg0, %cst) : (tensor<16xf32>, tensor<4xi32>) -> tensor<16x16x16x16xf32> + func.return %0 : tensor<16x16x16x16xf32> +} + +//===----------------------------------------------------------------------===// +// Complex op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @complex +func.func @complex(%arg0: tensor<3xf32>, %arg1: tensor<3xf32>) -> tensor<3xcomplex> { + // CHECK: chlo.broadcast_complex + %1 = "tf.Complex"(%arg0, %arg1) : (tensor<3xf32>, tensor<3xf32>) -> tensor<3xcomplex> + func.return %1 : tensor<3xcomplex> +} + +// ----- + +// CHECK-LABEL: func @imag +func.func @imag(%arg0: tensor<3xcomplex>) -> tensor<3xf32> { + // CHECK: mhlo.imag + %1 = "tf.Imag"(%arg0) : (tensor<3xcomplex>) -> tensor<3xf32> + func.return %1 : tensor<3xf32> +} + +// ----- + +// CHECK-LABEL: func @real +func.func @real(%arg0: tensor<3xcomplex>) -> tensor<3xf32> { + // CHECK: mhlo.real + %1 = "tf.Real"(%arg0) : (tensor<3xcomplex>) -> tensor<3xf32> + func.return %1 : tensor<3xf32> +} + +//===----------------------------------------------------------------------===// +// Concat op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @concat_v2 +func.func @concat_v2(%arg0: tensor<3x3xf32>, %arg1: tensor<3x3xf32>) -> tensor<6x3xf32> { + // CHECK: "mhlo.concatenate"({{.*}}) <{dimension = 0 : i64}> : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<6x3xf32> + %axis = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + %1 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<3x3xf32>, tensor<3x3xf32>, tensor) -> tensor<6x3xf32> + func.return %1 : tensor<6x3xf32> +} + +// ----- + +// CHECK-LABEL: func @concat_v2_neg_axis +func.func @concat_v2_neg_axis(%arg0: tensor<3x3xf32>, %arg1: tensor<3x3xf32>) -> tensor<6x3xf32> { + // CHECK: "mhlo.concatenate"({{.*}}) <{dimension = 0 : i64}> : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<6x3xf32> + + %axis = "tf.Const"() { value = dense<-2> : tensor } : () -> tensor + %1 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<3x3xf32>, tensor<3x3xf32>, tensor) -> tensor<6x3xf32> + func.return %1 : tensor<6x3xf32> +} + +// ----- + +// CHECK-LABEL: func @concat_v2_1d_axis +func.func @concat_v2_1d_axis(%arg0: tensor<3x3xf32>, %arg1: tensor<3x3xf32>) -> tensor<3x6xf32> { + // CHECK: "mhlo.concatenate"({{.*}}) <{dimension = 1 : i64}> : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x6xf32> + + %axis = "tf.Const"() { value = dense<[1]> : tensor<1xi64> } : () -> tensor<1xi64> + %1 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<3x3xf32>, tensor<3x3xf32>, tensor<1xi64>) -> tensor<3x6xf32> + func.return %1 : tensor<3x6xf32> +} + +// ----- + +// CHECK-LABEL: func @concat_v2_non_const_axis +func.func @concat_v2_non_const_axis(%arg0: tensor<3x3xf32>, %arg1: tensor<3x3xf32>, %axis: tensor) -> tensor<3x6xf32> { + // CHECK: "tf.ConcatV2" + %1 = "tf.ConcatV2"(%arg0, %arg1, %axis) : (tensor<3x3xf32>, tensor<3x3xf32>, tensor) -> tensor<3x6xf32> + func.return %1 : tensor<3x6xf32> +} + +//===----------------------------------------------------------------------===// +// Pad op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @padv2_1D +func.func @padv2_1D(%arg0: tensor<3xf32>, %arg1: tensor) -> tensor<6xf32> { + %padding = "tf.Const"() { value = dense<[[1, 2]]> : tensor<1x2xi64> } : () -> tensor<1x2xi64> + // CHECK: "mhlo.pad"(%arg0, %arg1) <{ + // CHECK-SAME: edge_padding_high = dense<2> : tensor<1xi64>, + // CHECK-SAME: edge_padding_low = dense<1> : tensor<1xi64>, + // CHECK-SAME: interior_padding = dense<0> : tensor<1xi64> + %1 = "tf.PadV2"(%arg0, %padding, %arg1) : (tensor<3xf32>, tensor<1x2xi64>, tensor) -> tensor<6xf32> + func.return %1 : tensor<6xf32> +} + +// ----- + +// CHECK-LABEL: func @padv2_2D +func.func @padv2_2D(%arg0: tensor<3x2xf32>, %arg1: tensor) -> tensor<6x9xf32> { + %padding = "tf.Const"() { value = dense<[[1,2],[3,4]]> : tensor<2x2xi64> } : () -> tensor<2x2xi64> + // CHECK: "mhlo.pad"(%arg0, %arg1) <{ + // CHECK-SAME: edge_padding_high = dense<[2, 4]> : tensor<2xi64>, + // CHECK-SAME: edge_padding_low = dense<[1, 3]> : tensor<2xi64>, + // CHECK-SAME: interior_padding = dense<0> : tensor<2xi64> + %1 = "tf.PadV2"(%arg0, %padding, %arg1) : (tensor<3x2xf32>, tensor<2x2xi64>, tensor) -> tensor<6x9xf32> + func.return %1 : tensor<6x9xf32> +} + +// ----- + +// CHECK-LABEL: func @padv2_i32_paddings +func.func @padv2_i32_paddings(%arg0: tensor<3x2xf32>, %arg1: tensor) -> tensor<6x9xf32> { + %padding = "tf.Const"() { value = dense<[[1,2],[3,4]]> : tensor<2x2xi32> } : () -> tensor<2x2xi32> + // CHECK: "mhlo.pad"(%arg0, %arg1) <{ + // CHECK-SAME: edge_padding_high = dense<[2, 4]> : tensor<2xi64>, + // CHECK-SAME: edge_padding_low = dense<[1, 3]> : tensor<2xi64>, + // CHECK-SAME: interior_padding = dense<0> : tensor<2xi64> + %1 = "tf.PadV2"(%arg0, %padding, %arg1) : (tensor<3x2xf32>, tensor<2x2xi32>, tensor) -> tensor<6x9xf32> + func.return %1 : tensor<6x9xf32> +} + +// ----- + +// CHECK-LABEL: func @padv2_dynamic +func.func @padv2_dynamic(%arg0: tensor, %arg1: tensor, %arg2: tensor<1x2xi64>) -> tensor { + // CHECK: "mhlo.transpose"({{.*}}) <{permutation = dense<[1, 0]> : tensor<2xi64>}> : (tensor<1x2xi64>) -> tensor<2x1xi64> + // CHECK: mhlo.reshape {{.*}} : (tensor<2x1xi64>) -> tensor<2xi64> + // CHECK: "mhlo.slice"({{.*}}) <{limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi64>) -> tensor<1xi64> + // CHECK: "mhlo.slice"({{.*}}) <{limit_indices = dense<2> : tensor<1xi64>, start_indices = dense<1> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<2xi64>) -> tensor<1xi64> + // CHECK: mhlo.dynamic_pad {{.*}} : (tensor, tensor, tensor<1xi64>, tensor<1xi64>, tensor<1xi64>) -> tensor + %1 = "tf.PadV2"(%arg0, %arg2, %arg1) : (tensor, tensor<1x2xi64>, tensor) -> tensor + func.return %1 : tensor +} + +//===----------------------------------------------------------------------===// +// Identity op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @identity +func.func @identity(%arg0: tensor<1xi32>) -> tensor<1xi32> { + // CHECK-NEXT: return %arg0 : tensor<1xi32> + %0 = "tf.Identity"(%arg0) : (tensor<1xi32>) -> tensor<1xi32> + func.return %0: tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @identityN +func.func @identityN(%arg0: tensor<1xi32>, %arg1: tensor<1xf32>) -> (tensor<1xi32>, tensor<1xf32>) { + // CHECK-NEXT: return %arg0, %arg1 : tensor<1xi32>, tensor<1xf32> + %0:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor<1xi32>, tensor<1xf32>) -> (tensor<1xi32>, tensor<1xf32>) + func.return %0#0, %0#1: tensor<1xi32>, tensor<1xf32> +} + +// ----- + +// CHECK-LABEL: func @stopgradient +func.func @stopgradient(%arg0: tensor<1xi32>) -> tensor<1xi32> { + // CHECK-NEXT: return %arg0 : tensor<1xi32> + %0 = "tf.StopGradient"(%arg0) : (tensor<1xi32>) -> tensor<1xi32> + func.return %0: tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @preventgradient +func.func @preventgradient(%arg0: tensor<1xi32>) -> tensor<1xi32> { + // CHECK-NEXT: return %arg0 : tensor<1xi32> + %0 = "tf.PreventGradient"(%arg0) {message = "fin gradients"} : (tensor<1xi32>) -> tensor<1xi32> + func.return %0: tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @checkNumerics +func.func @checkNumerics(%arg0: tensor<1xf32>) -> tensor<1xf32> { + // CHECK-NEXT: return %arg0 : tensor<1xf32> + %0 = "tf.CheckNumerics"(%arg0) {message = "check numerics"} : (tensor<1xf32>) -> tensor<1xf32> + func.return %0: tensor<1xf32> +} + +//===----------------------------------------------------------------------===// +// InfeedDequeueTuple legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @infeed_dequeue_tuple +func.func @infeed_dequeue_tuple() -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>) { +// CHECK: [[TOKEN:%.*]] = mhlo.create_token : !mhlo.token +// CHECK: [[INFEED:%.*]]:3 = "mhlo.infeed"([[TOKEN]]) <{infeed_config = ""{{.*}}}> : (!mhlo.token) -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>, !mhlo.token) +// CHECK: return [[INFEED]]#0, [[INFEED]]#1 + %0:2 = "tf.InfeedDequeueTuple"() : () -> (tensor<1x8x4x4xi32>, tensor<1x100x1xf32>) + func.return %0#0, %0#1 : tensor<1x8x4x4xi32>, tensor<1x100x1xf32> +} + +// ----- + +// CHECK-LABEL: func @infeed_dequeue_tuple_dynamic_error +func.func @infeed_dequeue_tuple_dynamic_error() -> (tensor<3x3xf32>, tensor<4x?xf32>) { + // We expect legalization to fail for dynamic shapes: + // CHECK: [[INFEED:%.*]] = "tf.InfeedDequeueTuple"{{.*}} + %0:2 = "tf.InfeedDequeueTuple"() : () -> (tensor<3x3xf32>, tensor<4x?xf32>) + func.return %0#0, %0#1 : tensor<3x3xf32>, tensor<4x?xf32> +} + +// The following op sharding is used: +// Proto debug string: +// type: TUPLE +// tuple_shardings { +// type: MAXIMAL +// tile_assignment_dimensions: 1 +// tile_assignment_devices: 0 +// } +// Serialized string: +// "\08\02*\08\08\01\1A\01\01\22\01\00" + +// CHECK-LABEL: infeed_dequeue_tuple_sharding +func.func @infeed_dequeue_tuple_sharding() -> tensor<8xi32> { + // CHECK: "mhlo.infeed" + // An additional sharding is added at the end to account for token result. + // Proto debug string: + // type: TUPLE + // tuple_shardings { + // type: MAXIMAL + // tile_assignment_dimensions: 1 + // tile_assignment_devices: 0 + // } + // tuple_shardings { + // type: MAXIMAL + // tile_assignment_dimensions: 1 + // tile_assignment_devices: 0 + // } + // CHECK-SAME: mhlo.sharding = "\08\02*\08\08\01\1A\01\01\22\01\00*\08\08\01\1A\01\01\22\01\00" + %0 = "tf.InfeedDequeueTuple"() {_XlaSharding = "\08\02*\08\08\01\1A\01\01\22\01\00"} : () -> tensor<8xi32> + func.return %0 : tensor<8xi32> +} + +//===----------------------------------------------------------------------===// +// Nullary op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @const +func.func @const() -> tensor<2xi32> { + // CHECK: mhlo.constant dense<0> : tensor<2xi32> + %0 = "tf.Const"() {device = "", name = "", dtype = "tfdtype$DT_INT32", value = dense<0> : tensor<2xi32>} : () -> (tensor<2xi32>) + func.return %0: tensor<2xi32> +} + +// ----- + +// CHECK-LABEL: @const_dynamic_output +func.func @const_dynamic_output() -> tensor<*xi32> { + // CHECK: [[CONST:%.*]] = mhlo.constant dense<0> : tensor<2xi32> + // CHECK: [[CAST:%.*]] = tensor.cast [[CONST]] : tensor<2xi32> to tensor<*xi32> + %0 = "tf.Const"() {value = dense<0> : tensor<2xi32>} : () -> (tensor<*xi32>) + // CHECK: return [[CAST]] + func.return %0: tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: @opaque_const +func.func @opaque_const() -> tensor>> { + // CHECK-NOT: mhlo.constant + %0 = "tf.Const"() {device = "", name = "", dtype = "tfdtype$DT_INT32", value = #tf_type : tensor} : () -> tensor>> + func.return %0 : tensor>> +} + +//===----------------------------------------------------------------------===// +// Matmul op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: matmul_notranspose +// CHECK-SAME: (%[[A:.*]]: tensor<5x7xf32>, %[[B:.*]]: tensor<7x11xf32>) +func.func @matmul_notranspose(%a: tensor<5x7xf32>, %b: tensor<7x11xf32>) -> tensor<5x11xf32> { + // CHECK: "mhlo.dot"(%[[A]], %[[B]]) + %0 = "tf.MatMul"(%a, %b) {transpose_a = false, transpose_b = false} : (tensor<5x7xf32>, tensor<7x11xf32>) -> tensor<5x11xf32> + + func.return %0 : tensor<5x11xf32> +} + +// ----- + +// CHECK-LABEL: matmul_transpose_b +// CHECK-SAME: (%[[A:.*]]: tensor<5x7xf32>, %[[B:.*]]: tensor<11x7xf32>) +func.func @matmul_transpose_b(%a: tensor<5x7xf32>, %b: tensor<11x7xf32>) -> tensor<5x11xf32> { + // CHECK: %[[UPDATED_B:.*]] = "mhlo.transpose"(%[[B]]) <{permutation = dense<[1, 0]> : tensor<2xi64>}> + // CHECK: "mhlo.dot"(%[[A]], %[[UPDATED_B]]) + %0 = "tf.MatMul"(%a, %b) {transpose_a = false, transpose_b = true} : (tensor<5x7xf32>, tensor<11x7xf32>) -> tensor<5x11xf32> + + func.return %0 : tensor<5x11xf32> +} + +// ----- + +// CHECK-LABEL: matmul_transpose_both +// CHECK-SAME: (%[[A:.*]]: tensor<7x5xf32>, %[[B:.*]]: tensor<11x7xf32>) +func.func @matmul_transpose_both(%a: tensor<7x5xf32>, %b: tensor<11x7xf32>) -> tensor<5x11xf32> { + // CHECK: %[[UPDATED_A:.*]] = "mhlo.transpose"(%[[A]]) <{permutation = dense<[1, 0]> : tensor<2xi64>}> + // CHECK: %[[UPDATED_B:.*]] = "mhlo.transpose"(%[[B]]) <{permutation = dense<[1, 0]> : tensor<2xi64>}> + // CHECK: "mhlo.dot"(%[[UPDATED_A]], %[[UPDATED_B]]) + %0 = "tf.MatMul"(%a, %b) {transpose_a = true, transpose_b = true} : (tensor<7x5xf32>, tensor<11x7xf32>) -> tensor<5x11xf32> + + func.return %0 : tensor<5x11xf32> +} + +// Verify that MatMul with ranked inputs are lowered to HLO. +// CHECK-LABEL: matmul_ranked +func.func @matmul_ranked(%a: tensor, %b: tensor<7x?xf32>) -> tensor { + // CHECK: "mhlo.dot" + %0 = "tf.MatMul"(%a, %b) {transpose_a = false, transpose_b = false} : (tensor, tensor<7x?xf32>) -> tensor + + func.return %0 : tensor +} + +// Verify SparseMatMul is legalized to dot. +// CHECK-LABEL: test_sparse_mat_mul +func.func @test_sparse_mat_mul(%arg0: tensor<3x4xf32>, %arg1: tensor<4x5xf32>) -> tensor<3x5xf32> { + // CHECK: "mhlo.dot" + %0 = "tf.SparseMatMul"(%arg0, %arg1) {a_is_sparse = true, b_is_sparse = false, transpose_a = false, transpose_b = false} : (tensor<3x4xf32>, tensor<4x5xf32>) -> tensor<3x5xf32> + func.return %0: tensor<3x5xf32> +} + +// SparseMatMul where one operand needs to be transposed and the other one not. +// +// CHECK-LABEL: @test_sparse_mat_mul_with_transpose +// CHECK-SAME: %[[ARG0:.*]]: tensor<3x4xf32> +// CHECK-SAME: %[[ARG1:.*]]: tensor<5x4xf32> +// CHECK-SAME: -> tensor<3x5xf32> +// CHECK: %[[TRANSPOSE:.*]] = "mhlo.transpose"(%[[ARG1]]) +// CHECK-SAME: permutation = dense<[1, 0]> +// CHECK-SAME: -> tensor<4x5xf32> +// CHECK: %[[RESULT:.*]] = "mhlo.dot"(%[[ARG0]], %[[TRANSPOSE]]) +// CHECK-SAME: -> tensor<3x5xf32> +// CHECK: return %[[RESULT]] +func.func @test_sparse_mat_mul_with_transpose(%arg0: tensor<3x4xf32>, %arg1: tensor<5x4xf32>) -> tensor<3x5xf32> { + %0 = "tf.SparseMatMul"(%arg0, %arg1) {a_is_sparse = true, b_is_sparse = false, transpose_a = false, transpose_b = true} : (tensor<3x4xf32>, tensor<5x4xf32>) -> tensor<3x5xf32> + func.return %0: tensor<3x5xf32> +} + +// SparseMatMul where one operand needs to be casted and the other one not. +// +// CHECK-LABEL: @test_sparse_mat_mul_with_cast +// CHECK-SAME: %[[ARG0:.*]]: tensor<3x4xf32> +// CHECK-SAME: %[[ARG1:.*]]: tensor<4x5xbf16> +// CHECK-SAME: -> tensor<3x5xf32> +// CHECK: %[[CAST:.*]] = mhlo.convert %[[ARG1]] +// CHECK-SAME: -> tensor<4x5xf32> +// CHECK: %[[RESULT:.*]] = "mhlo.dot"(%[[ARG0]], %[[CAST]]) +// CHECK-SAME: -> tensor<3x5xf32> +// CHECK: return %[[RESULT]] +func.func @test_sparse_mat_mul_with_cast(%arg0: tensor<3x4xf32>, %arg1: tensor<4x5xbf16>) -> tensor<3x5xf32> { + %0 = "tf.SparseMatMul"(%arg0, %arg1) {a_is_sparse = true, b_is_sparse = false, transpose_a = false, transpose_b = false} : (tensor<3x4xf32>, tensor<4x5xbf16>) -> tensor<3x5xf32> + func.return %0: tensor<3x5xf32> +} + +//===----------------------------------------------------------------------===// +// MaxPool op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: maxpool_valid_padding +// CHECK-SAME: %[[ARG:.*]]: tensor +func.func @maxpool_valid_padding(%arg0: tensor<2x12x20x7xi32>) -> tensor<2x3x5x7xi32> { + // CHECK: %[[INIT:.*]] = mhlo.constant dense<-2147483648> : tensor + // CHECK: "mhlo.reduce_window"(%[[ARG]], %[[INIT]]) + // CHECK: <{window_dimensions = dense<[1, 2, 2, 1]> : tensor<4xi64>, window_strides = dense<[1, 4, 4, 1]> : tensor<4xi64>}> + // CHECK: mhlo.maximum + // CHECK: mhlo.return + + %0 = "tf.MaxPool"(%arg0) {data_format = "NHWC", ksize = [1, 2, 2, 1], padding = "VALID", strides = [1, 4, 4, 1]} : (tensor<2x12x20x7xi32>) -> tensor<2x3x5x7xi32> + func.return %0 : tensor<2x3x5x7xi32> +} + +// ----- + +// CHECK-LABEL: maxpool_same_padding +// CHECK-SAME: %[[ARG:.*]]: tensor +func.func @maxpool_same_padding(%arg0: tensor<2x13x25x7xi32>) -> tensor<2x4x7x7xi32> { + // CHECK: padding = dense<{{\[\[}}0, 0], [0, 1], [1, 1], [0, 0]]> : tensor<4x2xi64> + + %0 = "tf.MaxPool"(%arg0) {data_format = "NHWC", ksize = [1, 2, 3, 1], padding = "SAME", strides = [1, 4, 4, 1]} : (tensor<2x13x25x7xi32>) -> tensor<2x4x7x7xi32> + func.return %0 : tensor<2x4x7x7xi32> +} + +// ----- + +// CHECK-LABEL: maxpool_3d_valid_padding +// CHECK-SAME: %[[ARG:.*]]: tensor +func.func @maxpool_3d_valid_padding(%arg0: tensor<2x8x12x20x7xf32>) -> tensor<2x8x3x5x7xf32> { + // CHECK: %[[INIT:.*]] = mhlo.constant dense<0xFF800000> : tensor + // CHECK: "mhlo.reduce_window"(%[[ARG]], %[[INIT]]) + // CHECK: <{window_dimensions = dense<[1, 1, 2, 2, 1]> : tensor<5xi64>, window_strides = dense<[1, 1, 4, 4, 1]> : tensor<5xi64>}> + // CHECK: mhlo.maximum + // CHECK: mhlo.return + + %0 = "tf.MaxPool3D"(%arg0) {data_format = "NDHWC", ksize = [1, 1, 2, 2, 1], padding = "VALID", strides = [1, 1, 4, 4, 1]} : (tensor<2x8x12x20x7xf32>) -> tensor<2x8x3x5x7xf32> + func.return %0 : tensor<2x8x3x5x7xf32> +} + +// ----- + +// CHECK-LABEL: maxpool_3d_same_padding +// CHECK-SAME: %[[ARG:.*]]: tensor +func.func @maxpool_3d_same_padding(%arg0: tensor<2x8x13x25x7xf32>) -> tensor<2x8x4x7x7xf32> { + // CHECK: padding = dense<{{\[\[}}0, 0], [0, 0], [0, 1], [1, 1], [0, 0]]> : tensor<5x2xi64> + + %0 = "tf.MaxPool3D"(%arg0) {data_format = "NDHWC", ksize = [1, 1, 2, 3, 1], padding = "SAME", strides = [1, 1, 4, 4, 1]} : (tensor<2x8x13x25x7xf32>) -> tensor<2x8x4x7x7xf32> + func.return %0 : tensor<2x8x4x7x7xf32> +} + +// ----- + +// CHECK-LABEL: maxpool_explicit_padding +func.func @maxpool_explicit_padding(%arg0: tensor<2x12x20x7xi32>) -> tensor<2x3x5x7xi32> { + // CHECK: tf.MaxPool + // TODO(b/165938852): need to support explicit padding in max_pool. + + %0 = "tf.MaxPool"(%arg0) {data_format = "NHWC", ksize = [1, 2, 2, 1], padding = "EXPLICIT", strides = [1, 4, 4, 1]} : (tensor<2x12x20x7xi32>) -> tensor<2x3x5x7xi32> + func.return %0 : tensor<2x3x5x7xi32> +} + +//===----------------------------------------------------------------------===// +// MaxPoolGrad op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @max_pool_grad_valid +// CHECK-SAME: %[[INPUT:.*]]: tensor<10x24x24x64xf32>, %arg1: tensor<10x12x12x64xf32>, %[[GRAD:.*]]: tensor<10x12x12x64xf32> +func.func @max_pool_grad_valid(%orig_input: tensor<10x24x24x64xf32>, %orig_output: tensor<10x12x12x64xf32>, %grad: tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> { + // CHECK: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: %[[RESULT:.*]] = "mhlo.select_and_scatter"(%[[INPUT]], %[[GRAD]], %[[ZERO]]) <{window_dimensions = dense<[1, 2, 2, 1]> : tensor<4xi64>, window_strides = dense<[1, 2, 2, 1]> : tensor<4xi64>}> ({ + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.compare GE, %[[VALUE_A]], %[[VALUE_B]] : (tensor, tensor) -> tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }, { + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.add %[[VALUE_A]], %[[VALUE_B]] : tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }) : (tensor<10x24x24x64xf32>, tensor<10x12x12x64xf32>, tensor) -> tensor<10x24x24x64xf32> + // CHECK: return %[[RESULT]] : tensor<10x24x24x64xf32> + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + ksize = [1, 2, 2, 1], + padding = "VALID", + strides = [1, 2, 2, 1] + } : (tensor<10x24x24x64xf32>, tensor<10x12x12x64xf32>, tensor<10x12x12x64xf32>) -> tensor<10x24x24x64xf32> + func.return %result : tensor<10x24x24x64xf32> +} + +// ----- + +// CHECK-LABEL: @max_pool_3d_grad_valid +// CHECK-SAME: %[[INPUT:.*]]: tensor<10x8x24x24x64xf32>, %arg1: tensor<10x8x12x12x64xf32>, %[[GRAD:.*]]: tensor<10x8x12x12x64xf32> +func.func @max_pool_3d_grad_valid(%orig_input: tensor<10x8x24x24x64xf32>, %orig_output: tensor<10x8x12x12x64xf32>, %grad: tensor<10x8x12x12x64xf32>) -> tensor<10x8x24x24x64xf32> { + // CHECK: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: %[[RESULT:.*]] = "mhlo.select_and_scatter"(%[[INPUT]], %[[GRAD]], %[[ZERO]]) <{window_dimensions = dense<[1, 1, 2, 2, 1]> : tensor<5xi64>, window_strides = dense<[1, 1, 2, 2, 1]> : tensor<5xi64>}> ({ + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.compare GE, %[[VALUE_A]], %[[VALUE_B]] : (tensor, tensor) -> tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }, { + // CHECK: ^bb0(%[[VALUE_A:.*]]: tensor, %[[VALUE_B:.*]]: tensor): + // CHECK: %[[SELECT_RESULT:.*]] = mhlo.add %[[VALUE_A]], %[[VALUE_B]] : tensor + // CHECK: mhlo.return %[[SELECT_RESULT]] : tensor + // CHECK: }) : (tensor<10x8x24x24x64xf32>, tensor<10x8x12x12x64xf32>, tensor) -> tensor<10x8x24x24x64xf32> + // CHECK: return %[[RESULT]] : tensor<10x8x24x24x64xf32> + %result = "tf.MaxPool3DGrad"(%orig_input, %orig_output, %grad) {data_format = "NDHWC", ksize = [1, 1, 2, 2, 1], padding = "VALID", strides = [1, 1, 2, 2, 1]} : (tensor<10x8x24x24x64xf32>, tensor<10x8x12x12x64xf32>, tensor<10x8x12x12x64xf32>) -> tensor<10x8x24x24x64xf32> + func.return %result : tensor<10x8x24x24x64xf32> +} + +// ----- + +// CHECK-LABEL: @max_pool_grad_same +func.func @max_pool_grad_same(%orig_input: tensor<2x13x25x7xf32>, %orig_output: tensor<2x4x7x7xf32>, %grad: tensor<2x4x7x7xf32>) -> tensor<2x13x25x7xf32> { + // CHECK: padding = dense<{{\[\[}}0, 0], [0, 1], [1, 1], [0, 0]]> : tensor<4x2xi64> + %result = "tf.MaxPoolGrad"(%orig_input, %orig_output, %grad) { + data_format = "NHWC", + ksize = [1, 2, 3, 1], + padding = "SAME", + strides = [1, 4, 4, 1] + } : (tensor<2x13x25x7xf32>, tensor<2x4x7x7xf32>, tensor<2x4x7x7xf32>) -> tensor<2x13x25x7xf32> + func.return %result : tensor<2x13x25x7xf32> +} + +// ----- + +// CHECK-LABEL: @max_pool_3d_grad_same +func.func @max_pool_3d_grad_same(%orig_input: tensor<2x8x13x25x7xf32>, %orig_output: tensor<2x8x4x7x7xf32>, %grad: tensor<2x8x4x7x7xf32>) -> tensor<2x8x13x25x7xf32> { + // CHECK: padding = dense<{{\[\[}}0, 0], [0, 0], [0, 1], [1, 1], [0, 0]]> : tensor<5x2xi64> + %result = "tf.MaxPool3DGrad"(%orig_input, %orig_output, %grad) {data_format = "NDHWC", ksize = [1, 1, 2, 3, 1], padding = "SAME", strides = [1, 1, 4, 4, 1]} : (tensor<2x8x13x25x7xf32>, tensor<2x8x4x7x7xf32>, tensor<2x8x4x7x7xf32>) -> tensor<2x8x13x25x7xf32> + func.return %result : tensor<2x8x13x25x7xf32> +} + +//===----------------------------------------------------------------------===// +// OneHot op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL:one_hot +func.func @one_hot(%indices: tensor<3xi32>, %on_value: tensor, %off_value: tensor) -> tensor<3x5xf32> { + // CHECK: %[[IOTA:.*]] = "mhlo.iota"() <{iota_dimension = 1 : i64}> : () -> tensor<3x5xi32> + // CHECK: %[[BCAST_ARG0:.+]] = "mhlo.broadcast_in_dim"(%arg0) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> : (tensor<3xi32>) -> tensor<3x5xi32> + // CHECK: %[[COMPARE:.*]] = mhlo.compare EQ, %[[BCAST_ARG0]], %[[IOTA]] : (tensor<3x5xi32>, tensor<3x5xi32>) -> tensor<3x5xi1> + // CHECK: %[[ON_VALUE:.*]] = "mhlo.broadcast"(%arg1) <{broadcast_sizes = dense<[3, 5]> : tensor<2xi64>}> : (tensor) -> tensor<3x5xf32> + // CHECK: %[[OFF_VALUE:.*]] = "mhlo.broadcast"(%arg2) <{broadcast_sizes = dense<[3, 5]> : tensor<2xi64>}> : (tensor) -> tensor<3x5xf32> + // CHECK: %[[RESULT:.*]] = mhlo.select %[[COMPARE]], %[[ON_VALUE]], %[[OFF_VALUE]] : tensor<3x5xi1>, tensor<3x5xf32> + // CHECK: return %[[RESULT]] : tensor<3x5xf32> + %depth = "tf.Const"() { value = dense<5> : tensor } : () -> tensor + %result = "tf.OneHot"(%indices, %depth, %on_value, %off_value) {axis = -1 : i64} : (tensor<3xi32>, tensor, tensor, tensor) -> tensor<3x5xf32> + func.return %result : tensor<3x5xf32> +} + +//===----------------------------------------------------------------------===// +// tf.OutfeedEnqueueTuple legalization +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @outfeed_enqueue_tuple +// CHECK-SAME: [[VAL_0:%.*]]: tensor<3xi32>, [[VAL_1:%.*]]: tensor<4xf32>) +func.func @outfeed_enqueue_tuple(%data_1: tensor<3xi32>, %data_2: tensor<4xf32>) -> () { +// CHECK: [[TOKEN:%.*]] = mhlo.create_token : !mhlo.token +// CHECK: "mhlo.outfeed"([[VAL_0]], [[VAL_1]], [[TOKEN]]) <{outfeed_config = ""}> : (tensor<3xi32>, tensor<4xf32>, !mhlo.token) -> !mhlo.token + "tf.OutfeedEnqueueTuple"(%data_1, %data_2) : (tensor<3xi32>, tensor<4xf32>) -> () + func.return +} + +//===----------------------------------------------------------------------===// +// Pack op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @pack +func.func @pack(%arg0: tensor<2xi32>, %arg1: tensor<2xi32>) -> tensor<2x2xi32> { + // CHECK: mhlo.reshape {{.*}} : (tensor<2xi32>) -> tensor<1x2xi32> + // CHECK: mhlo.reshape {{.*}} : (tensor<2xi32>) -> tensor<1x2xi32> + // CHECK: "mhlo.concatenate"({{.*}}) <{dimension = 0 : i64}> : (tensor<1x2xi32>, tensor<1x2xi32>) -> tensor<2x2xi32> + + %0 = "tf.Pack"(%arg0, %arg1) : (tensor<2xi32>, tensor<2xi32>) -> tensor<2x2xi32> + func.return %0 : tensor<2x2xi32> +} + +//===----------------------------------------------------------------------===// +// PartitionedCall op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @partitioned_call +func.func @partitioned_call(%arg0: tensor) -> tensor { + // CHECK: call @pcall_func(%arg0) : (tensor) -> tensor + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + + +func.func @pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// CHECK-LABEL: func @partitioned_call_multi_input +func.func @partitioned_call_multi_input(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: call @pcall_multi_input(%arg0, %arg1) : (tensor, tensor) -> tensor + %0 = "tf.PartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @pcall_multi_input} : (tensor, tensor) -> (tensor) + func.return %0 : tensor +} + + +func.func @pcall_multi_input(%arg0: tensor, %arg1: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// CHECK-LABEL: func @partitioned_call_multi_in_out +func.func @partitioned_call_multi_in_out(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: call @pcall_multi_in_out(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + %0, %1 = "tf.PartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @pcall_multi_in_out} : (tensor, tensor) -> (tensor, tensor) + func.return %0, %1 : tensor, tensor +} + + +func.func @pcall_multi_in_out(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + func.return %arg1, %arg0 : tensor, tensor +} + +// CHECK-LABEL: func @unhandled_partitioned_call +func.func @unhandled_partitioned_call(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor, tensor) { + // The argument types don't match the parameter types for the + // pcall_multi_in_out function. That's fine for a PartitionedCallOp but not + // for a standard CallOp, so this op can't be lowered. + // CHECK: "tf.PartitionedCall" + %0, %1 = "tf.PartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @pcall_multi_in_out} : (tensor<*xi32>, tensor<*xi32>) -> (tensor, tensor) + func.return %0, %1 : tensor, tensor +} + + +// CHECK-LABEL: func @unhandled_partitioned_call_2 +func.func @unhandled_partitioned_call_2(%arg0: tensor, %arg1: tensor<*xi32>) -> (tensor, tensor) { + // CHECK: "tf.PartitionedCall" + %0, %1 = "tf.PartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @pcall_multi_in_out} : (tensor, tensor<*xi32>) -> (tensor, tensor) + func.return %0, %1 : tensor, tensor +} + +// ----- + +// CHECK-LABEL: func @no_args_and_results +func.func @no_args_and_results() { + // CHECK: call @callee() : () -> () + // CHECK: call @callee() : () -> () + // CHECK: call @callee() : () -> () + "tf.PartitionedCall"() {config = "", config_proto = "", executor_type = "", f = @callee} : () -> () + "tf.StatefulPartitionedCall"() {config = "", config_proto = "", executor_type = "", f = @callee} : () -> () + "tf.LegacyCall"() {config = "", config_proto = "", executor_type = "", f = @callee} : () -> () + func.return +} + +func.func @callee() { + func.return +} + +//===----------------------------------------------------------------------===// +// ReverseV2 op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @reverse_func_32 +func.func @reverse_func_32(%arg0: tensor<5xi32>) -> tensor<5xi32> { + %axis = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> (tensor<1xi32>) + + // CHECK: [[VAL:%.+]] = "mhlo.reverse"(%arg0) <{dimensions = dense<0> : tensor<1xi64>}> + %reversed = "tf.ReverseV2"(%arg0, %axis) : (tensor<5xi32>, tensor<1xi32>) -> tensor<5xi32> + + // CHECK: return [[VAL]] : tensor<5xi32> + func.return %reversed : tensor<5xi32> +} + +// ----- + +// CHECK-LABEL: @reverse_func_64 +func.func @reverse_func_64(%arg0: tensor<5xi32>) -> tensor<5xi32> { + %axis = "tf.Const"() {value = dense<0> : tensor<1xi64>} : () -> (tensor<1xi64>) + + // CHECK: [[VAL:%.+]] = "mhlo.reverse"(%arg0) <{dimensions = dense<0> : tensor<1xi64>}> + %reversed = "tf.ReverseV2"(%arg0, %axis) : (tensor<5xi32>, tensor<1xi64>) -> tensor<5xi32> + + // CHECK: return [[VAL]] : tensor<5xi32> + func.return %reversed : tensor<5xi32> +} + +// ----- + +// CHECK-LABEL: @reverse_func_neg +func.func @reverse_func_neg(%arg0: tensor<5x5xi32>) -> tensor<5x5xi32> { + %axis = "tf.Const"() {value = dense<[-1]> : tensor<1xi32>} : () -> (tensor<1xi32>) + + // CHECK: [[VAL:%.+]] = "mhlo.reverse"(%arg0) <{dimensions = dense<1> : tensor<1xi64>}> + %reversed = "tf.ReverseV2"(%arg0, %axis) : (tensor<5x5xi32>, tensor<1xi32>) -> tensor<5x5xi32> + + // CHECK: return [[VAL]] : tensor<5x5xi32> + func.return %reversed : tensor<5x5xi32> +} + +//===----------------------------------------------------------------------===// +// StatefulPartitionedCall op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @stateful_partitioned_call +// CHECK-SAME: [[ARG:%.+]]: tensor +func.func @stateful_partitioned_call(%arg0: tensor) -> tensor { + // CHECK: call @stateful_pcall_func([[ARG]]) : (tensor) -> tensor + %0 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @stateful_pcall_func} : (tensor) -> (tensor) + func.return %0 : tensor +} + +func.func @stateful_pcall_func(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// ----- + +// CHECK-LABEL: func @stateful_partitioned_call_multi_in_out +// CHECK-SAME: ([[ARG0:%.+]]: tensor, [[ARG1:%.+]]: tensor) +func.func @stateful_partitioned_call_multi_in_out(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK: call @stateful_pcall_multi_in_out([[ARG0]], [[ARG1]]) : (tensor, tensor) -> (tensor, tensor) + %0, %1 = "tf.StatefulPartitionedCall"(%arg0, %arg1) {config = "", config_proto = "", executor_type = "", f = @stateful_pcall_multi_in_out} : (tensor, tensor) -> (tensor, tensor) + func.return %0, %1 : tensor, tensor +} + +func.func @stateful_pcall_multi_in_out(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + func.return %arg1, %arg0 : tensor, tensor +} + +//===----------------------------------------------------------------------===// +// Elu op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @elu +func.func @elu(%arg0: tensor<1xf32>) -> tensor<1xf32> { + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor<1xf32> + // CHECK-DAG: %[[PRED:.*]] = mhlo.compare GT, %arg0, %[[ZERO]] + // CHECK-DAG: %[[EXP:.*]] = mhlo.exponential_minus_one %arg0 + // CHECK: %[[RESULT:.*]] = mhlo.select %[[PRED]], %arg0, %[[EXP]] + // CHECK: return %[[RESULT]] + %0 = "tf.Elu"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> + func.return %0: tensor<1xf32> +} + +// ----- + +// CHECK-LABEL: func @elu_grad +// CHECK-SAME: (%[[GRADIENTS:.*]]: tensor<4x8xf32>, %[[FEATURES:.*]]: tensor) +func.func @elu_grad(%gradients: tensor<4x8xf32>, %features: tensor) -> tensor<4x8xf32> { + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK-DAG: %[[ONE:.*]] = mhlo.constant dense<1.000000e+00> : tensor + // CHECK-DAG: %[[PRED:.*]] = chlo.broadcast_compare %[[FEATURES]], %[[ZERO]] {broadcast_dimensions = array, comparison_direction = #chlo} + // CHECK-DAG: %[[ADD1:.*]] = chlo.broadcast_add %[[FEATURES]], %[[ONE]] {broadcast_dimensions = array} + // CHECK-DAG: %[[MULGRAD:.*]] = mhlo.multiply %[[GRADIENTS]], %[[ADD1]] : (tensor<4x8xf32>, tensor) -> tensor<4x8xf32> + // CHECK: %[[RESULT:.*]] = mhlo.select %[[PRED]], %[[GRADIENTS]], %[[MULGRAD]] + // CHECK: return %[[RESULT]] + %2 = "tf.EluGrad"(%gradients, %features) : (tensor<4x8xf32>, tensor) -> tensor<4x8xf32> + func.return %2 : tensor<4x8xf32> +} + +//===----------------------------------------------------------------------===// +// Relu op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @relu +func.func @relu(%arg0: tensor<1xi32>) -> tensor<1xi32> { + // CHECK: %[[ZERO:.*]] = mhlo.constant dense<0> : tensor + // CHECK: chlo.broadcast_maximum %[[ZERO]], %arg0 {broadcast_dimensions = array} : (tensor, tensor<1xi32>) -> tensor<1xi32> + %0 = "tf.Relu"(%arg0) : (tensor<1xi32>) -> tensor<1xi32> + func.return %0: tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @relu_unsigned +func.func @relu_unsigned(%arg0: tensor) -> tensor { + // CHECK: %[[ZERO:.*]] = mhlo.constant dense<0> : tensor + // CHECK: chlo.broadcast_maximum %[[ZERO]], %arg0 {broadcast_dimensions = array} : (tensor, tensor) -> tensor + %0 = "tf.Relu"(%arg0) : (tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @relu6 +func.func @relu6(%arg0: tensor<1xi32>) -> tensor<1xi32> { + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: %[[SIX:.*]] = mhlo.constant dense<6> : tensor + // CHECK: mhlo.clamp %[[ZERO]], %arg0, %[[SIX]] : (tensor, tensor<1xi32>, tensor) -> tensor<1xi32> + %0 = "tf.Relu6"(%arg0) : (tensor<1xi32>) -> tensor<1xi32> + func.return %0: tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @relu6_unsigned +func.func @relu6_unsigned(%arg0: tensor) -> tensor { + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: %[[SIX:.*]] = mhlo.constant dense<6> : tensor + // CHECK: mhlo.clamp %[[ZERO]], %arg0, %[[SIX]] : (tensor, tensor, tensor) -> tensor + %0 = "tf.Relu6"(%arg0) : (tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @leaky_relu +func.func @leaky_relu(%arg0: tensor<1x4x4x3xf32>) -> tensor<1x4x4x3xf32> attributes {tf.entry_function = {}} { + // CHECK-NEXT: %[[ALPHA:.*]] = "chlo.constant_like"(%arg0) <{value = 2.000000e-01 : f32}> : (tensor<1x4x4x3xf32>) -> tensor<1x4x4x3xf32> + // CHECK-NEXT: %[[ZERO:.*]] = "chlo.constant_like"(%arg0) <{value = 0.000000e+00 : f32}> : (tensor<1x4x4x3xf32>) -> tensor<1x4x4x3xf32> + // CHECK-NEXT: %[[LEAKY:.*]] = mhlo.multiply %[[INP:.*]], %[[ALPHA]] : tensor<1x4x4x3xf32> + // CHECK-NEXT: %[[CMP:.*]] = mhlo.compare GT, %[[INP]], %[[ZERO]] : (tensor<1x4x4x3xf32>, tensor<1x4x4x3xf32>) -> tensor<1x4x4x3xi1> + // CHECK-NEXT: %[[RES:.*]] = mhlo.select %[[CMP]], %[[INP]], %[[LEAKY]] : tensor<1x4x4x3xi1>, tensor<1x4x4x3xf32> + // CHECK-NEXT: return %[[RES]] : tensor<1x4x4x3xf32> + %0 = "tf.LeakyRelu"(%arg0) {alpha = 2.000000e-01 : f32, device = ""} : (tensor<1x4x4x3xf32>) -> tensor<1x4x4x3xf32> + func.return %0 : tensor<1x4x4x3xf32> +} + +// ----- + +// CHECK-LABEL: func @leaky_relu_grad +func.func @leaky_relu_grad(%arg0: tensor<1x4x4xf32>, %arg1: tensor<1x4x4xf32>) -> tensor<1x4x4xf32> attributes {tf.entry_function = {}} { + // CHECK-NEXT: %[[ALPHA:.*]] = "chlo.constant_like"(%arg1) <{value = 2.000000e-01 : f32}> : (tensor<1x4x4xf32>) -> tensor<1x4x4xf32> + // CHECK-NEXT: %[[ZERO:.*]] = "chlo.constant_like"(%arg1) <{value = 0.000000e+00 : f32}> : (tensor<1x4x4xf32>) -> tensor<1x4x4xf32> + // CHECK-NEXT: %[[LEAKYGRAD:.*]] = mhlo.multiply %[[GRADIENT:.*]], %[[ALPHA]] : tensor<1x4x4xf32> + // CHECK-NEXT: %[[CMP:.*]] = mhlo.compare GT, %[[INP:.*]], %[[ZERO]] : (tensor<1x4x4xf32>, tensor<1x4x4xf32>) -> tensor<1x4x4xi1> + // CHECK-NEXT: %[[RES:.*]] = mhlo.select %[[CMP]], %[[GRADIENT]], %[[LEAKYGRAD]] : tensor<1x4x4xi1>, tensor<1x4x4xf32> + // CHECK-NEXT: return %[[RES]] : tensor<1x4x4xf32> + %0 = "tf.LeakyReluGrad"(%arg0, %arg1) {alpha = 2.000000e-01 : f32, device = ""} : (tensor<1x4x4xf32>, tensor<1x4x4xf32>) -> tensor<1x4x4xf32> + func.return %0 : tensor<1x4x4xf32> +} + +// ----- + +// CHECK-LABEL: func @softsign +func.func @softsign(%arg0: tensor<4x10xf32>) -> tensor<4x10xf32> { + // CHECK-NEXT: %[[ONE:.*]] = mhlo.constant dense<1.000000e+00> : tensor<4x10xf32> + // CHECK-NEXT: %[[ABS:.*]] = mhlo.abs %{{.*}} : tensor<4x10xf32> + // CHECK-NEXT: %[[ADD:.*]] = mhlo.add %[[ONE]], %[[ABS]] : tensor<4x10xf32> + // CHECK-NEXT: %[[DIV:.*]] = mhlo.divide %{{.*}}, %[[ADD]] : tensor<4x10xf32> + // CHECK-NEXT: return %[[DIV]] : tensor<4x10xf32> + %0 = "tf.Softsign"(%arg0) : (tensor<4x10xf32>) -> tensor<4x10xf32> + func.return %0 : tensor<4x10xf32> +} + +// ----- + +// CHECK-LABEL: func @softsign_grad +func.func @softsign_grad(%arg0: tensor<4x10xf32>, %arg1: tensor<4x10xf32>) -> tensor<4x10xf32> { + + // CHECK-NEXT: %[[ONE:.*]] = mhlo.constant dense<1.000000e+00> : tensor + // CHECK-NEXT: %[[ABS:.*]] = mhlo.abs %{{.*}} : tensor<4x10xf32> + // CHECK-NEXT: %[[BROADCAST_ADD:.*]] = chlo.broadcast_add %[[ONE]], %[[ABS]] {broadcast_dimensions = array} : (tensor, tensor<4x10xf32>) -> tensor<4x10xf32> + // CHECK-NEXT: %[[MUL:.*]] = mhlo.multiply %[[BROADCAST_ADD]], %[[BROADCAST_ADD]] : tensor<4x10xf32> + // CHECK-NEXT: %[[BROADCAST_DIV:.*]] = chlo.broadcast_divide %{{.*}}, %[[MUL]] : (tensor<4x10xf32>, tensor<4x10xf32>) -> tensor<4x10xf32> + // CHECK-NEXT: return %[[BROADCAST_DIV]] : tensor<4x10xf32> + %0 = "tf.SoftsignGrad"(%arg0, %arg1) : (tensor<4x10xf32>, tensor<4x10xf32>) -> tensor<4x10xf32> + func.return %0 : tensor<4x10xf32> +} + +//===----------------------------------------------------------------------===// +// Roll op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @Roll_0D +func.func @Roll_0D(%arg0: tensor<512xi32>, %shift: tensor) -> tensor<512xi32> { + %axis = "tf.Const"() {value = dense<0> : tensor} : () -> (tensor) + // CHECK-DAG: %[[ZERO:.*]] = mhlo.constant dense<0> : tensor + // CHECK-DAG: %[[AXIS_SIZE:.*]] = mhlo.constant dense<512> : tensor + // CHECK: %[[T1:.+]] = mhlo.remainder %arg1, %[[AXIS_SIZE]] : tensor + // CHECK: %[[T2:.+]] = mhlo.add %[[T1]], %[[AXIS_SIZE]] : tensor + // CHECK: %[[T3:.+]] = mhlo.remainder %[[T2]], %[[AXIS_SIZE]] : tensor + // CHECK: %[[CONCAT:.+]] = "mhlo.concatenate"(%arg0, %arg0) <{dimension = 0 : i64}> + // CHECK: %[[OFFSET:.+]] = mhlo.subtract %[[AXIS_SIZE]], %[[T3]] : tensor + // CHECK: "mhlo.dynamic_slice"(%[[CONCAT]], %[[OFFSET]]) + // CHECK-SAME: {slice_sizes = dense<512> : tensor<1xi64>} + // CHECK-SAME: (tensor<1024xi32>, tensor) -> tensor<512xi32> + %0 = "tf.Roll"(%arg0, %shift, %axis) {device = ""} : (tensor<512xi32>, tensor, tensor) -> tensor<512xi32> + func.return %0 : tensor<512xi32> +} + +//===----------------------------------------------------------------------===// +// Select op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @select_batch_static +func.func @select_batch_static(%arg0: tensor<2xi1>, %arg1: tensor<2x6x8xi32>, %arg2: tensor<2x6x8xi32>) -> tensor<2x6x8xi32> { + // CHECK: %[[BCAST:.*]] = "mhlo.dynamic_broadcast_in_dim"(%arg0, %{{.*}}) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> : (tensor<2xi1>, tensor<3xindex>) -> tensor<2x6x8xi1> + // CHECK: mhlo.select %[[BCAST]], %arg1, %arg2 + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<2xi1>, tensor<2x6x8xi32>, tensor<2x6x8xi32>) -> tensor<2x6x8xi32> + func.return %0: tensor<2x6x8xi32> +} + +// ----- + +// CHECK-LABEL: func @select_batch_static_r1 +func.func @select_batch_static_r1(%arg0: tensor, %arg1: tensor<2x6x8xi32>, %arg2: tensor<2x6x8xi32>) -> tensor<2x6x8xi32> { + // CHECK: mhlo.select %arg0, %arg1, %arg2 + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor, tensor<2x6x8xi32>, tensor<2x6x8xi32>) -> tensor<2x6x8xi32> + func.return %0: tensor<2x6x8xi32> +} + +// ----- + +// CHECK-LABEL: func @select_batch_static_all_same +func.func @select_batch_static_all_same(%arg0: tensor<2x6x8xi1>, %arg1: tensor<2x6x8xi32>, %arg2: tensor<2x6x8xi32>) -> tensor<2x6x8xi32> { + // CHECK: mhlo.select %arg0, %arg1, %arg2 + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<2x6x8xi1>, tensor<2x6x8xi32>, tensor<2x6x8xi32>) -> tensor<2x6x8xi32> + func.return %0: tensor<2x6x8xi32> +} + +// ----- + +// CHECK-LABEL: func @select_batch_dynamic_r1 +func.func @select_batch_dynamic_r1(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK-NEXT: %[[SHAPE0:.*]] = shape.shape_of %arg0 : tensor -> tensor<1xindex> + // CHECK-NEXT: %[[SHAPE1:.*]] = shape.shape_of %arg1 : tensor -> tensor<3xindex> + // CHECK-NEXT: %[[SHAPE2:.*]] = shape.shape_of %arg2 : tensor -> tensor<3xindex> + // CHECK-NEXT: %[[SHAPEEQ1:.*]] = shape.cstr_eq %[[SHAPE1]], %[[SHAPE2]] : tensor<3xindex>, tensor<3xindex> + // CHECK-NEXT: %[[C1:.*]] = arith.constant 1 : index + // CHECK-NEXT: %[[HEAD:.*]], %[[TAIL:.*]] = "shape.split_at"(%[[SHAPE1]], %[[C1]]) : (tensor<3xindex>, index) -> (tensor<1xindex>, tensor<2xindex>) + // CHECK-NEXT: %[[SHAPEEQ2:.*]] = shape.cstr_eq %[[SHAPE0]], %[[HEAD]] : tensor<1xindex>, tensor<1xindex> + // CHECK-NEXT: %[[SHAPEEQ:.*]] = shape.assuming_all %[[SHAPEEQ1]], %[[SHAPEEQ2]] + // CHECK-NEXT: %[[ASSUMING:.*]] = shape.assuming %[[SHAPEEQ]] -> (tensor) { + // CHECK-NEXT: %[[SHAPE1E:.*]] = shape.to_extent_tensor %[[SHAPE1]] : tensor<3xindex> -> tensor<3xindex> + // CHECK-NEXT: %[[BCAST:.*]] = "mhlo.dynamic_broadcast_in_dim"(%arg0, %[[SHAPE1E]]) <{broadcast_dimensions = dense<0> : tensor<1xi64>}> : (tensor, tensor<3xindex>) -> tensor + // CHECK-NEXT: %[[SELECT:.*]] = mhlo.select %[[BCAST]], %arg1, %arg2 : tensor, tensor + // CHECK-NEXT: shape.assuming_yield %[[SELECT]] : tensor + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: func @select_batch_dynamic +func.func @select_batch_dynamic(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK-NEXT: %[[SHAPE0:.*]] = shape.shape_of %arg0 : tensor -> tensor<3xindex> + // CHECK-NEXT: %[[SHAPE1:.*]] = shape.shape_of %arg1 : tensor -> tensor<3xindex> + // CHECK-NEXT: %[[SHAPE2:.*]] = shape.shape_of %arg2 : tensor -> tensor<3xindex> + // CHECK-NEXT: %[[SHAPEEQ1:.*]] = shape.cstr_eq %[[SHAPE1]], %[[SHAPE2]] : tensor<3xindex>, tensor<3xindex> + // CHECK-NEXT: %[[SHAPEEQ2:.*]] = shape.cstr_eq %[[SHAPE0]], %[[SHAPE1]] : tensor<3xindex>, tensor<3xindex> + // CHECK-NEXT: %[[SHAPEEQ:.*]] = shape.assuming_all %[[SHAPEEQ1]], %[[SHAPEEQ2]] + // CHECK-NEXT: %[[ASSUMING:.*]] = shape.assuming %[[SHAPEEQ]] -> (tensor) { + // CHECK-NEXT: %[[SELECT:.*]] = mhlo.select %arg0, %arg1, %arg2 : tensor, tensor + // CHECK-NEXT: shape.assuming_yield %[[SELECT]] : tensor + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor + func.return %0: tensor +} + +// ----- + +// CHECK-LABEL: testSelectInvalidUnranked +func.func @testSelectInvalidUnranked(%arg0: tensor<6x7xi1>, %arg1: tensor<*xf16>, %arg2: tensor<*xf16>) -> tensor<*xf16> { + // CHECK-NEXT: tf.Select + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<6x7xi1>, tensor<*xf16>, tensor<*xf16>) -> tensor<*xf16> + func.return %0: tensor<*xf16> +} + +// ----- + +// CHECK-LABEL: testSelectThenUnranked +func.func @testSelectThenUnranked(%arg0: tensor<3xi1>, %arg1: tensor<*xf16>, %arg2: tensor<3x2xf16>) -> tensor<*xf16> { + // CHECK-NEXT: tf.Select + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<3xi1>, tensor<*xf16>, tensor<3x2xf16>) -> tensor<*xf16> + func.return %0: tensor<*xf16> +} + +// ----- + +// CHECK-LABEL: testSelectElseUnranked +func.func @testSelectElseUnranked(%arg0: tensor<3xi1>, %arg1: tensor<3x2xf16>, %arg2: tensor<*xf16>) -> tensor<*xf16> { + // CHECK-NEXT: tf.Select + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<3xi1>, tensor<3x2xf16>, tensor<*xf16>) -> tensor<*xf16> + func.return %0: tensor<*xf16> +} + +// ----- + +// CHECK-LABEL: func @selectv2_dynamic_ranked +func.func @selectv2_dynamic_ranked(%arg0: tensor<1xi1>, %arg1: tensor<2x?x8xi32>, %arg2: tensor<2x8x8xi32>) -> tensor<2x?x8xi32> { + // CHECK: chlo.broadcast_select + %0 = "tf.SelectV2"(%arg0, %arg1, %arg2) : (tensor<1xi1>, tensor<2x?x8xi32>, tensor<2x8x8xi32>) -> tensor<2x?x8xi32> + func.return %0: tensor<2x?x8xi32> +} + +//===----------------------------------------------------------------------===// +// Fast Fourier Transform op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @fft_1D +func.func @fft_1D(%arg0: tensor<8xcomplex>) -> tensor<8xcomplex> { + // CHECK: "mhlo.fft"(%arg0) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<8xcomplex> + %0 = "tf.FFT"(%arg0) : (tensor<8xcomplex>) -> tensor<8xcomplex> + func.return %0 : tensor<8xcomplex> +} + +// ----- + +// CHECK-LABEL: func @ifft_1D +func.func @ifft_1D(%arg0: tensor<8xcomplex>) -> tensor<8xcomplex> { + // CHECK: "mhlo.fft"(%arg0) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<8xcomplex> + %0 = "tf.IFFT"(%arg0) : (tensor<8xcomplex>) -> tensor<8xcomplex> + func.return %0 : tensor<8xcomplex> +} + +// ----- + +// CHECK-LABEL: func @rfft_1D +func.func @rfft_1D(%arg0: tensor<8xf32>) -> tensor<5xcomplex> { + %fftlength = "tf.Const"() {value = dense<[8]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: "mhlo.fft"(%arg0) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<8xf32> + %0 = "tf.RFFT"(%arg0, %fftlength) : (tensor<8xf32>, tensor<1xi32>) -> tensor<5xcomplex> + func.return %0 : tensor<5xcomplex> +} + +// ----- + +// CHECK-LABEL: func @rfft_1D_padded +func.func @rfft_1D_padded(%arg0: tensor<7xf32>) -> tensor<5xcomplex> { + %fftlength = "tf.Const"() {value = dense<[8]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: %[[PADDED:.*]] = "mhlo.pad"(%arg0, %{{.*}}) <{edge_padding_high = dense<1> : tensor<1xi64>, edge_padding_low = dense<0> : tensor<1xi64>, interior_padding = dense<0> : tensor<1xi64>}> : (tensor<7xf32>, tensor) -> tensor<8xf32> + // CHECK: "mhlo.fft"(%[[PADDED]]) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<8xf32> + %0 = "tf.RFFT"(%arg0, %fftlength) : (tensor<7xf32>, tensor<1xi32>) -> tensor<5xcomplex> + func.return %0 : tensor<5xcomplex> +} + +// ----- + +// CHECK-LABEL: func @rfft_1D_sliced +func.func @rfft_1D_sliced(%arg0: tensor<2x9xf32>) -> tensor<2x5xcomplex> { + %fftlength = "tf.Const"() {value = dense<[8]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: %[[SLICED:.*]] = "mhlo.slice"(%arg0) <{limit_indices = dense<[2, 8]> : tensor<2xi64>, start_indices = dense<0> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>}> : (tensor<2x9xf32>) -> tensor<2x8xf32> + // CHECK: "mhlo.fft"(%[[SLICED]]) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<2x8xf32> + %0 = "tf.RFFT"(%arg0, %fftlength) : (tensor<2x9xf32>, tensor<1xi32>) -> tensor<2x5xcomplex> + func.return %0 : tensor<2x5xcomplex> +} + +// ----- + +// CHECK-LABEL: func @irfft_1D +func.func @irfft_1D(%arg0: tensor<8xcomplex>) -> tensor<8xf32> { + %fftlength = "tf.Const"() {value = dense<[8]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: %[[SLICED:.*]] = "mhlo.slice"(%arg0) <{limit_indices = dense<5> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}> : (tensor<8xcomplex>) -> tensor<5xcomplex> + // CHECK: "mhlo.fft"(%[[SLICED]]) <{fft_length = dense<8> : tensor<1xi64>, fft_type = #mhlo}> : (tensor<5xcomplex> + %0 = "tf.IRFFT"(%arg0, %fftlength) : (tensor<8xcomplex>, tensor<1xi32>) -> tensor<8xf32> + func.return %0 : tensor<8xf32> +} + +// ----- + +// CHECK-LABEL: fft_1D_dynamic +func.func @fft_1D_dynamic(%arg0: tensor>) -> tensor<8xcomplex> { + // CHECK: "tf.FFT" + %0 = "tf.FFT"(%arg0) : (tensor>) -> tensor<8xcomplex> + func.return %0 : tensor<8xcomplex> +} + +// ----- + +// CHECK-LABEL: rfft_1D_dynamic +func.func @rfft_1D_dynamic(%arg0: tensor) -> tensor<8xcomplex> { + %fftlength = "tf.Const"() {value = dense<[8]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: "tf.RFFT" + %0 = "tf.RFFT"(%arg0, %fftlength) : (tensor, tensor<1xi32>) -> tensor<8xcomplex> + func.return %0 : tensor<8xcomplex> +} + +//===----------------------------------------------------------------------===// +// Shape op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: func @shape_1D +func.func @shape_1D(%arg0: tensor) -> tensor<1xi32> { + // CHECK: [[SHAPE:%.+]] = shape.shape_of %arg0 + // CHECK: [[TENSOR:%.+]] = arith.index_cast [[SHAPE]] : tensor<1xindex> to tensor<1xi32> + %0 = "tf.Shape"(%arg0) : (tensor) -> tensor<1xi32> + + // CHECK: return [[TENSOR]] + func.return %0 : tensor<1xi32> +} + +// ----- + +// CHECK-LABEL: func @shape_2D +func.func @shape_2D(%arg0: tensor) -> tensor<2xi32> { + // CHECK: [[SHAPE:%.+]] = shape.shape_of %arg0 + // CHECK: [[TENSOR:%.+]] = arith.index_cast [[SHAPE]] : tensor<2xindex> to tensor<2xi32> + %0 = "tf.Shape"(%arg0) : (tensor) -> tensor<2xi32> + + // CHECK: return [[TENSOR]] + func.return %0 : tensor<2xi32> +} + +// ----- + +// CHECK-LABEL: func @shape_rankless +func.func @shape_rankless(%arg0: tensor<*xf32>) -> tensor { + // CHECK: [[SHAPE:%.+]] = shape.shape_of %arg0 + // CHECK: [[TENSOR:%.+]] = arith.index_cast [[SHAPE]] : tensor to tensor + %0 = "tf.Shape"(%arg0) : (tensor<*xf32>) -> tensor + + // CHECK: return [[TENSOR]] + func.return %0 : tensor +} + +//===----------------------------------------------------------------------===// +// Transpose op legalization. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @transpose_noop +func.func @transpose_noop(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> { + %permutation = "tf.Const"() {value = dense<[0, 1]> : tensor<2xi64>} : () -> (tensor<2xi64>) + // CHECK: return %arg0 + %0 = "tf.Transpose"(%arg0, %permutation) : (tensor<2x3xf32>, tensor<2xi64>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> +} + +// ----- + +// CHECK-LABEL: @transpose_2d +func.func @transpose_2d(%arg0: tensor<2x3xf32>) -> tensor<3x2xf32> { + %permutation = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi64>} : () -> (tensor<2xi64>) + // CHECK: "mhlo.transpose" + %0 = "tf.Transpose"(%arg0, %permutation) : (tensor<2x3xf32>, tensor<2xi64>) -> tensor<3x2xf32> + func.return %0 : tensor<3x2xf32> +} + +// ----- + +// CHECK-LABEL: @transpose_3d_int32 +func.func @transpose_3d_int32(%arg0: tensor<1x2x3xf32>) -> tensor<3x2x1xf32> { + %permutation = "tf.Const"() {value = dense<[2, 1, 0]> : tensor<3xi32>} : () -> (tensor<3xi32>) + // CHECK: "mhlo.transpose" + %0 = "tf.Transpose"(%arg0, %permutation) : (tensor<1x2x3xf32>, tensor<3xi32>) -> tensor<3x2x1xf32> + func.return %0 : tensor<3x2x1xf32> +} + +// ----- + +// CHECK-LABEL: @transpose_3d +func.func @transpose_3d(%arg0: tensor<1x2x3xf32>) -> tensor<3x2x1xf32> { + %permutation = "tf.Const"() {value = dense<[2, 1, 0]> : tensor<3xi64>} : () -> (tensor<3xi64>) + // CHECK: "mhlo.transpose" + %0 = "tf.Transpose"(%arg0, %permutation) : (tensor<1x2x3xf32>, tensor<3xi64>) -> tensor<3x2x1xf32> + func.return %0 : tensor<3x2x1xf32> +} + +// ----- + +// CHECK-LABEL: @transpose_dynamic_2d +func.func @transpose_dynamic_2d(%arg0: tensor) -> tensor<4x?xf32> { + %permutation = "tf.Const"() {value = dense<[1, 0]> : tensor<2xi64>} : () -> (tensor<2xi64>) + // CHECK: "mhlo.transpose" + %0 = "tf.Transpose"(%arg0, %permutation) : (tensor, tensor<2xi64>) -> tensor<4x?xf32> + func.return %0 : tensor<4x?xf32> +} + +//===----------------------------------------------------------------------===// +// Unary op legalizations. +//===----------------------------------------------------------------------===// + +// ----- + +// CHECK-LABEL: @abs +func.func @abs(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.abs %arg0 : tensor<2xf32> + %0 = "tf.Abs"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @abs_dynamic +func.func @abs_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.abs %arg0 : tensor + %0 = "tf.Abs"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @acos +// CHLO-LABEL: @acos +func.func @acos(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: chlo.acos %arg0 : tensor<2xf32> + // CHLO: %[[TEMP_0:.*]] = mhlo.constant dense<1.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_1:.*]] = mhlo.subtract %[[TEMP_0]], %arg0 : tensor<2xf32> + // CHLO: %[[TEMP_2:.*]] = mhlo.add %[[TEMP_0]], %arg0 : tensor<2xf32> + // CHLO: %[[TEMP_3:.*]] = mhlo.multiply %[[TEMP_1]], %[[TEMP_2]] : tensor<2xf32> + // CHLO: %[[TEMP_4:.*]] = mhlo.sqrt %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_5:.*]] = mhlo.atan2 %[[TEMP_4]], %arg0 : tensor<2xf32> + // CHLO: return %[[TEMP_5]] : tensor<2xf32> + %0 = "tf.Acos"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @acos_complex +// CHLO-LABEL: @acos_complex +func.func @acos_complex(%arg0: tensor<2xcomplex>) -> tensor<2xcomplex> { + // CHECK: chlo.acos + // CHLO: %[[TEMP_0:.*]] = mhlo.real %[[TEMP_arg0:.*]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_1:.*]] = mhlo.abs %[[TEMP_0]] : tensor<2xf32> + // CHLO: %[[TEMP_2:.*]] = mhlo.imag %[[TEMP_arg0:.*]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_3:.*]] = mhlo.abs %[[TEMP_2]] : tensor<2xf32> + // CHLO: %[[TEMP_4:.*]] = mhlo.maximum %[[TEMP_1]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_5:.*]] = mhlo.constant dense<3.40282347E+38> : tensor<2xf32> + // CHLO: %[[TEMP_6:.*]] = mhlo.sqrt %[[TEMP_5]] : tensor<2xf32> + // CHLO: %[[TEMP_7:.*]] = mhlo.constant dense<8.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_8:.*]] = mhlo.divide %[[TEMP_6]], %[[TEMP_7]] : tensor<2xf32> + // CHLO: %[[TEMP_9:.*]] = mhlo.compare GE, %[[TEMP_4]], %[[TEMP_8]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_10:.*]] = mhlo.constant dense<1.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_11:.*]] = mhlo.compare LE, %[[TEMP_1]], %[[TEMP_10]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_12:.*]] = mhlo.constant dense<5.000000e-01> : tensor<2xf32> + // CHLO: %[[TEMP_13:.*]] = mhlo.add %[[TEMP_1]], %[[TEMP_10]] : tensor<2xf32> + // CHLO: %[[TEMP_14:.*]] = mhlo.abs %[[TEMP_13]] : tensor<2xf32> + // CHLO: %[[TEMP_15:.*]] = mhlo.maximum %[[TEMP_14]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_16:.*]] = mhlo.minimum %[[TEMP_14]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_17:.*]] = mhlo.compare EQ, %[[TEMP_15]], %[[TEMP_16]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_18:.*]] = mhlo.constant dense<1.41421354> : tensor<2xf32> + // CHLO: %[[TEMP_19:.*]] = mhlo.multiply %[[TEMP_18]], %[[TEMP_15]] : tensor<2xf32> + // CHLO: %[[TEMP_20:.*]] = mhlo.divide %[[TEMP_16]], %[[TEMP_15]] : tensor<2xf32> + // CHLO: %[[TEMP_21:.*]] = mhlo.multiply %[[TEMP_20]], %[[TEMP_20]] : tensor<2xf32> + // CHLO: %[[TEMP_22:.*]] = mhlo.add %[[TEMP_10]], %[[TEMP_21]] : tensor<2xf32> + // CHLO: %[[TEMP_23:.*]] = mhlo.sqrt %[[TEMP_22]] : tensor<2xf32> + // CHLO: %[[TEMP_24:.*]] = mhlo.compare EQ, %[[TEMP_23]], %[[TEMP_10]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_25:.*]] = mhlo.constant dense<0.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_26:.*]] = mhlo.compare GT, %[[TEMP_21]], %[[TEMP_25]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_27:.*]] = mhlo.and %[[TEMP_24]], %[[TEMP_26]] : tensor<2xi1> + // CHLO: %[[TEMP_28:.*]] = mhlo.multiply %[[TEMP_15]], %[[TEMP_21]] : tensor<2xf32> + // CHLO: %[[TEMP_29:.*]] = mhlo.constant dense<2.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_30:.*]] = mhlo.divide %[[TEMP_28]], %[[TEMP_29]] : tensor<2xf32> + // CHLO: %[[TEMP_31:.*]] = mhlo.add %[[TEMP_15]], %[[TEMP_30]] : tensor<2xf32> + // CHLO: %[[TEMP_32:.*]] = mhlo.multiply %[[TEMP_15]], %[[TEMP_23]] : tensor<2xf32> + // CHLO: %[[TEMP_33:.*]] = mhlo.select %[[TEMP_27]], %[[TEMP_31]], %[[TEMP_32]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_34:.*]] = mhlo.select %[[TEMP_17]], %[[TEMP_19]], %[[TEMP_33]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_35:.*]] = mhlo.subtract %[[TEMP_1]], %[[TEMP_10]] : tensor<2xf32> + // CHLO: %[[TEMP_36:.*]] = mhlo.abs %[[TEMP_35]] : tensor<2xf32> + // CHLO: %[[TEMP_37:.*]] = mhlo.maximum %[[TEMP_36]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_38:.*]] = mhlo.minimum %[[TEMP_36]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_39:.*]] = mhlo.compare EQ, %[[TEMP_37]], %[[TEMP_38]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_40:.*]] = mhlo.multiply %[[TEMP_18]], %[[TEMP_37]] : tensor<2xf32> + // CHLO: %[[TEMP_41:.*]] = mhlo.divide %[[TEMP_38]], %[[TEMP_37]] : tensor<2xf32> + // CHLO: %[[TEMP_42:.*]] = mhlo.multiply %[[TEMP_41]], %[[TEMP_41]] : tensor<2xf32> + // CHLO: %[[TEMP_43:.*]] = mhlo.add %[[TEMP_10]], %[[TEMP_42]] : tensor<2xf32> + // CHLO: %[[TEMP_44:.*]] = mhlo.sqrt %[[TEMP_43]] : tensor<2xf32> + // CHLO: %[[TEMP_45:.*]] = mhlo.compare EQ, %[[TEMP_44]], %[[TEMP_10]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_46:.*]] = mhlo.compare GT, %[[TEMP_42]], %[[TEMP_25]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_47:.*]] = mhlo.and %[[TEMP_45]], %[[TEMP_46]] : tensor<2xi1> + // CHLO: %[[TEMP_48:.*]] = mhlo.multiply %[[TEMP_37]], %[[TEMP_42]] : tensor<2xf32> + // CHLO: %[[TEMP_49:.*]] = mhlo.divide %[[TEMP_48]], %[[TEMP_29]] : tensor<2xf32> + // CHLO: %[[TEMP_50:.*]] = mhlo.add %[[TEMP_37]], %[[TEMP_49]] : tensor<2xf32> + // CHLO: %[[TEMP_51:.*]] = mhlo.multiply %[[TEMP_37]], %[[TEMP_44]] : tensor<2xf32> + // CHLO: %[[TEMP_52:.*]] = mhlo.select %[[TEMP_47]], %[[TEMP_50]], %[[TEMP_51]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_53:.*]] = mhlo.select %[[TEMP_39]], %[[TEMP_40]], %[[TEMP_52]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_54:.*]] = mhlo.add %[[TEMP_34]], %[[TEMP_53]] : tensor<2xf32> + // CHLO: %[[TEMP_55:.*]] = mhlo.multiply %[[TEMP_12]], %[[TEMP_54]] : tensor<2xf32> + // CHLO: %[[TEMP_56:.*]] = mhlo.add %[[TEMP_55]], %[[TEMP_1]] : tensor<2xf32> + // CHLO: %[[TEMP_57:.*]] = mhlo.multiply %[[TEMP_12]], %[[TEMP_56]] : tensor<2xf32> + // CHLO: %[[TEMP_58:.*]] = mhlo.multiply %[[TEMP_3]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_59:.*]] = mhlo.add %[[TEMP_34]], %[[TEMP_13]] : tensor<2xf32> + // CHLO: %[[TEMP_60:.*]] = mhlo.divide %[[TEMP_58]], %[[TEMP_59]] : tensor<2xf32> + // CHLO: %[[TEMP_61:.*]] = mhlo.subtract %[[TEMP_53]], %[[TEMP_35]] : tensor<2xf32> + // CHLO: %[[TEMP_62:.*]] = mhlo.add %[[TEMP_60]], %[[TEMP_61]] : tensor<2xf32> + // CHLO: %[[TEMP_63:.*]] = mhlo.multiply %[[TEMP_57]], %[[TEMP_62]] : tensor<2xf32> + // CHLO: %[[TEMP_64:.*]] = mhlo.sqrt %[[TEMP_63]] : tensor<2xf32> + // CHLO: %[[TEMP_65:.*]] = mhlo.divide %[[TEMP_57]], %[[TEMP_59]] : tensor<2xf32> + // CHLO: %[[TEMP_66:.*]] = mhlo.add %[[TEMP_53]], %[[TEMP_35]] : tensor<2xf32> + // CHLO: %[[TEMP_67:.*]] = mhlo.divide %[[TEMP_57]], %[[TEMP_66]] : tensor<2xf32> + // CHLO: %[[TEMP_68:.*]] = mhlo.add %[[TEMP_65]], %[[TEMP_67]] : tensor<2xf32> + // CHLO: %[[TEMP_69:.*]] = mhlo.sqrt %[[TEMP_68]] : tensor<2xf32> + // CHLO: %[[TEMP_70:.*]] = mhlo.multiply %[[TEMP_3]], %[[TEMP_69]] : tensor<2xf32> + // CHLO: %[[TEMP_71:.*]] = mhlo.select %[[TEMP_11]], %[[TEMP_64]], %[[TEMP_70]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_72:.*]] = mhlo.select %[[TEMP_9]], %[[TEMP_3]], %[[TEMP_71]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_73:.*]] = mhlo.constant dense<9.99999995E+11> : tensor<2xf32> + // CHLO: %[[TEMP_74:.*]] = mhlo.multiply %[[TEMP_8]], %[[TEMP_73]] : tensor<2xf32> + // CHLO: %[[TEMP_75:.*]] = mhlo.compare LT, %[[TEMP_1]], %[[TEMP_74]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_76:.*]] = mhlo.constant dense<9.99999997E-7> : tensor<2xf32> + // CHLO: %[[TEMP_77:.*]] = mhlo.multiply %[[TEMP_8]], %[[TEMP_76]] : tensor<2xf32> + // CHLO: %[[TEMP_78:.*]] = mhlo.constant dense<1.000000e+02> : tensor<2xf32> + // CHLO: %[[TEMP_79:.*]] = mhlo.multiply %[[TEMP_8]], %[[TEMP_78]] : tensor<2xf32> + // CHLO: %[[TEMP_80:.*]] = mhlo.select %[[TEMP_75]], %[[TEMP_77]], %[[TEMP_79]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_81:.*]] = mhlo.compare GE, %[[TEMP_3]], %[[TEMP_80]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_82:.*]] = mhlo.select %[[TEMP_81]], %[[TEMP_3]], %[[TEMP_1]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_83:.*]] = mhlo.select %[[TEMP_81]], %[[TEMP_80]], %[[TEMP_8]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_84:.*]] = mhlo.compare GE, %[[TEMP_82]], %[[TEMP_83]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_85:.*]] = mhlo.log %[[TEMP_29]] : tensor<2xf32> + // CHLO: %[[TEMP_86:.*]] = mhlo.log %[[TEMP_82]] : tensor<2xf32> + // CHLO: %[[TEMP_87:.*]] = mhlo.add %[[TEMP_85]], %[[TEMP_86]] : tensor<2xf32> + // CHLO: %[[TEMP_88:.*]] = mhlo.constant dense<0x7F800000> : tensor<2xf32> + // CHLO: %[[TEMP_89:.*]] = mhlo.compare EQ, %[[TEMP_3]], %[[TEMP_88]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_90:.*]] = mhlo.not %[[TEMP_89]] : tensor<2xi1> + // CHLO: %[[TEMP_91:.*]] = mhlo.and %[[TEMP_81]], %[[TEMP_90]] : tensor<2xi1> + // CHLO: %[[TEMP_92:.*]] = mhlo.divide %[[TEMP_1]], %[[TEMP_3]] : tensor<2xf32> + // CHLO: %[[TEMP_93:.*]] = mhlo.select %[[TEMP_91]], %[[TEMP_92]], %[[TEMP_25]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_94:.*]] = mhlo.multiply %[[TEMP_93]], %[[TEMP_93]] : tensor<2xf32> + // CHLO: %[[TEMP_95:.*]] = mhlo.log_plus_one %[[TEMP_94]] : tensor<2xf32> + // CHLO: %[[TEMP_96:.*]] = mhlo.multiply %[[TEMP_12]], %[[TEMP_95]] : tensor<2xf32> + // CHLO: %[[TEMP_97:.*]] = mhlo.add %[[TEMP_87]], %[[TEMP_96]] : tensor<2xf32> + // CHLO: %[[TEMP_98:.*]] = mhlo.constant dense<1.17549435E-38> : tensor<2xf32> + // CHLO: %[[TEMP_99:.*]] = mhlo.sqrt %[[TEMP_98]] : tensor<2xf32> + // CHLO: %[[TEMP_100:.*]] = mhlo.constant dense<4.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_101:.*]] = mhlo.multiply %[[TEMP_99]], %[[TEMP_100]] : tensor<2xf32> + // CHLO: %[[TEMP_102:.*]] = mhlo.compare LT, %[[TEMP_3]], %[[TEMP_101]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_103:.*]] = mhlo.compare LT, %[[TEMP_1]], %[[TEMP_10]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_104:.*]] = mhlo.and %[[TEMP_102]], %[[TEMP_103]] : tensor<2xi1> + // CHLO: %[[TEMP_105:.*]] = mhlo.multiply %[[TEMP_13]], %[[TEMP_35]] : tensor<2xf32> + // CHLO: %[[TEMP_106:.*]] = mhlo.add %[[TEMP_55]], %[[TEMP_10]] : tensor<2xf32> + // CHLO: %[[TEMP_107:.*]] = mhlo.divide %[[TEMP_105]], %[[TEMP_106]] : tensor<2xf32> + // CHLO: %[[TEMP_108:.*]] = mhlo.negate %[[TEMP_107]] : tensor<2xf32> + // CHLO: %[[TEMP_109:.*]] = mhlo.compare GE, %[[TEMP_1]], %[[TEMP_10]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_110:.*]] = mhlo.multiply %[[TEMP_12]], %[[TEMP_58]] : tensor<2xf32> + // CHLO: %[[TEMP_111:.*]] = mhlo.divide %[[TEMP_110]], %[[TEMP_59]] : tensor<2xf32> + // CHLO: %[[TEMP_112:.*]] = mhlo.multiply %[[TEMP_12]], %[[TEMP_66]] : tensor<2xf32> + // CHLO: %[[TEMP_113:.*]] = mhlo.add %[[TEMP_111]], %[[TEMP_112]] : tensor<2xf32> + // CHLO: %[[TEMP_114:.*]] = mhlo.constant dense<1.500000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_115:.*]] = mhlo.compare LE, %[[TEMP_55]], %[[TEMP_114]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_116:.*]] = mhlo.divide %[[TEMP_110]], %[[TEMP_61]] : tensor<2xf32> + // CHLO: %[[TEMP_117:.*]] = mhlo.add %[[TEMP_111]], %[[TEMP_116]] : tensor<2xf32> + // CHLO: %[[TEMP_118:.*]] = mhlo.subtract %[[TEMP_55]], %[[TEMP_10]] : tensor<2xf32> + // CHLO: %[[TEMP_119:.*]] = mhlo.select %[[TEMP_115]], %[[TEMP_117]], %[[TEMP_118]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_120:.*]] = mhlo.select %[[TEMP_109]], %[[TEMP_113]], %[[TEMP_119]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_121:.*]] = mhlo.select %[[TEMP_104]], %[[TEMP_108]], %[[TEMP_120]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_122:.*]] = mhlo.multiply %[[TEMP_121]], %[[TEMP_106]] : tensor<2xf32> + // CHLO: %[[TEMP_123:.*]] = mhlo.sqrt %[[TEMP_122]] : tensor<2xf32> + // CHLO: %[[TEMP_124:.*]] = mhlo.divide %[[TEMP_3]], %[[TEMP_123]] : tensor<2xf32> + // CHLO: %[[TEMP_125:.*]] = mhlo.add %[[TEMP_121]], %[[TEMP_123]] : tensor<2xf32> + // CHLO: %[[TEMP_126:.*]] = mhlo.log_plus_one %[[TEMP_125]] : tensor<2xf32> + // CHLO: %[[TEMP_127:.*]] = mhlo.select %[[TEMP_104]], %[[TEMP_124]], %[[TEMP_126]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_128:.*]] = mhlo.select %[[TEMP_84]], %[[TEMP_97]], %[[TEMP_127]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_129:.*]] = mhlo.complex %[[TEMP_72]], %[[TEMP_128]] : tensor<2xcomplex> + // CHLO: %[[TEMP_130:.*]] = mhlo.real %[[TEMP_129]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_131:.*]] = mhlo.real %[[TEMP_arg0:.*]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_132:.*]] = mhlo.atan2 %[[TEMP_130]], %[[TEMP_131]] : tensor<2xf32> + // CHLO: %[[TEMP_133:.*]] = mhlo.imag %[[TEMP_arg0:.*]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_134:.*]] = mhlo.constant dense<0.000000e+00> : tensor<2xf32> + // CHLO: %[[TEMP_135:.*]] = mhlo.compare LT, %[[TEMP_133]], %[[TEMP_134]] : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xi1> + // CHLO: %[[TEMP_136:.*]] = mhlo.imag %[[TEMP_129]] : (tensor<2xcomplex>) -> tensor<2xf32> + // CHLO: %[[TEMP_137:.*]] = mhlo.negate %[[TEMP_136]] : tensor<2xf32> + // CHLO: %[[TEMP_138:.*]] = mhlo.select %[[TEMP_135]], %[[TEMP_136]], %[[TEMP_137]] : tensor<2xi1>, tensor<2xf32> + // CHLO: %[[TEMP_139:.*]] = mhlo.complex %[[TEMP_132]], %[[TEMP_138]] : tensor<2xcomplex> + // CHLO: return %[[TEMP_139:.*]] : tensor<2xcomplex> + %0 = "tf.Acos"(%arg0) : (tensor<2xcomplex>) -> tensor<2xcomplex> + func.return %0 : tensor<2xcomplex> +} + +// ----- + +// CHECK-LABEL: @acos_dynamic +// CHLO-LABEL: @acos_dynamic +func.func @acos_dynamic(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: chlo.acos %arg0 : tensor<*xf32> + // `tf.Acos` is lowered to `chlo.constant_like` operations which can only be + // lowered further on ranked tensors. Unranked CHLO must be transformed to + // ranked code before further lowering. + // CHLO: "tf.Acos" + %0 = "tf.Acos"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// ----- + +// CHECK-LABEL: func @cast_dynamic_i2f +func.func @cast_dynamic_i2f(%arg0: tensor) -> tensor { + // CHECK: mhlo.convert %arg0 : (tensor) -> tensor + %0 = "tf.Cast"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: func @cast_i2f +func.func @cast_i2f(%arg0: tensor<2xi32>) -> tensor<2xf32> { + // CHECK: mhlo.convert %arg0 : (tensor<2xi32>) -> tensor<2xf32> + %0 = "tf.Cast"(%arg0) : (tensor<2xi32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @cast_c2f +func.func @cast_c2f(%arg0: tensor<2xcomplex>) -> tensor<2xf32> { + // CHECK: mhlo.convert %arg0 : (tensor<2xcomplex>) -> tensor<2xf32> + %0 = "tf.Cast"(%arg0) : (tensor<2xcomplex>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @ceil +func.func @ceil(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.ceil %arg0 : tensor<2xf32> + %0 = "tf.Ceil"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @ceil_dynamic +func.func @ceil_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.ceil %arg0 : tensor + %0 = "tf.Ceil"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @complex_abs +func.func @complex_abs(%arg0: tensor<2xcomplex>) -> tensor<2xf32> { + // CHECK: mhlo.abs %arg0 : (tensor<2xcomplex>) -> tensor<2xf32> + %0 = "tf.ComplexAbs"(%arg0) : (tensor<2xcomplex>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @cos +func.func @cos(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.cosine %arg0 : tensor<2xf32> + %0 = "tf.Cos"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @tan +func.func @tan(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.tan %arg0 : tensor<2xf32> + %0 = "tf.Tan"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @cos_dynamic +func.func @cos_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.cosine %arg0 : tensor + %0 = "tf.Cos"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @exp +func.func @exp(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.exponential %arg0 : tensor<2xf32> + %0 = "tf.Exp"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @expm1 +func.func @expm1(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.exponential_minus_one %arg0 : tensor<2xf32> + %0 = "tf.Expm1"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @exp_dynamic +func.func @exp_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.exponential %arg0 : tensor + %0 = "tf.Exp"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @floor +func.func @floor(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.floor %arg0 : tensor<2xf32> + %0 = "tf.Floor"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @floor_dynamic +func.func @floor_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.floor %arg0 : tensor + %0 = "tf.Floor"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @is_finite +func.func @is_finite(%arg0: tensor<2xf32>) -> tensor<2xi1> { + // CHECK: mhlo.is_finite %arg0 : (tensor<2xf32>) -> tensor<2xi1> + %0 = "tf.IsFinite"(%arg0) : (tensor<2xf32>) -> tensor<2xi1> + func.return %0 : tensor<2xi1> +} + +// ----- + +// CHECK-LABEL: func @is_finite_dynamic +func.func @is_finite_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.is_finite %arg0 : (tensor) -> tensor + %0 = "tf.IsFinite"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @log +func.func @log(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.log %arg0 : tensor<2xf32> + %0 = "tf.Log"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @log_dynamic +func.func @log_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.log %arg0 : tensor + %0 = "tf.Log"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @log1p +func.func @log1p(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.log_plus_one %arg0 : tensor<2xf32> + %0 = "tf.Log1p"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @log1p_dynamic +func.func @log1p_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.log_plus_one %arg0 : tensor + %0 = "tf.Log1p"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @neg +func.func @neg(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.negate %arg0 : tensor<2xf32> + %0 = "tf.Neg"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: func @neg_dynamic +func.func @neg_dynamic(%arg0: tensor) -> tensor { + // CHECK: mhlo.negate %arg0 : tensor + %0 = "tf.Neg"(%arg0) : (tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @sigmoid +func.func @sigmoid(%arg0: tensor<2xf32>) -> tensor<2xf32> { + // CHECK: mhlo.logistic + %0 = "tf.Sigmoid"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @sigmoid_complex +func.func @sigmoid_complex(%arg0: tensor<2xcomplex>) -> tensor<2xcomplex> { + // CHECK: mhlo.logistic + %0 = "tf.Sigmoid"(%arg0) : (tensor<2xcomplex>) -> tensor<2xcomplex> + func.return %0 : tensor<2xcomplex> +} + +// ----- + +// CHECK-LABEL: @sigmoid_grad +func.func @sigmoid_grad(%arg0: tensor<2xf32>, %arg1: tensor<2xf32>) -> tensor<2xf32> { + // CHECK-DAG: [[MUL0:%.+]] = mhlo.multiply %arg1, %arg0 : tensor<2xf32> + // CHECK-DAG: [[ONE:%.+]] = mhlo.constant dense<1.000000e+00> : tensor<2xf32> + // CHECK-DAG: [[SUB:%.+]] = mhlo.subtract [[ONE]], %arg0 : tensor<2xf32> + // CHECK-DAG: [[MUL1:%.+]] = mhlo.multiply [[MUL0]], [[SUB]] : tensor<2xf32> + // CHECK: return [[MUL1]] + %0 = "tf.SigmoidGrad"(%arg0, %arg1) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> +} + +// ----- + +// CHECK-LABEL: @sigmoid_grad_complex +func.func @sigmoid_grad_complex(%arg0: tensor<2xcomplex>, %arg1: tensor<2xcomplex>) -> tensor<2xcomplex> { + // CHECK-DAG: [[MUL0:%.+]] = mhlo.multiply %arg1, %arg0 : tensor<2xcomplex> + // CHECK-DAG: [[ONE:%.+]] = mhlo.constant dense<(1.000000e+00,0.000000e+00)> : tensor<2xcomplex> + // CHECK-DAG: [[SUB:%.+]] = mhlo.subtract [[ONE]], %arg0 : tensor<2xcomplex> + // CHECK-DAG: [[MUL1:%.+]] = mhlo.multiply [[MUL0]], [[SUB]] : tensor<2xcomplex> + // CHECK: return [[MUL1]] + %0 = "tf.SigmoidGrad"(%arg0, %arg1) : (tensor<2xcomplex>, tensor<2xcomplex>) -> tensor<2xcomplex> + func.return %0 : tensor<2xcomplex> +} + +// ----- + +// CHECK-LABEL: @sigmoid_grad_dynamic +func.func @sigmoid_grad_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: chlo.broadcast_multiply {{.*}} : (tensor, tensor) -> tensor + // CHECK: chlo.broadcast_subtract {{.*}} {broadcast_dimensions = array} : (tensor, tensor) -> tensor + // CHECK: chlo.broadcast_multiply {{.*}} : (tensor, tensor) -> tensor + %0 = "tf.SigmoidGrad"(%arg0, %arg1) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @func_xla_sharding_without_sharding_attribute +func.func @func_xla_sharding_without_sharding_attribute(%arg0: tensor<*xi32>) -> (tensor<*xi32>) { + // Not legalized due to missing sharding attribute. + // CHECK: tf.XlaSharding + %0 = "tf.XlaSharding"(%arg0) : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %1 : tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: @func_xla_sharding_inconsistent +func.func @func_xla_sharding_inconsistent(%arg0: tensor<*xi32>) -> (tensor<*xi32>) { + // Not legalized due to inconsistent sharding attributes. + // CHECK: tf.XlaSharding + // We can't use the OpSharding proto binary string because the ending 0 in the string is dropped by C++. + // _XlaSharding={devices=[2,1]1,0} binary "\010\003\032\002\002\001\"\002\001\000" + // _XlaShardingV2={devices=[2,1]<=[2]} binary "\010\003\032\002\002\001J\001\002R\001\000" + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "{devices=[2,1]1,0}", _XlaShardingV2 = "{devices=[2,1]<=[2]}"} : (tensor<*xi32>) -> tensor<*xi32> + %1 = "tf.A"(%0) : (tensor<*xi32>) -> (tensor<*xi32>) + func.return %1 : tensor<*xi32> +} + +// ----- + +// CHECK-LABEL: @func_xla_sharding_consistent +func.func @func_xla_sharding_consistent(%arg0: tensor<4x8xi32>) -> (tensor<4x8xi32>) { + // Legalized. + // CHECK: mhlo.custom_call @Sharding(%arg0) {mhlo.sharding = "{devices=[2,1]<=[2]}"} + // _XlaSharding={devices=[2,1]0,1} binary "\010\003\032\002\002\001\"\002\000\001" + // _XlaShardingV2={devices=[2,1]<=[2]} binary "\010\003\032\002\002\001J\001\002R\001\000" + %0 = "tf.XlaSharding"(%arg0) { _XlaSharding = "{devices=[2,1]0,1}", _XlaShardingV2 = "{devices=[2,1]<=[2]}"} : (tensor<4x8xi32>) -> tensor<4x8xi32> + %1 = "tf.A"(%0) : (tensor<4x8xi32>) -> (tensor<4x8xi32>) + func.return %1 : tensor<4x8xi32> +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..46d01abb951f4eecb91a8f74ff94ca3bc2c0db69 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/BUILD @@ -0,0 +1,60 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "graph_to_tf_executor_registration", + srcs = [ + "graph_to_tf_executor_registration.cc", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/translate:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow/translate/tools:file_tf_mlir_translate", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/compiler/mlir/tools:translate_cl_options", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TranslateLib", + "@tsl//tsl/platform:protobuf", + "@xla//xla/client:client_library", + "@xla//xla/client:compile_only_client", + "@xla//xla/service/cpu:cpu_compiler", + "@xla//xla/service/cpu:cpu_transfer_manager", + "@xla//xla/stream_executor:platform_manager", + "@xla//xla/stream_executor/host:host_platform", + "@xla//xla/stream_executor/host:host_platform_id", + "@xla//xla/tsl/platform:status", + ], + alwayslink = 1, +) + +tf_cc_test( + name = "graph_to_tf_executor_registration_test", + size = "small", + srcs = ["graph_to_tf_executor_registration_test.cc"], + deps = [ + ":graph_to_tf_executor_registration", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TranslateLib", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration.cc new file mode 100644 index 0000000000000000000000000000000000000000..f37ce160d6bcd5e00e2ddbfcb19cf402813f6756 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration.cc @@ -0,0 +1,193 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Tools/mlir-translate/Translation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/tools/file_tf_mlir_translate.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" +#include "tensorflow/compiler/mlir/tools/tf_mlir_translate_cl.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/client/client_library.h" +#include "xla/client/compile_only_client.h" +#include "xla/stream_executor/host/host_platform_id.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/types.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +using tsl::Status; + +static constexpr char kMlirToGraphCompilationCheckName[] = + "mlir-to-graph-compilation-check"; +// Use CPU arbitrarily in order to check that a graph compiles at all +static constexpr char kArbitraryDeviceName[] = "XLA_CPU_JIT"; + +static Status CompileGraph(tensorflow::Graph* graph, + xla::CompileOnlyClient* client) { + if (!graph || !client) { + return Status(absl::StatusCode::kInvalidArgument, + "Invalid graph or client"); + } + + tensorflow::FunctionDefLibrary flib; + auto flib_def = std::make_unique( + tensorflow::OpRegistry::Global(), flib); + + tensorflow::XlaCompiler::Options options; + options.device_type = tensorflow::DeviceType(kArbitraryDeviceName); + options.client = client; + options.flib_def = flib_def.get(); + tensorflow::XlaCompiler compiler(options); + + std::unique_ptr graph_copy = + std::make_unique(tensorflow::OpRegistry::Global()); + tensorflow::CopyGraph(*graph, graph_copy.get()); + + tensorflow::XlaCompiler::CompileOptions compile_options; + tensorflow::XlaCompiler::CompilationResult result; + return compiler.CompileGraph(compile_options, + kMlirToGraphCompilationCheckName, + std::move(graph_copy), {}, &result); +} + +static mlir::OwningOpRef GraphdefToMlirTranslateFunction( + llvm::StringRef input, mlir::MLIRContext* context) { + tensorflow::GraphdefToMlirOptions options{ + debug_info_file, xla_compile_device_type, + prune_unused_nodes, convert_legacy_fed_inputs, + graph_as_function, upgrade_legacy, + enable_shape_inference, unconditionally_use_set_output_shapes, + enable_soft_placement, set_original_tf_func_name}; + + auto module_or = tensorflow::GraphdefToMlirTranslateFunction( + input, input_arrays, input_dtypes, input_shapes, output_arrays, + control_output_arrays, options, context); + if (!module_or.status().ok()) return nullptr; + return std::move(module_or).value(); +} + +static mlir::TranslateToMLIRRegistration GraphdefToMlirTranslate( + "graphdef-to-mlir", "graphdef-to-mlir", GraphdefToMlirTranslateFunction); + +static mlir::LogicalResult MlirToGraphTranslateFunction( + mlir::ModuleOp module, llvm::raw_ostream& output) { + if (!module) return mlir::failure(); + + tensorflow::GraphExportConfig confs; + confs.export_entry_func_to_flib = export_entry_func_to_flib; + confs.export_original_tf_func_name = export_original_tf_func_name; + + std::unique_ptr flib_def; + auto graph = + std::make_unique(tensorflow::OpRegistry::Global()); + absl::flat_hash_set control_ret_nodes; + auto status = tensorflow::tf2xla::v2::ConvertTfExecutorToGraph( + module, confs, &graph, flib_def.get(), &control_ret_nodes); + if (!status.ok()) { + LOG(ERROR) << "Export to Graph failed: " << status; + return mlir::failure(); + } + + // Use Host platform, which should always exist, to make sure graphs compile. + auto platform = stream_executor::PlatformManager::PlatformWithId( + stream_executor::host::kHostPlatformId); + if (!platform.ok()) { + return mlir::failure(); + } + auto client = + xla::ClientLibrary::GetOrCreateCompileOnlyClient(platform.value()); + + tensorflow::XlaOpRegistry::RegisterCompilationKernels(); + + // Verify that the resulting graph can compile. + if (client.ok() && !CompileGraph(graph.get(), client.value()).ok()) { + return mlir::failure(); + } + + auto graphdef = std::make_unique(); + // Print the graph to the output after going through GraphDef conversion. + // The DumpGraphToFile would do this anyway so just skip straight to it. + graph->ToGraphDef(graphdef.get()); + output << tsl::LegacyUnredactedDebugString(*graphdef); + + return mlir::success(); +} + +static mlir::TranslateFromMLIRRegistration mlir_to_graph_translate( + /*name=*/"mlir-to-graph", /*description=*/"convert mlir to graph", + MlirToGraphTranslateFunction, [](mlir::DialectRegistry& registry) { + mlir::RegisterAllTensorFlowDialects(registry); + }); + +static llvm::LogicalResult MlirToGraphdefTranslateFunction( + mlir::ModuleOp module, llvm::raw_ostream& output) { + if (!module) return mlir::failure(); + + tensorflow::GraphExportConfig confs; + confs.export_entry_func_to_flib = export_entry_func_to_flib; + confs.export_original_tf_func_name = export_original_tf_func_name; + + tensorflow::FunctionLibraryDefinition flib_def( + tensorflow::OpRegistry::Global(), tensorflow::FunctionDefLibrary()); + auto graph = + std::make_unique(tensorflow::OpRegistry::Global()); + absl::flat_hash_set control_ret_nodes; + + auto status = tensorflow::tf2xla::v2::ConvertTfExecutorToGraph( + module, confs, &graph, &flib_def, &control_ret_nodes); + if (!status.ok()) { + LOG(ERROR) << "Export to Graph failed: " << status; + return mlir::failure(); + } + + tensorflow::GraphDef graphdef; + graph->ToGraphDef(&graphdef); + output << tsl::LegacyUnredactedDebugString(graphdef); + return mlir::success(); +} + +static mlir::TranslateFromMLIRRegistration mlir_to_graphdef_translate( + "mlir-to-graphdef", "mlir-to-graphdef", MlirToGraphdefTranslateFunction, + [](mlir::DialectRegistry& registry) { + mlir::RegisterAllTensorFlowDialects(registry); + }); + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..4da02ec12789a2eeb6dae22e8539b859540a6563 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/registration/graph_to_tf_executor_registration_test.cc @@ -0,0 +1,98 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Tools/mlir-translate/Translation.h" // from @llvm-project +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace tf2xla { +namespace v2 { +namespace testing { + +using mlir::LogicalResult; +using mlir::StringRef; +using mlir::Translation; +using mlir::TranslationParser; + +class MlirTranslationTest : public ::testing::Test { + private: + static constexpr char kMlirToGraphFlag[] = "-mlir-to-graph"; + + public: + MlirTranslationTest() : translation_(RegisterTranslation()) { + // Create fake command line args so that the parser gets chosen. + std::vector argv = {""}; + argv.push_back(kMlirToGraphFlag); + llvm::cl::ParseCommandLineOptions(argv.size(), &argv[0], + "TF MLIR translation test\n"); + } + + LogicalResult Translate(StringRef source, std::string& sink) { + auto source_manager = std::make_shared(); + auto source_buffer = llvm::MemoryBuffer::getMemBuffer(source); + source_manager->AddNewSourceBuffer(std::move(source_buffer), llvm::SMLoc()); + mlir::MLIRContext context; + llvm::raw_string_ostream os(sink); + + return (**translation_)(source_manager, os, &context); + } + + private: + llvm::cl::opt* + RegisterTranslation() { + // Can only register once per process. + static const auto requested_translation = + new llvm::cl::opt( + llvm::cl::desc("Translation to perform")); + return requested_translation; + } + llvm::cl::opt* translation_; +}; + +TEST_F(MlirTranslationTest, TranslatesMlirToGraph) { + static constexpr char kMlirSource[] = R"( +func.func @main() -> (tensor<1x2xf16>, tensor<2xf16>) { + %graph:2 = tf_executor.graph { + %0:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<1.0> : tensor<1x2xf16>} : () -> tensor<1x2xf16> loc("const1") + %1:2 = tf_executor.island wraps "tf.Const"() {device = "", dtype = "tfdtype$DT_HALF", value = dense<[1.0, 2.0]> : tensor<2xf16>} : () -> tensor<2xf16> loc("const2") + tf_executor.fetch %0#0, %1#0 : tensor<1x2xf16>, tensor<2xf16> + } + func.return %graph#0, %graph#1 : tensor<1x2xf16>, tensor<2xf16> +})"; + std::string result; + + auto status = Translate(kMlirSource, result); + + ASSERT_TRUE(status.succeeded()); + EXPECT_TRUE(absl::StrContains(result, "node {")); +} + +} // namespace testing +} // namespace v2 +} // namespace tf2xla +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_cpu.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_cpu.mlir new file mode 100644 index 0000000000000000000000000000000000000000..72ae819402b3ae1c3a93db5d962a3c9e788048d2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_cpu.mlir @@ -0,0 +1,12 @@ +// RUN: tf-opt "--tfxla-device-specific-transforms=device-type=XLA_CPU_JIT" -verify-diagnostics -split-input-file %s | FileCheck -dump-input=fail %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1399 : i32}} { + +// CHECK-LABEL: stateless_op +func.func @stateless_op() -> tensor { + // CHECK: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %0 = "tf.StatelessRandomGetAlg"() {device = ""} : () -> tensor + return %0 : tensor +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_gpu.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_gpu.mlir new file mode 100644 index 0000000000000000000000000000000000000000..55582849f35bfc9ccb80494a9c1234f1a5c79c46 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/tfxla_device_specific_transformations_gpu.mlir @@ -0,0 +1,12 @@ +// RUN: tf-opt "--tfxla-device-specific-transforms=device-type=XLA_GPU_JIT" -verify-diagnostics -split-input-file %s | FileCheck -dump-input=fail %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1399 : i32}} { + +// CHECK-LABEL: stateless_op +func.func @stateless_op() -> tensor { + // CHECK: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %0 = "tf.StatelessRandomGetAlg"() {device = ""} : () -> tensor + return %0 : tensor +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization-no-chlo.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization-no-chlo.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c565733f456c2667d8a8f95c62c14eaa01819a08 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization-no-chlo.mlir @@ -0,0 +1,11 @@ +// RUN: tf-opt "-tfxla-verify-legalization=legalize-chlo=false" -verify-diagnostics -split-input-file %s | FileCheck %s --dump-input=fail +// Tests the VerifyTFXLALegalization Pass, that just ensures we don't have +// any illegal ops at the end of the pipeline. This runs with +// legalize-chlo=false since errors can't be mixed with the legalize-chlo=True +// version. + +// CHECK-LABEL: allows_chlo +func.func @allows_chlo(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + %0 = "chlo.broadcast_add"(%arg0, %arg1) {broadcast_dimensions = array} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8d6daf248f0b9092b7b215b2496b52cadf2aaf58 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/tests/verify-tfxla-legalization.mlir @@ -0,0 +1,59 @@ +// RUN: tf-opt "-tfxla-verify-legalization=legalize-chlo=true" -verify-diagnostics -split-input-file %s | FileCheck -dump-input=fail %s +// Tests the VerifyTFXLALegalization Pass, that just ensures we don't have +// any illegal ops at the end of the pipeline. + +// CHECK-LABEL: allowsMHLO +func.func @allowsMHLO() -> (tensor<8x64x32x4xcomplex> {mhlo.sharding = ""}) { + %0 = mhlo.constant dense<(1.000000e+00,-1.000000e+00)> : tensor<128x32x4xcomplex> + %1 = mhlo.constant dense<(1.000000e+00,1.000000e+00)> : tensor<8x64x128xcomplex> + %2 = "mhlo.einsum"(%1, %0) <{einsum_config = "abc,cde->abde"}> : (tensor<8x64x128xcomplex>, tensor<128x32x4xcomplex>) -> tensor<8x64x32x4xcomplex> + return %2 : tensor<8x64x32x4xcomplex> +} + +// ----- + +func.func @invalid_non_mhlo() -> ( tensor<128x32x4xcomplex> {mhlo.sharding = ""}) { + // expected-error @+1 {{Could not legalize op: tf.Const}} + %cst = "tf.Const"() {value = dense<(1.000000e+00,-1.000000e+00)> : tensor<128x32x4xcomplex>} : () -> tensor<128x32x4xcomplex> + return %cst : tensor<128x32x4xcomplex> +} + +// ----- + +func.func @invalid_mixed_mhlo() -> (tensor<8x64x128xcomplex> {mhlo.sharding = ""}) { + %0 = mhlo.constant dense<(1.000000e+00,-1.000000e+00)> : tensor<128x32x4xcomplex> + // expected-error @+1 {{Could not legalize op: tf.Const}} + %cst_0 = "tf.Const"() {value = dense<(1.000000e+00,1.000000e+00)> : tensor<8x64x128xcomplex>} : () -> tensor<8x64x128xcomplex> + return %cst_0 : tensor<8x64x128xcomplex> +} + +// ----- + +func.func @fails_chlo(%arg0: tensor<1x32x10x32xi32>, %arg1: tensor<32xi32>) -> tensor<1x32x10x32xi32> { + // expected-error @+1 {{Could not legalize op: chlo.broadcast_add}} + %0 = "chlo.broadcast_add"(%arg0, %arg1) {broadcast_dimensions = array} : (tensor<1x32x10x32xi32>, tensor<32xi32>) -> tensor<1x32x10x32xi32> + func.return %0 : tensor<1x32x10x32xi32> +} + +// ----- + +func.func @multiple_failures() -> (tensor<8x64x128xcomplex> {mhlo.sharding = ""}) { + %0 = mhlo.constant dense<(1.000000e+00,-1.000000e+00)> : tensor<128x32x4xcomplex> + // expected-error @+1 {{Could not legalize op: tf.Const}} + %cst_0 = "tf.Const"() {value = dense<(1.000000e+00,1.000000e+00)> : tensor<8x64x128xcomplex>} : () -> tensor<8x64x128xcomplex> + // expected-error @+1 {{Could not legalize op: tf.XlaEinsum}} + %1 = "tf.XlaEinsum"(%cst_0, %0) {equation = "abc,cde->abde"} : (tensor<8x64x128xcomplex>, tensor<128x32x4xcomplex>) -> tensor<8x64x32x4xcomplex> + return %cst_0 : tensor<8x64x128xcomplex> +} + +// ----- + +func.func @nonstatic_shape_mhlo() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "i,j", outputs = "identity_RetVal"}} { + %0 = mhlo.constant dense<1.000000e+00> : tensor + %1 = mhlo.convert %0 : (tensor) -> tensor + %2 = mhlo.reshape %1 : (tensor) -> tensor<1xi64> + // expected-error @+1 {{Node `mhlo.dynamic_iota` must have compile-time constant}} + %3 = "mhlo.dynamic_iota"(%2) <{iota_dimension = 0 : i64}> : (tensor<1xi64>) -> tensor + %4 = mhlo.multiply %3, %3 : tensor + return %4 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..da4f5e1ea904e37674d185c3b027caa1d23b5e4c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/BUILD @@ -0,0 +1,532 @@ +# Description: +# TF2XLA Bridge transforms + +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("@xla//xla/tsl/platform:build_config_root.bzl", "if_static") +load("//tensorflow:tensorflow.bzl", "tf_cc_test") +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = ["//visibility:public"], + licenses = ["notice"], +) + +gentbl_cc_library( + name = "legalize_tf_patterns_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"generated_legalize_tf.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "legalize_tf_patterns.td", + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncTdFiles", + "@llvm-project//mlir:TensorOpsTdFiles", + "@stablehlo//:chlo_ops_td_files", + "@stablehlo//:stablehlo_ops_td_files", + "@xla//xla/mlir_hlo:hlo_ops_td_files", + ], +) + +gentbl_cc_library( + name = "xla_legalize_tf_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"xla_legalize_tf_passes.h.inc": [ + "-gen-pass-decls", + "-name=LegalizeTf", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "xla_legalize_tf_passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "tf_xla_passes_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"tf_xla_passes.h.inc": [ + "-gen-pass-decls", + "-name=TfXla", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_xla_passes.td", + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncTdFiles", + "@llvm-project//mlir:PassBaseTdFiles", + "@llvm-project//mlir:SparseTensorDialect", + "@llvm-project//mlir:TensorOpsTdFiles", + "@xla//xla/mlir_hlo:hlo_ops_td_files", + ], +) + +cc_library( + name = "tf_xla_passes", + srcs = [ + "xla_legalize_tf_passes.h.inc", + ], + hdrs = [ + "passes.h", + ], + deps = [ + ":tf_xla_passes_inc_gen", + ":xla_legalize_tf", + ":xla_legalize_tf_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:MemRefDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SparseTensorDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@xla//xla/mlir_hlo", + ], +) + +cc_library( + name = "legalize_utils", + srcs = ["utils.cc"], + hdrs = ["utils.h"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@xla//xla/mlir_hlo", + ], +) + +cc_library( + name = "test_utils", + testonly = True, + srcs = ["test_utils.cc"], + hdrs = ["test_utils.h"], + deps = [ + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "legalize_tf", + srcs = [ + "generated_legalize_tf.inc", + "legalize_tf.cc", + ], + hdrs = [ + "passes.h", + ], + # DEPRECATED: use v2/legalize_tf.h::LegalizeMlirToHlo instead. + visibility = [ + "//tensorflow/compiler/mlir/tensorflow/transforms:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/internal/passes:__pkg__", + ], + deps = [ + ":legalize_tf_patterns_inc_gen", + ":legalize_utils", + ":tf_xla_passes_inc_gen", + ":xla_legalize_tf_passes_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:dynamic_shape_utils", + "//tensorflow/compiler/mlir/tensorflow:xla_sharding_util", + "//tensorflow/core:framework", + "//tensorflow/core/kernels:conv_grad_shape_utils", + "@com_google_absl//absl/status", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:MemRefDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:chlo_ops", + "@stablehlo//:stablehlo_ops", + "@stablehlo//:stablehlo_pass_utils", + "@tsl//tsl/platform:bfloat16", + "@tsl//tsl/platform:tensor_float_32_hdr_lib", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:padding", + "@xla//xla/hlo/builder:sharding_builder", + "@xla//xla/hlo/builder/lib:conv_grad_size_util", + "@xla//xla/hlo/translate/hlo_to_mhlo:attribute_importer", + "@xla//xla/mlir_hlo", + "@xla//xla/mlir_hlo:convert_op_folder", + "@xla//xla/tsl/platform:status", + ] + if_static(["@tsl//tsl/platform:tensor_float_32_utils"]), +) + +cc_library( + name = "xla_legalize_targets", + srcs = [ + "xla_legalize_targets.cc", + ], + hdrs = [ + "xla_legalize_targets.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:chlo_ops", + "@stablehlo//:stablehlo_ops", + "@xla//xla/mlir_hlo", + ], +) + +tf_cc_test( + name = "xla_legalize_targets_test", + srcs = ["xla_legalize_targets_test.cc"], + deps = [ + ":xla_legalize_targets", + "//tensorflow/compiler/mlir/tensorflow", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:chlo_ops", + ], +) + +tf_cc_test( + name = "verify_tfxla_legalization_test", + srcs = ["verify_tfxla_legalization_test.cc"], + deps = [ + ":legalize_tf", + ":test_utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/core:test", + "//tensorflow/core/lib/monitoring:cell_reader", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "xla_legalize_tf", + srcs = [ + "infeed_ops_xla_adjust_layout.cc", + "legalize_tf_collective.cc", + "legalize_tf_communication.cc", + "tf_xla_passes.h.inc", + "tfxla_device_specific_transforms.cc", + "verify_tfxla_legalization.cc", + "xla_legalize_tf.cc", + "xla_legalize_tf_passes.h.inc", + ], + hdrs = [ + "passes.h", + ], + deps = [ + ":legalization_op_config", + ":legalize_tf", + ":legalize_utils", + ":tf_xla_passes_inc_gen", + ":xla_legalize_targets", + ":xla_legalize_tf_passes_inc_gen", + ":xla_legalize_tf_with_tf2xla", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow/transforms:lower_tf_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:set_tpu_infeed_layout", + "//tensorflow/compiler/tf2xla/kernels:rng_converter_utils", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/util/quantization:uniform_quant_ops_params", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:MemRefDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:QuantOps", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:SparseTensorDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + "@stablehlo//:base", + "@stablehlo//:chlo_ops", + "@stablehlo//:stablehlo_ops", + "@stablehlo//:stablehlo_passes", + "@xla//xla:shape_util", + "@xla//xla:side_effect_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:padding", + "@xla//xla/hlo/builder:sharding_builder", + "@xla//xla/hlo/translate/hlo_to_mhlo:attribute_importer", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/mlir_hlo", + "@xla//xla/mlir_hlo:convert_op_folder", + "@xla//xla/mlir_hlo:mhlo_passes", + "@xla//xla/mlir_hlo:type_conversion", + "@xla//xla/tpu:c_api_conversions", + "@xla//xla/tpu:tpu_api", + ], +) + +cc_library( + name = "tf2xla_rewriter", + srcs = [ + "tf2xla_rewriter.cc", + ], + hdrs = [ + "tf2xla_rewriter.h", + ], + visibility = ["//visibility:private"], + deps = [ + ":legalize_tf", + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:tpu_embedding_ops_registry", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/tf2xla:xla_compilation_device", + "//tensorflow/compiler/tf2xla:xla_context", + "//tensorflow/compiler/tf2xla:xla_expression", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:core_cpu", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:framework_types_hdr", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:allocator", + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SparseTensorDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:base", + "@stablehlo//:stablehlo_ops", + "@xla//xla:xla_data_proto_cc", + "@xla//xla:xla_proto_cc", + "@xla//xla/hlo/builder:xla_builder", + "@xla//xla/hlo/builder:xla_computation", + "@xla//xla/hlo/ir:hlo", + "@xla//xla/hlo/translate/hlo_to_mhlo:hlo_function_importer", + "@xla//xla/hlo/translate/hlo_to_mhlo:hlo_to_mlir_hlo", + "@xla//xla/hlo/translate/mhlo_to_hlo:type_to_shape", + "@xla//xla/service:hlo_proto_cc", + "@xla//xla/tsl/platform:env", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:status", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "tf2xla_rewriter_test", + srcs = [ + "tf2xla_rewriter_test.cc", + ], + deps = [ + ":test_utils", + ":tf2xla_rewriter", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/core:ops", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:stablehlo_ops", + "@xla//xla:shape_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/hlo/builder:xla_builder", + "@xla//xla/hlo/builder:xla_computation", + "@xla//xla/tsl/lib/core:status_test_util", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:status", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "xla_legalize_tf_with_tf2xla", + srcs = [ + "legalize_tf_with_tf2xla.cc", + ], + hdrs = [ + "legalize_tf_with_tf2xla_passes.h", + "passes.h", + ], + deps = [ + ":legalization_op_config", + ":tf2xla_rewriter", + ":tf_xla_passes_inc_gen", + ":xla_legalize_tf_passes_inc_gen", + "//tensorflow/compiler/mlir:op_or_arg_name_mapper", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tpu_embedding_ops_registry", + "//tensorflow/compiler/tf2xla:xla_compilation_device", + "//tensorflow/compiler/tf2xla:xla_context", + "//tensorflow/compiler/tf2xla:xla_expression", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:core_cpu_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TensorDialect", + "@llvm-project//mlir:TransformUtils", + "@stablehlo//:base", + ], +) + +tf_cc_test( + name = "xla_legalize_tf_test", + srcs = [ + "xla_legalize_tf_test.cc", + ], + deps = [ + ":tf_xla_passes", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:deserialize_mlir_module_utils", + "//tensorflow/compiler/tf2xla:xla_compilation_device", + "//tensorflow/compiler/tf2xla:xla_context", + "//tensorflow/compiler/tf2xla:xla_expression", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core/framework:allocator", + "//tensorflow/core/lib/monitoring:cell_reader", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:TransformUtils", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "legalization_op_config", + srcs = ["legalization_op_config.cc"], + hdrs = ["legalization_op_config.h"], + visibility = [ + "//tensorflow/compiler/mlir/tensorflow/transforms:__pkg__", + "//tensorflow/compiler/mlir/tf2xla/internal:__subpackages__", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tpu_embedding_ops_registry", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Support", + ], +) + +tf_cc_test( + name = "legalization_op_config_test", + srcs = ["legalization_op_config_test.cc"], + deps = [ + ":legalization_op_config", + ":legalize_tf", + "//tensorflow/compiler/jit", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/core:protos_all_cc", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "split_into_island_per_op_pass", + srcs = ["split_into_island_per_op_pass.cc"], + hdrs = [ + "split_into_island_per_op_pass.h", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_executor_inc_gen", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_pass_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:Pass", + ], +) + +tf_cc_test( + name = "split_into_island_per_op_pass_test", + srcs = ["split_into_island_per_op_pass_test.cc"], + deps = [ + ":split_into_island_per_op_pass", + "//tensorflow/compiler/mlir:register_common_dialects", + "//tensorflow/compiler/mlir/tensorflow", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/infeed_ops_xla_adjust_layout.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/infeed_ops_xla_adjust_layout.cc new file mode 100644 index 0000000000000000000000000000000000000000..af92e121ec80dc2ed9b12633056f1ab3e315193c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/infeed_ops_xla_adjust_layout.cc @@ -0,0 +1,73 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/set_tpu_infeed_layout.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/layout.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/shape.h" + +namespace mlir { +namespace mhlo { +namespace { + +#define GEN_PASS_DEF_INFEEDOPSXLAADJUSTLAYOUT +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +class InfeedsOpsXlaAdjustLayout + : public impl::InfeedOpsXlaAdjustLayoutBase { + public: + void runOnOperation() override; + + private: + static void runOnInfeedOp(::mlir::mhlo::InfeedOp op) { + OpBuilder builder(op.getContext()); + SmallVector result_types(op.getResultTypes().begin(), + op.getResultTypes().end()); + if (!op->getAttr("layout")) { + auto layout = mlir::GetTPUInfeedLayout(result_types, builder); + if (failed(layout)) return; + + op->setAttr("layout", layout.value()); + } + } +}; + +void InfeedsOpsXlaAdjustLayout::runOnOperation() { + getOperation().walk(runOnInfeedOp); +} + +} // anonymous namespace + +std::unique_ptr> +CreateInfeedsOpsXlaAdjustLayoutPass() { + return std::make_unique(); +} + +} // namespace mhlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.cc new file mode 100644 index 0000000000000000000000000000000000000000..844bab7047bc1d5c39a978ce4c696f2d0c09ccdf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.cc @@ -0,0 +1,564 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" + +#include "llvm/ADT/DenseSet.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tpu_embedding_ops_registry.h" + +namespace mlir { +namespace hlo { + +namespace { + +// Returns ops that should use MLIR legalization. +// All other ops not in this list should use XlaOpKernel. +const llvm::DenseSet& MlirAlwaysOps() { + // The static variable is a pointer in order to avoid destruction upon thread + // termination. + static const llvm::DenseSet* ops = new llvm::DenseSet< + mlir::TypeID>{ + // Ops that should always use the MLIR legalization. + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + + // MatrixDiagPartV3 should use the MLIR implementation due to performance. + TypeID::get(), + + // Ops that are legalized in the old bridge using MlirXlaOpKernel + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + + // Ops that have no XlaOpKernel. + TypeID::get(), + TypeID::get(), + + // Const op has a simple legalization and it is much more efficient to + // lower + // within MLIR. + TypeID::get(), + + // AssertOp with string types are not supported by the fallback. + TypeID::get(), + + // TF2XLA fallback pattern doesn't support these op as MLIR hlo builder + // doesn't override the necessary builder methods. These ops have simple + // lowering pattern so this should be safe. + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + + // These ops have undetermined bugs, may not be legalizable with + // XlaOpKernel + // legalization in TF2XLA fallback. By legalization with MLIR, we can fix + // the bug. b/195583695 describes the motivation of this change. + // See b/216355804 how to reproduce the bug regarding tf.RandomUniform Op + + // Conditional ops + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + return *ops; +} + +bool IsOpTypeAllowedTf2XlaFallback(const TypeID& type_id) { + // Allowlisted TensorFlow ops are known to have well behaved tf2xla kernels + // building valid MLIR using MlirHloBuilder. + // TODO(hinsu): Drop explicit allowlist when MLIR based bridge is enabled for + // all tf2xla kernels. + // Use a pointer for the static set, so the set is not destructed upon thread + // end, which would not be thread safe. + + static auto* ops = [] { + llvm::SmallDenseSet* ops_set = new llvm::SmallDenseSet< + mlir::TypeID, 512>{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + // CaseOp isn't actually supported but is enabled for testing to + // make sure ops with symbol ref attributes are filtered out. + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + // TODO(hinsu): Canonicalize QuantizeAndDequantize and + // QuantizeAndDequantizeV2 to QuantizeAndDequantizeV3 by converting + // attributes to operands. + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get< + TF::XlaSparseDenseMatmulGradWithAdagradMomentumAndCsrInputOp>(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get< + TF::XlaSparseDenseMatmulGradWithAdagradAndStaticBufferSizeOp>(), + TypeID::get< + TF::XlaSparseDenseMatmulGradWithAdagradMomentumAndStaticBufferSizeOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulGradWithAdamAndStaticBufferSizeOp>(), + TypeID::get< + TF::XlaSparseDenseMatmulGradWithFtrlAndStaticBufferSizeOp>(), + TypeID::get< + TF::XlaSparseDenseMatmulGradWithSgdAndStaticBufferSizeOp>(), // NOLINT + TypeID::get(), + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithSgdAndCsrInputOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithAdagradAndCsrInputOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithAdagradMomentumAndCsrInputOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithAdamAndCsrInputOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithFtrlAndCsrInputOp>(), // NOLINT + TypeID::get< + TF::XlaSparseDenseMatmulCustomCombinerOnTcGradWithCsrInputOp>(), + TypeID::get(), // NOLINT + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + + // Add the ops from the TPUEmbeddingOpsRegistry. + for (auto op_type_id : + TF::TPUEmbeddingOpsRegistry::Global().GetOpsTypeIds()) { + ops_set->insert(op_type_id); + } + return ops_set; + }(); + + return ops->count(type_id); +} + +/// List of ops that should use XlaOpKernel legalization only in the case of +/// prefer_tf2xla. All other ops not in this list should use MLIR legalization +/// only or not be legalized by the new bridge. +bool IsOpTypeAllowedTf2XlaPreferred(const TypeID& type_id) { + // Use a pointer for the static set, so the set is not destructed upon thread + // end, which would not be thread safe. + // clang-format off + static auto* ops = + new llvm::SmallDenseSet{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + // clang-format on + + return ops->contains(type_id); +} + +const llvm::DenseSet& DynamicTensorflowOps() { + // The static variable is a pointer in order to avoid destruction upon thread + // termination. + static const llvm::DenseSet* ops = + new llvm::DenseSet{ + TypeID::get(), + TypeID::get(), + TypeID::get(), + TypeID::get(), + }; + return *ops; +} + +} // namespace + +bool HasTf2XlaFallback(const TypeID& type_id) { + return IsOpTypeAllowedTf2XlaFallback(type_id) || + IsOpTypeAllowedTf2XlaPreferred(type_id); +} + +bool IsTypeLegalizedWithMlir(const TypeID& type_id) { + return MlirAlwaysOps().contains(type_id); +} + +bool IsOpAllowedTf2xlaFallback(const TypeID& type_id) { + return IsOpTypeAllowedTf2XlaFallback(type_id); +} + +bool IsOpAllowedTf2xlaPreferred(const TypeID& type_id) { + return IsOpTypeAllowedTf2XlaPreferred(type_id); +} + +bool IsDynamicPadderOp(const TypeID& type_id) { + return DynamicTensorflowOps().contains(type_id); +} + +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h new file mode 100644 index 0000000000000000000000000000000000000000..183f97569bf37accebcb197061db1e4f07bb6f13 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h @@ -0,0 +1,45 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZATION_OP_CONFIG_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZATION_OP_CONFIG_H_ + +#include "mlir/Support/TypeID.h" // from @llvm-project + +namespace mlir { +namespace hlo { + +// Given the type ID, check if it's legalized with MLIR. +bool IsTypeLegalizedWithMlir(const TypeID& type_id); + +// Returns true if the op is considered a dynamic padder op. +bool IsDynamicPadderOp(const TypeID& type_id); + +// Returns True if this op has a Tf2XLA fallback. Currently, this is not the +// inverse of the !IsOpLegalizedWithMlir, but it should be. +bool HasTf2XlaFallback(const TypeID& type_id); + +// Whether this type is allowed to have a TF2XLA fallback. +bool IsOpAllowedTf2xlaFallback(const TypeID& type_id); + +// Whether this type is Preferred to use a TF2XLA fallback kernel when using +// the MLIR bridge. If this is true, then the TF2XLA fallback kernel will be +// used over the MLIR lowering. +bool IsOpAllowedTf2xlaPreferred(const TypeID& type_id); + +} // namespace hlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZATION_OP_CONFIG_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..53d775993618e7a5d6630e2e0ddd63f86efe69af --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config_test.cc @@ -0,0 +1,183 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" + +#include +#include +#include +#include + +#include +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/core/framework/kernel_def.pb.h" + +namespace mlir { +namespace hlo { + +TEST(LegalizationOpConfigTest, ExpectsTrueForMlirTypeID) { + EXPECT_TRUE(IsTypeLegalizedWithMlir(TypeID::get())); + EXPECT_FALSE(HasTf2XlaFallback(TypeID::get())); + EXPECT_FALSE(IsOpAllowedTf2xlaFallback(TypeID::get())); + EXPECT_FALSE(IsOpAllowedTf2xlaPreferred(TypeID::get())); +} + +TEST(LegalizationOpConfigTest, ExpectsTrueForTF2XLATypeID) { + EXPECT_TRUE(HasTf2XlaFallback(TypeID::get())); + EXPECT_TRUE(IsOpAllowedTf2xlaPreferred(TypeID::get())); + EXPECT_FALSE(IsTypeLegalizedWithMlir(TypeID::get())); +} + +TEST(LegalizationOpConfigTest, ChecksDynamicPadderOps) { + EXPECT_TRUE( + IsDynamicPadderOp(TypeID::get())); + EXPECT_FALSE(IsDynamicPadderOp(TypeID::get())); +} + +// This test is kind of odd. We go through all the Tensorflow types and check +// whether they are legalized with MLIR, TF2XLA, or both. Ideally the sets are +// disjoint, but until that happens, this tests ensures that the set doesn't +// grow. +TEST(LegalizationOpConfigTest, CountLoweringsSet) { + int mlir_lowering_count = 0; + int tf2xla_fallback_count = 0; + int non_categorized_count = 0; + + DialectRegistry dialect_registry; + dialect_registry.insert(); + + MLIRContext context(dialect_registry); + context.loadAllAvailableDialects(); + + for (auto operation : context.getRegisteredOperations()) { + if (IsTypeLegalizedWithMlir(operation.getTypeID())) { + mlir_lowering_count++; + } else if (HasTf2XlaFallback(operation.getTypeID())) { + tf2xla_fallback_count++; + } else { + non_categorized_count++; + } + } + + // If an op moves from one lowering implementation to a different one (e.g. + // from MLIR to TF2XLA), these numbers should change. Or if TF Dialect adds + // a new op, we should expect these to change too. + EXPECT_EQ(mlir_lowering_count, 67); + EXPECT_EQ(tf2xla_fallback_count, 333); + EXPECT_EQ(non_categorized_count, 437); +} + +// Just a counter test to see which ops have duplicate lowerings. This isn't a +// correctness test versus a test to easily ensure the op registry is kept +// in sync. +TEST(LegalizationOpConfigTest, CountTypesWhichHaveBothMlirAndTf2xlaFallback) { + int double_lowering_count = 0; + + DialectRegistry dialect_registry; + dialect_registry.insert(); + + MLIRContext context(dialect_registry); + context.loadAllAvailableDialects(); + + for (auto operation : context.getRegisteredOperations()) { + if (IsTypeLegalizedWithMlir(operation.getTypeID()) && + HasTf2XlaFallback(operation.getTypeID())) { + double_lowering_count++; + } + } + + // TODO(b/288876609): This should get to zero. + EXPECT_EQ(double_lowering_count, 1); +} + +// Counts which ops have MLIR only lowerings. This isn't a +// correctness test versus a test to easily ensure the op registry is kept +// in sync. +TEST(LegalizationOpConfigTest, CountAllMlirLoweringPatterns) { + DialectRegistry dialect_registry; + mlir::RegisterCommonToolingDialects(dialect_registry); + + MLIRContext context(dialect_registry); + context.loadAllAvailableDialects(); + + RewritePatternSet mlir_legalize_lower_patterns(&context); + hlo::PopulateLegalizeTfPatterns(&context, &mlir_legalize_lower_patterns); + + int mlir_only_patterns = 0; + for (auto& pattern : mlir_legalize_lower_patterns.getNativePatterns()) { + std::optional pat_op_name = pattern->getRootKind(); + if (!pat_op_name) { + continue; + } + + if (!HasTf2XlaFallback(pat_op_name->getRegisteredInfo()->getTypeID())) { + mlir_only_patterns++; + } + } + + EXPECT_EQ(mlir_only_patterns, 63); +} + +// Counts which ops have lowerings without XlaOpKernels. This isn't a +// correctness test versus a test to easily ensure the op registry is kept +// in sync. +TEST(LegalizationOpConfigTest, MlirLoweringWithoutXlaKernel) { + tensorflow::XlaOpRegistry::RegisterCompilationKernels(); + std::vector kernel_defs = + tensorflow::XlaOpRegistry::DeviceKernels( + tensorflow::DEVICE_CPU_XLA_JIT, + /*include_compilation_only_kernels=*/true); + + std::set xla_op_kernels; + for (auto kernel_def : kernel_defs) { + std::string tf_name = "tf." + kernel_def->op(); + xla_op_kernels.insert(tf_name); + } + + DialectRegistry dialect_registry; + mlir::RegisterCommonToolingDialects(dialect_registry); + + MLIRContext context(dialect_registry); + context.loadAllAvailableDialects(); + + RewritePatternSet mlir_legalize_lower_patterns(&context); + hlo::PopulateLegalizeTfPatterns(&context, &mlir_legalize_lower_patterns); + + int mlir_without_xla_count = 0; + for (auto& pattern : mlir_legalize_lower_patterns.getNativePatterns()) { + std::optional pat_op_name = pattern->getRootKind(); + if (!pat_op_name) { + continue; + } + + if (xla_op_kernels.find(pat_op_name->getStringRef().str()) == + xla_op_kernels.end()) { + mlir_without_xla_count++; + } + } + + EXPECT_EQ(mlir_without_xla_count, 13); +} + +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..f74f5b8c1932f060a6d8e6c132719f84b47d7677 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf.cc @@ -0,0 +1,7049 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements logic for lowering TensorFlow dialect to XLA dialect. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/Dialect/Traits.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/ImplicitLocOpBuilder.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "stablehlo/transforms/PassUtils.h" // from @stablehlo // IWYU pragma: keep, legalize_tf_patterns.td +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dynamic_shape_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/xla_sharding_util.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/utils.h" +#include "xla/hlo/builder/lib/conv_grad_size_util.h" +#include "xla/hlo/builder/padding.h" +#include "xla/hlo/builder/sharding_builder.h" +#include "xla/hlo/translate/hlo_to_mhlo/attribute_importer.h" +#include "xla/mlir_hlo/utils/convert_op_folder.h" +#include "xla/mlir_hlo/utils/hlo_utils.h" +#include "xla/tsl/platform/status.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/kernel_shape_util.h" +#include "tensorflow/core/framework/rng_alg.h" +#include "tensorflow/core/kernels/conv_grad_shape_utils.h" +#include "tensorflow/core/util/padding.h" +#include "tensorflow/core/util/tensor_format.h" +#include "tsl/platform/bfloat16.h" +#include "tsl/platform/tensor_float_32_utils.h" + +namespace mlir { +namespace hlo { + +// Methods from utils.h +using mhlo::BuildReduceBody; +using mhlo::GetI64ElementsAttr; +using mhlo::GetScalarConstOfType; +using mhlo::GetScalarNegZeroOfType; + +namespace { + +constexpr char kShardingAttr[] = "mhlo.sharding"; + +/// Returns the feature dimension for the given format and input type. +static size_t GetFeatureDimension(tensorflow::TensorFormat format, + RankedTensorType input_ty) { + return GetTensorFeatureDimIndex(input_ty.getRank(), format); +} + +// Gets all integer values from the given attribute and push them to `values`. +void GetI64ArrayAttrValues(Attribute attr, SmallVectorImpl *values) { + auto array_attr = mlir::cast(attr); + values->reserve(array_attr.getValue().size()); + for (Attribute val : array_attr.getValue()) + values->push_back(mlir::cast(val).getValue().getSExtValue()); +} + +DenseI64ArrayAttr GetI64ArrayAttr(ArrayRef values, Builder *builder) { + return builder->getDenseI64ArrayAttr(values); +} + +static DenseI64ArrayAttr ToDenseI64ArrayAttr(DenseIntElementsAttr attr, + Builder *builder) { + if (!attr) return {}; + if (attr.getElementType().isInteger(64)) { + return GetI64ArrayAttr(llvm::to_vector(attr.getValues()), builder); + } + + // Requires conversion to i64 first. + std::vector values; + values.reserve(attr.getNumElements()); + for (auto value : attr.getValues()) { + values.push_back(value.getValue().getSExtValue()); + } + return GetI64ArrayAttr(values, builder); +} + +static DenseI64ArrayAttr ToDenseI64ArrayAttr(ElementsAttr attr, + Builder *builder) { + return ToDenseI64ArrayAttr( + mlir::cast( + hlo::convertElementsAttr(attr, builder->getIntegerType(64))), + builder); +} + +// Returns 1D 32-bit dense elements attribute with the given values. +static DenseIntElementsAttr GetI32ElementsAttr(ArrayRef values, + Builder *builder) { + RankedTensorType ty = tensorflow::GetTypeFromTFTensorShape( + {static_cast(values.size())}, builder->getIntegerType(32)); + return DenseIntElementsAttr::get(ty, values); +} + +// Returns a 1-d i64 elements attribute populated with numbers from start to +// end, excluding. +static DenseI64ArrayAttr GetI64ArrayAttrForSeq(int start, int end, + Builder *builder) { + int size = end - start; + + SmallVector vals; + vals.resize(size); + std::iota(vals.begin(), vals.end(), start); + return builder->getDenseI64ArrayAttr(vals); +} + +// Returns a 1-d i64 elements attribute populated with `val` repeated `size` +// times. +static DenseI64ArrayAttr GetI64ArrayAttrForValue(int size, int64_t val, + Builder *builder) { + llvm::SmallVector vals(size, val); + return builder->getDenseI64ArrayAttr(vals); +} + +// Returns the corresponding type that should be used for performing sum +// accumulation over the given input type. +Type GetSumAccumulationType(Type input_type) { + MLIRContext *ctx = input_type.getContext(); + if (input_type.isBF16() || input_type.isF16()) return Float32Type::get(ctx); + if (input_type.isSignlessInteger(8) || input_type.isSignlessInteger(16)) + return IntegerType::get(ctx, 32); + return input_type; +} + +// Returns axis in HLO format from TF elements attr with exactly one element or +// is an IntegerAttr, containing axis in the TensorFlow format. TensorFlow +// format supports negative indexing unlike HLO. +static IntegerAttr GetHLOAxisFromTFAxis(Attribute attr, int64_t rank, + Builder *b) { + IntegerAttr intAttr = mlir::dyn_cast_or_null(attr); + if (auto elementAttr = mlir::dyn_cast_or_null(attr)) { + SmallVector index(elementAttr.getShapedType().getRank(), 0); + intAttr = elementAttr.getValues()[index]; + } + + assert(intAttr && "Invalid attribute passed to GetHLOAxisFromTFAxis"); + + int64_t axis = intAttr.getInt(); + if (axis < 0) { + axis += rank; + } + return b->getI64IntegerAttr(axis); +} + +// Returns a PrecisionConfig as an array attribute based on whether TF32 +// execution is enabled +static ArrayAttr GetPrecisionConfig(Builder *builder) { + mlir::stablehlo::Precision precision = + tsl::tensor_float_32_execution_enabled() ? stablehlo::Precision::DEFAULT + : stablehlo::Precision::HIGHEST; + llvm::SmallVector attr_vec; + const int num_inputs = 2; + for (int i = 0; i < num_inputs; i++) { + attr_vec.push_back( + mlir::stablehlo::PrecisionAttr::get(builder->getContext(), precision)); + } + return builder->getArrayAttr(attr_vec); +} + +// If `value` is an IntegerAttr, returns the integer value for the HLO axis +// corresponding to the tensorflow axis. In particular, the tensorflow axis can +// be negative, in which case, the corresponding HLO axis is +// (axis + rank-of-the-tensor). +static std::optional GetIntegerHLOAxisFromTFAxis(Value value, + int64_t rank) { + DenseIntElementsAttr attrs; + if (!matchPattern(value, m_Constant(&attrs)) || + attrs.getType().getRank() != 0) { + return std::nullopt; + } + int64_t axis = attrs.getValues()[0].getInt(); + return axis < 0 ? axis + rank : axis; +} + +/// Returns a `ConvertOp` that casts the elements to a i64 type while retaining +/// the shape of the input value. +static stablehlo::ConvertOp CastValueToI64(Location loc, Value value, + PatternRewriter *rewriter) { + return stablehlo::ConvertOp::create(*rewriter, loc, value, + rewriter->getIntegerType(64)); +} + +// Creates an unpack op along the 0th dimension of the tensor. The `value` input +// must be a ranked tensor. +static TF::UnpackOp UnpackTensorAlongZeroDim(Location loc, Value value, + PatternRewriter *rewriter) { + auto indices_type = mlir::cast(value.getType()); + int num_outputs = indices_type.getShape().front(); + SmallVector unpacked_indices_type( + num_outputs, + tensorflow::GetTypeFromTFTensorShape({}, indices_type.getElementType())); + auto unpacked_indices = + TF::UnpackOp::create(*rewriter, loc, unpacked_indices_type, value, + IntegerAttr::get(rewriter->getIntegerType(64), 0)); + return unpacked_indices; +} + +// Returns size of dimension at the specified index, if ranked tensor. +// Otherwise, returns -1. +// +// Aborts if the type is ranked but doesn't have the dimension. +int64_t GetDimSize(Type ty, int64_t index) { + RankedTensorType ranked_ty = mlir::dyn_cast(ty); + if (!ranked_ty) return -1; + + return ranked_ty.getDimSize(index); +} + +template +tensorflow::TensorShape ToTensorShape(llvm::ArrayRef sizes) { + return tensorflow::TensorShape( + llvm::SmallVector(sizes.begin(), sizes.end())); +} + +template +tensorflow::TensorShape ToTensorShape( + llvm::iterator_range> sizes) { + return tensorflow::TensorShape( + llvm::SmallVector(sizes.begin(), sizes.end())); +} + +// Returns a limit scalar const op for the given type. +// Requires FloatType or IntegerType +static stablehlo::ConstantOp GetScalarLimitConstOfType(Type ty, Location loc, + hlo::ScalarLimit limit, + OpBuilder *builder) { + return stablehlo::ConstantOp::create(*builder, loc, + hlo::getScalarLimitOfType(ty, limit)); +} + +// Deprecated: This is maintained to aid in porting old code that is not yet +// dynamic shape aware and uses broadcasting modes that CHLO does not support. +// Gets the resulting type from a broadcast between two types for statically +// shaped types. This is to be used for legacy lowerings that both use non +// left-padded broadcasting and static shapes. Its use should not be permitted +// in new code. +// May return nullptr on invalid static broadcast dimensions. +// ABSL_DEPRECATED() +static RankedTensorType GetStaticBroadcastType( + RankedTensorType x, RankedTensorType y, + DenseIntElementsAttr broadcast_dimensions_attr) { + auto element_type = x.getElementType(); + auto shape_x = x.getShape(); + auto shape_y = y.getShape(); + + if (shape_x.size() == shape_y.size()) { + llvm::SmallVector out_shape(shape_x.size()); + for (int i = 0; i < shape_x.size(); i++) { + auto x_val = shape_x[i]; + auto y_val = shape_y[i]; + out_shape[i] = std::max(x_val, y_val); + } + return tensorflow::GetTypeFromTFTensorShape(out_shape, element_type); + } + + auto shape_large = shape_x.size() > shape_y.size() ? shape_x : shape_y; + auto shape_small = shape_x.size() <= shape_y.size() ? shape_x : shape_y; + + llvm::SmallVector broadcast_dimensions; + // Explicit broadcast dimensions. + for (const APInt &int_value : broadcast_dimensions_attr) { + broadcast_dimensions.push_back(int_value.getSExtValue()); + } + if (broadcast_dimensions.size() != shape_small.size()) { + return nullptr; + } + llvm::SmallVector out_shape(shape_large.begin(), + shape_large.end()); + + // Update according to the broadcast dimensions. + for (const auto &index_pair : llvm::enumerate(broadcast_dimensions)) { + auto old_value = out_shape[index_pair.value()]; + auto new_value = shape_small[index_pair.index()]; + out_shape[index_pair.value()] = std::max(old_value, new_value); + } + return tensorflow::GetTypeFromTFTensorShape(out_shape, element_type); +} + +// Deprecated: This is maintained to aid in porting old code that is not yet +// dynamic shape aware and uses broadcasting modes that CHLO does not support. +// Applies static binary broadcasting to a binary elementwise op. +// This is a legacy helper to provide general broadcasting support in legacy, +// static shaped code that relies on non-left-padded broadcasting semantics. +template +static Value StaticBinaryBroadcast(Location loc, Value x, Value y, + DenseIntElementsAttr broadcast_dims, + OpBuilder &builder) { + auto x_type = mlir::cast(x.getType()); + auto y_type = mlir::cast(y.getType()); + auto result_type = GetStaticBroadcastType(x_type, y_type, broadcast_dims); + if (!result_type) { + emitError(loc) << "could not binary broadcast " << x_type << ", " << y_type + << " with broadcast_dims = " << broadcast_dims; + return nullptr; + } + auto larger_broadcast_dims = + GetI64ArrayAttrForSeq(0, result_type.getRank(), &builder); + auto broadcast_dims_attr = ToDenseI64ArrayAttr(broadcast_dims, &builder); + if (x_type.getRank() < y_type.getRank()) { + if (x_type != result_type) { + x = stablehlo::BroadcastInDimOp::create(builder, loc, result_type, x, + broadcast_dims_attr); + } + if (y_type != result_type) { + y = stablehlo::BroadcastInDimOp::create(builder, loc, result_type, y, + larger_broadcast_dims); + } + } else { + if (x_type != result_type) { + x = stablehlo::BroadcastInDimOp::create(builder, loc, result_type, x, + larger_broadcast_dims); + } + if (y_type != result_type) { + y = stablehlo::BroadcastInDimOp::create(builder, loc, result_type, y, + broadcast_dims_attr); + } + } + return BinaryOp::create(builder, loc, x, y); +} + +// Gets a 1D tensor type suitable for expressing extents of the given tensor +// value type. If the value type is ranked, the result will be statically +// shaped. Otherwise, it will have a dynamic dimension. +static RankedTensorType GetExtentsTensorTypeFor(TensorType value_type) { + Builder b(value_type.getContext()); + int64_t dim = value_type.hasRank() ? value_type.getRank() : -1; + return tensorflow::GetTypeFromTFTensorShape({dim}, b.getIndexType()); +} + +// Given a value (broadcast_to) and a feature dimension, broadcasts a 1D +// value (broadcast_from) along that feature dimension. This is a shortcut +// for the cases where a 1D tensor must be broadcast along a specific feature +// dimension, which can vary based on data layout, etc. +// +// The extent of `broadcast_from` dim0 must be equal to the extent of the +// feature_dim of `broadcast_to`. +// +// Example: +// [1x2x3x4], [2], 1 -> [1x2x3x4] +// TODO(laurenzo): Swap the order of broadcast_to and broadcast_from for +// consistency. Possibly also rename for clarity. +static Value Broadcast1DToFeatureDim(Location loc, Value broadcast_to, + Value broadcast_from, int64_t feature_dim, + OpBuilder &builder) { + auto broadcast_dims = GetI64ArrayAttr({feature_dim}, &builder); + auto to_type = mlir::cast(broadcast_to.getType()); + auto result_shape = shape::ShapeOfOp::create(builder, loc, broadcast_to); + auto result_extents_type = GetExtentsTensorTypeFor(to_type); + auto result_extents = shape::ToExtentTensorOp::create( + builder, loc, result_extents_type, result_shape); + return stablehlo::DynamicBroadcastInDimOp::create( + builder, loc, to_type, broadcast_from, result_extents, broadcast_dims); +} + +// Broadcasts `input` to the shape of `broadcast_to` value following +// TF::BroadcastTo semantics. +// +// Requires that input is a ranked tensor. +// +// TODO(hinsu): Utilize TF::ShapeOp followed by TF::BroadcastTo once ShapeOp +// supports unranked inputs in the lowering. +static Value BroadcastToShapeOf(Location loc, Value input, Value broadcast_to, + OpBuilder &builder) { + auto result_shape = shape::ShapeOfOp::create(builder, loc, broadcast_to); + auto to_type = mlir::cast(broadcast_to.getType()); + auto result_extents_type = GetExtentsTensorTypeFor(to_type); + auto result_extents = shape::ToExtentTensorOp::create( + builder, loc, result_extents_type, result_shape); + int64_t rank = mlir::cast(input.getType()).getRank(); + auto broadcast_dims = GetI64ArrayAttrForSeq(0, rank, &builder); + return stablehlo::DynamicBroadcastInDimOp::create( + builder, loc, to_type, input, result_extents, broadcast_dims); +} + +// Builds a set of operations for applying reduction on the input value. A +// tf.sum op is created and will be legalized to tfl ops automatically. +static Value ApplyReduction(Location loc, Value input, + DenseIntElementsAttr reduce_dims, + OpBuilder *builder) { + auto reduce_dims_op = + stablehlo::ConstantOp::create(*builder, loc, reduce_dims); + return TF::SumOp::create(*builder, loc, input, reduce_dims_op, + builder->getBoolAttr(false)); +} + +// Creates a stablehlo.rng_uniform op with `builder` to generate `num_elements` +// 32-bit integer numbers in the range of [`lower_limit`, `upper_limit`). +static stablehlo::RngOp CreateRngUniform32(Location loc, int num_elements, + int lower_limit, int upper_limit, + OpBuilder *builder) { + auto shape_tensor = stablehlo::ConstantOp::create( + *builder, loc, GetI64ElementsAttr({num_elements}, builder)); + + auto lower = stablehlo::ConstantOp::create( + *builder, loc, builder->getI32IntegerAttr(lower_limit)); + auto upper = stablehlo::ConstantOp::create( + *builder, loc, builder->getI32IntegerAttr(upper_limit)); + + return stablehlo::RngOp::create(*builder, loc, lower, upper, shape_tensor, + ::mlir::stablehlo::RngDistribution::UNIFORM); +} + +using WhileBodyFnType = llvm::function_ref old_values, + SmallVectorImpl *new_values, OpBuilder *builder)>; + +// Creates a stablehlo.while op with `builder` to loop `num_interations` times, +// each time calling the given `body_fn` on a set of values to generate a new +// set of values. Returns the final set of values via `final_values`. The +// initial set of values is passed in via `init_values`. +// +// This effectively does: +// +// ```c++ +// SmallVector old_values = init_values; +// SmallVector new_values; +// for (int i = 0; i < num_iterations; ++i) { +// body_fn(old_values, &new_values, ...); +// old_values = new_values; +// } +// ``` +// +// Under the hood an induction variable is prepended to values to control the +// number of iterations, but that is transparent to `body_fn`, which does not +// need to care about that. +static void CreateWhile32(Location loc, int num_iterations, + WhileBodyFnType body_fn, ArrayRef init_values, + SmallVectorImpl *final_values, + OpBuilder *builder) { + int value_count = init_values.size() + 1; + + // Prepend a loop induction variable to the initial values. + SmallVector init_values_with_loop_iv; + SmallVector init_types_with_loop_iv; + init_values_with_loop_iv.reserve(value_count); + init_types_with_loop_iv.reserve(value_count); + + // The initial value for the loop induction variable is 0. + init_values_with_loop_iv.push_back(stablehlo::ConstantOp::create( + *builder, loc, builder->getI32IntegerAttr(0))); + init_values_with_loop_iv.append(init_values.begin(), init_values.end()); + + // Accumulate types of all the init values. + for (const auto &init_value_with_loop_iv : init_values_with_loop_iv) + init_types_with_loop_iv.push_back(init_value_with_loop_iv.getType()); + + // Create the while op. + auto while_op = stablehlo::WhileOp::create( + *builder, loc, init_types_with_loop_iv, init_values_with_loop_iv); + auto ivs_count = init_types_with_loop_iv.size(); + + { + OpBuilder::InsertionGuard guard(*builder); + + // Build up the only block in the condition region. + Region &condition = while_op.getCond(); + Block *block = builder->createBlock(&condition); + block->addArguments(init_types_with_loop_iv, + SmallVector(ivs_count, loc)); + + // Get the loop induction variable and compare it against the upper limit. + auto loop_iv = block->getArgument(0); + auto upper_limit = stablehlo::ConstantOp::create( + *builder, loc, builder->getI32IntegerAttr(num_iterations)); + Value compare = + stablehlo::CompareOp::create(*builder, loc, loop_iv, upper_limit, + stablehlo::ComparisonDirection::LT); + + stablehlo::ReturnOp::create(*builder, loc, compare); + } + + { + OpBuilder::InsertionGuard guard(*builder); + + // Build up the only block in the body region. + Region &body = while_op.getBody(); + Block *block = builder->createBlock(&body); + block->addArguments(init_types_with_loop_iv, + SmallVector(ivs_count, loc)); + + SmallVector new_values; // Generated by this iteration + new_values.reserve(value_count); + + // Feed all values excluding the loop induction variable to body_fn. + body_fn(loc, block->getArgument(0), + ArrayRef(block->getArguments().begin() + 1, + block->getArguments().end()), + &new_values, builder); + + // Increment the loop induction variable by one. + auto one = stablehlo::ConstantOp::create(*builder, loc, + builder->getI32IntegerAttr(1)); + auto scalar_broadcast_dims = builder->getDenseI64ArrayAttr({}); + auto plus_one = chlo::BroadcastAddOp::create( + *builder, loc, block->getArgument(0), one, scalar_broadcast_dims); + // Prepend with the updated loop induction variable. + new_values.insert(new_values.begin(), plus_one); + + stablehlo::ReturnOp::create(*builder, loc, new_values); + } + + // TODO(jpienaar): Support multi-operand while op. + final_values->reserve(init_values.size()); + for (int i = 0, e = init_values.size(); i < e; ++i) + final_values->push_back(while_op.getResult(i + 1)); +} + +//===----------------------------------------------------------------------===// +// BatchNorm op utilities. +//===----------------------------------------------------------------------===// + +static IntegerAttr getFeatureDimensionAttr(Builder &b, + tensorflow::TensorFormat format, + Value input) { + return b.getI64IntegerAttr(GetFeatureDimension( + format, mlir::cast(input.getType()))); +} + +//===----------------------------------------------------------------------===// +// FFT op utilities. +//===----------------------------------------------------------------------===// + +// Returns the 1D i64 elements attribute populated with the inner-most dim of +// the value. +static DenseI64ArrayAttr GetInnerDimFromValue(ShapedType type, + Builder *builder) { + if (type.getRank() == 0) { + return builder->getDenseI64ArrayAttr({}); + } + return builder->getDenseI64ArrayAttr(type.getShape().back()); +} + +// Returns True if the inner-most dim is static. +bool CheckInnerDimStatic(ShapedType type, Builder *builder) { + if (!type.hasRank()) { + return false; + } + return !type.isDynamicDim(type.getShape().size() - 1); +} + +//===----------------------------------------------------------------------===// +// MatMul op utilities. +//===----------------------------------------------------------------------===// + +// If the 'transpose' attribute is true returns ElementsAttr to transpose 2D +// matrix. Otherwise, returns ElementsAttr for identity transpose. +static DenseIntElementsAttr Get2DTransposePerm(BoolAttr transpose, Builder *b) { + if (transpose.getValue()) return GetI64ElementsAttr({1, 0}, b); + return GetI64ElementsAttr({0, 1}, b); +} + +//===----------------------------------------------------------------------===// +// Pad op utilities. +//===----------------------------------------------------------------------===// + +// Slices input attribute of rank two and returns the specified column. +// +// Always returns 64 bit integer attribute regardless of bitwidth of the input +// attribute. +static DenseI64ArrayAttr SliceDenseIntElementsAttrColumn2D(ElementsAttr input, + int column) { + auto int_attr = mlir::cast(input); + auto shaped_type = int_attr.getType(); + auto shape = shaped_type.getShape(); + + if (shape.size() != 2) return DenseI64ArrayAttr(); + + llvm::SmallVector values; + values.reserve(shaped_type.getNumElements() / shape[1]); + + for (const auto &it : llvm::enumerate(int_attr.getValues())) { + if (static_cast(it.index() % shape[1]) == column) { + values.push_back(it.value().getSExtValue()); + } + } + + return DenseI64ArrayAttr::get(input.getContext(), values); +} + +// Returns interior padding to use in HLO Pad op based on the TensorFlow padding +// in TensorFlow PadV2 op. +static DenseI64ArrayAttr GetInteriorPadding(ElementsAttr tf_padding) { + auto length = tf_padding.getShapedType().getShape()[0]; + std::vector padding(length, 0); + return DenseI64ArrayAttr::get(tf_padding.getContext(), padding); +} + +//===----------------------------------------------------------------------===// +// Binary op utilities. +//===----------------------------------------------------------------------===// + +// Returns whether the two values are guaranteed to be broadcastable to the +// same shape, this broadcasts size 1 tensors up to any rank. Dynamic dimensions +// must be broadcasted with a size 1 tensor or another dynamic dimension. +// Returns false on rankless. +static bool AreBroadcastCompatible(Value x, Value y) { + auto x_rankless = mlir::dyn_cast(x.getType()); + auto y_rankless = mlir::dyn_cast(y.getType()); + if (!x_rankless || !y_rankless) { + return false; + } + + // Check that the shapes can be broadcasted. + auto shape_x = x_rankless.getShape(); + auto shape_y = y_rankless.getShape(); + + int rank_diff = shape_x.size() - shape_y.size(); + int offset_x = rank_diff > 0 ? rank_diff : 0; + int offset_y = rank_diff < 0 ? -rank_diff : 0; + for (int i = 0, s = std::min(shape_x.size(), shape_y.size()); i < s; i++) { + int index_x = i + offset_x; + int index_y = i + offset_y; + if ((shape_x[index_x] == -1 && shape_y[index_y] != 1) || + (shape_y[index_y] == -1 && shape_x[index_x] != 1)) { + return false; + } + } + + return true; +} + +// Return a new TensorType the same rank and dimensions as the input with an +// updated element type. +static Type ChangeTensorElementType(Builder *b, Type tensor_type, + Type element_type) { + RankedTensorType ranked_type = mlir::dyn_cast(tensor_type); + if (ranked_type) { + return tensorflow::GetTypeFromTFTensorShape(ranked_type.getShape(), + element_type); + } + + return UnrankedTensorType::get(element_type); +} + +//===----------------------------------------------------------------------===// +// Softmax op utilities. +//===----------------------------------------------------------------------===// + +// Returns the type to use for accumulating the given type. +static Type GetAccumulationType(Type ty) { + // Upcast 16 bit sum reductions to 32 bit to reduce the precision loss from + // repeated floating point additions. + return (ty.isF16() || ty.isBF16()) ? Float32Type::get(ty.getContext()) : ty; +} + +//===----------------------------------------------------------------------===// +// Softplus op utilities. +//===----------------------------------------------------------------------===// + +static DenseElementsAttr GetEpsilonValue(Type ty) { + auto element_ty = mlir::cast(ty).getElementType(); + auto scalar_ty = tensorflow::GetTypeFromTFTensorShape({}, element_ty); + if (element_ty.isF16()) { + uint16_t raw_epsilon = Eigen::numext::bit_cast( + Eigen::NumTraits::epsilon()); + auto value = APFloat(APFloat::IEEEhalf(), APInt(16, raw_epsilon)); + return DenseElementsAttr::get(scalar_ty, value); + } else if (element_ty.isBF16()) { + uint16_t raw_epsilon = Eigen::numext::bit_cast( + Eigen::NumTraits::epsilon()); + auto value = APFloat(APFloat::BFloat(), APInt(16, raw_epsilon)); + return DenseElementsAttr::get(scalar_ty, value); + } else if (element_ty.isF32()) { + auto value = APFloat(std::numeric_limits::epsilon()); + return DenseElementsAttr::get(scalar_ty, value); + } else if (element_ty.isF64()) { + auto value = APFloat(std::numeric_limits::epsilon()); + return DenseElementsAttr::get(scalar_ty, value); + } + llvm_unreachable("unsupported element type for tf.SoftPlus"); +} + +//===----------------------------------------------------------------------===// +// ArgMax/ArgMin op utilities. +//===----------------------------------------------------------------------===// + +static void BuildArgMinMaxReductionBody( + Type input_element_type, Type index_element_type, + stablehlo::ComparisonDirection direction, Region *body, + OpBuilder *builder) { + OpBuilder::InsertionGuard insertion_point_gurad(*builder); + + Type input_type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, input_element_type); + Type index_type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, index_element_type); + Block *block = builder->createBlock(body); + Location loc = body->getLoc(); + block->addArguments({input_type, index_type, input_type, index_type}, + SmallVector(4, loc)); + + Value lhs_val = block->getArgument(0); + Value lhs_index = block->getArgument(1); + Value rhs_val = block->getArgument(2); + Value rhs_index = block->getArgument(3); + + ImplicitLocOpBuilder b(loc, *builder); + Value compare_dt = + stablehlo::CompareOp::create(b, lhs_val, rhs_val, direction); + Value selected_input = + stablehlo::SelectOp::create(b, input_type, compare_dt, lhs_val, rhs_val); + + Value compare_eq = stablehlo::CompareOp::create( + b, lhs_val, rhs_val, stablehlo::ComparisonDirection::EQ); + Value min_index = stablehlo::MinOp::create(b, lhs_index, rhs_index); + Value min_val_index = stablehlo::SelectOp::create(b, index_type, compare_dt, + lhs_index, rhs_index); + Value selected_index = stablehlo::SelectOp::create(b, index_type, compare_eq, + min_index, min_val_index); + + Value return_values[] = {selected_input, selected_index}; + stablehlo::ReturnOp::create(b, return_values); +} + +//===----------------------------------------------------------------------===// +// PartitionedCall op utilities. +//===----------------------------------------------------------------------===// + +// Verify that the arguments to be passed into the function are the same types +// as the function paramter types. +static bool ArgTypesMatchCallee(mlir::Operation *op, OperandRange args, + SymbolRefAttr func) { + auto module = op->getParentOfType(); + auto function = + dyn_cast_or_null(SymbolTable::lookupSymbolIn(module, func)); + FunctionType function_ty = function.getFunctionType(); + + for (auto arg_in : llvm::zip(args, function_ty.getInputs())) { + if (std::get<0>(arg_in).getType() != std::get<1>(arg_in)) { + // Argument type and input type mismatch. + return false; + } + } + return true; +} + +//===----------------------------------------------------------------------===// +// Slice op utilities. +//===----------------------------------------------------------------------===// + +static bool CanBeTranslatedToDynamicSlice(Value input, Value start_indices, + DenseIntElementsAttr slice_sizes) { + auto input_ty = mlir::dyn_cast(input.getType()); + if (!input_ty) return false; + auto start_indices_ty = + mlir::dyn_cast(start_indices.getType()); + if (!start_indices_ty) return false; + + int64_t input_rank = input_ty.getRank(); + ArrayRef input_shape = input_ty.getShape(); + DenseIntElementsAttr constant_start_indices; + bool is_constant_start = + matchPattern(start_indices, m_Constant(&constant_start_indices)); + + for (int64_t i = 0; i < input_rank; ++i) { + int64_t input_size = input_shape[i]; + int64_t slice_size = slice_sizes.getValues()[i].getInt(); + // A slice_size of -1 means "all elements from start_index to the end". + // In order to support these semantics, we need to know both the start index + // and the shape of the input dimension. + if (slice_size < 0 && (!is_constant_start || input_size < 0)) return false; + } + return true; +} + +// TF slice size can be -1, which represents all elements from start_index to +// the end. HLO slice size can't be -1. As such, we need to translate TF slice +// size -1 to HLO slice size. +static DenseI64ArrayAttr TFSliceSizes2HLOSliceSizes( + Value input, Value start_indices, DenseIntElementsAttr slice_sizes, + Builder *builder) { + DenseIntElementsAttr constant_start_indices; + if (!matchPattern(start_indices, m_Constant(&constant_start_indices))) { + return ToDenseI64ArrayAttr(slice_sizes, builder); + } + + auto input_ty = mlir::dyn_cast(input.getType()); + int64_t input_rank = input_ty.getRank(); + ArrayRef input_shape = input_ty.getShape(); + SmallVector normalized_sizes; + + for (int64_t i = 0; i < input_rank; ++i) { + int64_t input_size = input_shape[i]; + int64_t start_index = + constant_start_indices.getValues()[i].getInt(); + int64_t slice_size = slice_sizes.getValues()[i].getInt(); + normalized_sizes.push_back(slice_size == -1 ? input_size - start_index + : slice_size); + } + + return GetI64ArrayAttr(normalized_sizes, builder); +} + +//===----------------------------------------------------------------------===// +// XlaGather op utilities. +//===----------------------------------------------------------------------===// + +bool HasValidGatherDims(StringAttr attr) { + ::xla::GatherDimensionNumbers dims; + return dims.ParseFromString(attr.getValue().str()); +} + +stablehlo::GatherDimensionNumbersAttr GetGatherDimNumsAttr(StringAttr attr, + Builder *builder) { + ::xla::GatherDimensionNumbers dims; + if (!dims.ParseFromString(attr.getValue().str())) return {}; + return ::xla::stablehlo::ConvertGatherDimensionNumbers(dims, builder); +} + +//===----------------------------------------------------------------------===// +// XlaDot op utilities. +//===----------------------------------------------------------------------===// + +bool HasValidDotDims(StringAttr attr) { + ::xla::DotDimensionNumbers dims; + return dims.ParseFromString(attr.getValue().str()); +} + +stablehlo::DotDimensionNumbersAttr GetDotDimNumsAttr(StringAttr attr, + Builder *builder) { + ::xla::DotDimensionNumbers dims; + if (!dims.ParseFromString(attr.getValue().str())) return {}; + return ::xla::stablehlo::ConvertDotDimensionNumbers(dims, builder); +} + +bool HasValidPrecisionConfig(StringAttr attr) { + ::xla::PrecisionConfig precision; + return precision.ParseFromString(attr.getValue().str()); +} + +mlir::ArrayAttr GetPrecisionConfigAttr(StringAttr attr, Builder *builder) { + ::xla::PrecisionConfig precision; + if (!precision.ParseFromString(attr.getValue().str())) return {}; + return ::xla::stablehlo::ConvertPrecisionConfig(&precision, builder); +} + +//===----------------------------------------------------------------------===// +// XlaVariadicReduceV2 op utilities. +//===----------------------------------------------------------------------===// + +static void BuildBodyWithCall(PatternRewriter &rewriter, const Location &loc, + mlir::SymbolRefAttr func, + mlir::FunctionType func_ty, Region *body) { + OpBuilder::InsertionGuard guard(rewriter); + + Block *block = rewriter.createBlock(body); + auto inputs = func_ty.getInputs(); + block->addArguments(inputs, SmallVector(inputs.size(), loc)); + mlir::func::CallOp call_op = mlir::func::CallOp::create( + rewriter, loc, func, func_ty.getResults(), block->getArguments()); + stablehlo::ReturnOp::create(rewriter, loc, call_op.getResults()); +} + +//===----------------------------------------------------------------------===// +// Op converters. +//===----------------------------------------------------------------------===// + +NamedAttribute GetConvDimensionNumbersAttr(ArrayRef spatial_dims, + tensorflow::TensorFormat format, + Builder *builder) { + int64_t num_spatial_dims = spatial_dims.size(); + int64_t num_dims = num_spatial_dims + 2; + + int64_t batch_dim = GetTensorBatchDimIndex(num_dims, format); + int64_t feature_dim = GetTensorFeatureDimIndex(num_dims, format); + + // Filters data_format is always HWIO so input channels dimension is after + // all spatial dimensions. + int64_t kernel_input_feature_dim = num_spatial_dims; + int64_t kernel_output_feature_dim = num_spatial_dims + 1; + SmallVector kernel_spatial_dimensions; + kernel_spatial_dimensions.resize(num_spatial_dims); + std::iota(kernel_spatial_dimensions.begin(), kernel_spatial_dimensions.end(), + 0); + + return builder->getNamedAttr( + "dimension_numbers", + stablehlo::ConvDimensionNumbersAttr::get( + builder->getContext(), batch_dim, feature_dim, spatial_dims, + kernel_input_feature_dim, kernel_output_feature_dim, + kernel_spatial_dimensions, batch_dim, feature_dim, spatial_dims)); +} + +// Converts a TF::BiasAddOp to HLO. +// This differs from a normal TF::AddOp with respect to how the data_format +// is handled, which can optionally require a general broadcast of the +// 'bias' term in a way that is not compatible with the standard left-padded +// broadcast semantics (i.e. NCHW will broadcast into dimension 1). +// The correct 'bias' broadcast will be synthesized manually. +class ConvertBiasAddOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::BiasAddOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + auto value_type = mlir::dyn_cast(op.getValue().getType()); + if (!value_type) return failure(); + auto feature_dim = GetFeatureDimension(data_format, value_type); + auto bias_broadcast = Broadcast1DToFeatureDim( + loc, op.getValue(), op.getBias(), feature_dim, rewriter); + Value add = + stablehlo::AddOp::create(rewriter, loc, op.getValue(), bias_broadcast); + if (add.getType() != op.getType()) { + add = tensor::CastOp::create(rewriter, loc, op.getType(), add); + } + rewriter.replaceOp(op, {add}); + return success(); + } +}; + +// Conterts tf.Conv2D to stablehlo.dynamic_conv. +// TODO(disc): To recover static special case's performance with adding folding, +// canonicalization func and removing ConvertConvOp. +template +class ConvertConvDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + bool GetPaddingValues(OpT &op, PatternRewriter &rewriter, Value input_size, + Value filter_size, int64_t dilation_rate, + int64_t stride, tensorflow::Padding padding_type, + Type shape_scalar_type, Value *padding_low, + Value *padding_high) const { + // Stride must be > 0 + if (stride <= 0) return false; + // Dilation rate must be >= 1 + if (dilation_rate < 1) return false; + + Location loc = op.getLoc(); + switch (padding_type) { + case tensorflow::Padding::VALID: { + auto zero = + arith::ConstantIntOp::create(rewriter, loc, shape_scalar_type, 0); + *padding_low = *padding_high = zero; + break; + } + case tensorflow::Padding::EXPLICIT: + break; + case tensorflow::Padding::SAME: { + auto zero = + arith::ConstantIntOp::create(rewriter, loc, shape_scalar_type, 0); + auto one = + arith::ConstantIntOp::create(rewriter, loc, shape_scalar_type, 1); + auto two = + arith::ConstantIntOp::create(rewriter, loc, shape_scalar_type, 2); + // See also the parallel implementation in + // GetWindowedOutputSizeFromDimsV2. effective_filter_size = (filter_size + // - 1) * dilation_rate + 1 + Value stride_value = arith::ConstantIntOp::create( + rewriter, loc, shape_scalar_type, stride); + Value dilation_rate_value = arith::ConstantIntOp::create( + rewriter, loc, shape_scalar_type, dilation_rate); + Value effective_filter_size_op = arith::AddIOp::create( + rewriter, loc, one, + arith::MulIOp::create( + rewriter, loc, dilation_rate_value, + arith::SubIOp::create(rewriter, loc, filter_size, one))); + // output_size = (input_size + stride - 1) / stride; + Value output_size = arith::DivUIOp::create( + rewriter, loc, + arith::AddIOp::create( + rewriter, loc, input_size, + arith::SubIOp::create(rewriter, loc, stride_value, one)), + stride_value); + // std::max(int64{0}, (output_size - 1) * stride + + // effective_filter_size - input_size); + Value padding_needed = arith::SubIOp::create( + rewriter, loc, + arith::AddIOp::create( + rewriter, loc, effective_filter_size_op, + arith::MulIOp::create( + rewriter, loc, stride_value, + arith::SubIOp::create(rewriter, loc, output_size, one))), + input_size); + Value cond = mlir::arith::CmpIOp::create( + rewriter, loc, arith::CmpIPredicate::sge, padding_needed, zero); + padding_needed = mlir::arith::SelectOp::create( + rewriter, loc, padding_needed.getType(), cond, padding_needed, + zero); + *padding_low = + arith::DivUIOp::create(rewriter, loc, padding_needed, two); + *padding_high = + arith::SubIOp::create(rewriter, loc, padding_needed, *padding_low); + break; + } + } + return true; + } + + LogicalResult matchAndRewriteDynamicConv(OpT op, + PatternRewriter &rewriter) const { + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + tensorflow::Padding padding; + if (!GetPaddingFromString(op.getPadding().str(), &padding).ok()) + return failure(); + + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + auto filter_ty = mlir::dyn_cast(op.getFilter().getType()); + auto result_ty = mlir::dyn_cast(op.getType()); + if (!input_ty || !filter_ty || !result_ty) return failure(); + // TODO(disc): Remove this constraint once fold and canonicalization + // implemented. + if (input_ty.hasStaticShape() && filter_ty.hasStaticShape()) + return failure(); + + ArrayRef dilations = op.getDilations().getValue(); + ArrayRef strides = op.getStrides().getValue(); + ArrayRef explicit_paddings; + if (padding == tensorflow::Padding::EXPLICIT) { + // EXPLICIT padding mode and the associated attribute is attached to + // Conv2D. + explicit_paddings = + op->template getAttrOfType("explicit_paddings").getValue(); + } + + SmallVector spatial_dim_indices; + SmallVector rhs_dilations; + SmallVector window_strides; + SmallVector paddings; + + auto get_int = [](Attribute attr) { + return mlir::cast(attr).getInt(); + }; + + constexpr int num_dims = num_spatial_dims + 2; + + Location loc = op.getLoc(); + auto shape_scalar_type = rewriter.getIntegerType(32); + + auto get_const = [&](int64_t val) { + return mlir::arith::ConstantIntOp::create(rewriter, loc, + shape_scalar_type, val); + }; + auto get_dim_value = [&](Value val, int64_t dim) { + Value dim_value = tensor::DimOp::create(rewriter, loc, val, dim); + return arith::IndexCastOp::create(rewriter, loc, shape_scalar_type, + dim_value); + }; + + for (auto i : llvm::seq(0, num_spatial_dims)) { + const int64_t dim = GetTensorSpatialDimIndex(num_dims, data_format, i); + spatial_dim_indices.push_back(dim); + + const int64_t dilation = get_int(dilations[dim]); + rhs_dilations.push_back(dilation); + const int64_t stride = get_int(strides[dim]); + window_strides.push_back(stride); + + Value pad_low, pad_high; + if (padding == tensorflow::Padding::EXPLICIT) { + pad_low = get_const(get_int(explicit_paddings[2 * dim])); + pad_high = get_const(get_int(explicit_paddings[2 * dim + 1])); + } else { + auto input_size = get_dim_value(op.getInput(), dim); + auto filter_size = get_dim_value(op.getFilter(), i); + if (!GetPaddingValues(op, rewriter, input_size, filter_size, dilation, + stride, padding, shape_scalar_type, &pad_low, + &pad_high)) { + return failure(); + } + } + paddings.push_back(pad_low); + paddings.push_back(pad_high); + } + auto rhs_dilations_attr = rewriter.getNamedAttr( + "rhs_dilation", GetI64ArrayAttr(rhs_dilations, &rewriter)); + + auto window_strides_attr = rewriter.getNamedAttr( + "window_strides", GetI64ArrayAttr(window_strides, &rewriter)); + + auto dimension_numbers_attr = GetConvDimensionNumbersAttr( + spatial_dim_indices, data_format, &rewriter); + + const int64_t input_channels = + GetDimSize(input_ty, GetTensorFeatureDimIndex(num_dims, data_format)); + // Filters data_format is always HWIO so input channels dimension is after + // all spatial dimensions. + const int64_t filter_channels = GetDimSize(filter_ty, num_spatial_dims); + // TensorFlow convolution op verifies that the number of input channels is + // divisible by the number of filter channels. + // For depthwise convolution the feature_group_count argument would be set + // to the input feature dimension. + const int64_t feature_group_count = + depthwise_conv ? input_channels : input_channels / filter_channels; + auto feature_group_count_attr = rewriter.getNamedAttr( + "feature_group_count", rewriter.getI64IntegerAttr(feature_group_count)); + + auto batch_group_count_attr = rewriter.getNamedAttr( + "batch_group_count", rewriter.getI64IntegerAttr(1)); + + auto precision_config_attr = rewriter.getNamedAttr( + "precision_config", GetPrecisionConfig(&rewriter)); + + Value paddings_op = tensor::FromElementsOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(2 * num_spatial_dims, + rewriter.getI32Type()), + paddings); + + SmallVector operands(op.getOperands()); + operands.push_back(paddings_op); + // Reshape the filter to {spatial_dims...., 1,in_channels * + // channel_multiplier} + if (depthwise_conv) { + ArrayRef filter_shape = filter_ty.getShape(); + llvm::SmallVector new_shape( + filter_shape.begin(), filter_shape.begin() + num_spatial_dims); + new_shape.push_back(1); + new_shape.push_back(filter_shape[num_spatial_dims] * + filter_shape[num_spatial_dims + 1]); + operands[1] = stablehlo::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(new_shape, + filter_ty.getElementType()), + operands[1]); + } + NamedAttribute attrs[] = {rhs_dilations_attr, window_strides_attr, + dimension_numbers_attr, feature_group_count_attr, + batch_group_count_attr, precision_config_attr}; + rewriter.replaceOpWithNewOp( + op, op.getType(), operands, llvm::ArrayRef(attrs)); + return success(); + } + + LogicalResult matchAndRewrite(OpT op, + PatternRewriter &rewriter) const override { + return matchAndRewriteDynamicConv(op, rewriter); + } +}; + +using ConvertConv2DDynamic = + ConvertConvDynamic; + +// Converts the TensorFlow conv op in template to the generic HLO conv op by +// converting TensorFlow op attributes to HLO op attributes. +// +// Sample result for Conv2D: +// +// %conv = "stablehlo.convolution"(%input, %filter) { +// strides = [1, 2], +// paddings = [[1, 0], [1, 1]], +// ... +// } +// +// This pattern is not defined using declarative rewrite rules as computation of +// the paddings attribute anyway requires multiple source op attributes and +// result op attributes. Defining it as declarative rewrite rule will introduce +// some duplication in the C++ helper methods. +template +class ConvertConvOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + tensorflow::Padding padding; + if (!GetPaddingFromString(op.getPadding().str(), &padding).ok()) + return failure(); + + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + auto filter_ty = mlir::dyn_cast(op.getFilter().getType()); + + // With the exception of input's batch dimension, input and filter need to + // have static shape for calculation of HLO paddings and feature group count + // attributes. Filter is validated here, input is mostly validated at use. + if (!input_ty || !filter_ty || !filter_ty.hasStaticShape()) + return failure(); + + ArrayRef dilations = op.getDilations().getValue(); + ArrayRef strides = op.getStrides().getValue(); + ArrayRef explicit_paddings; + if (padding == tensorflow::Padding::EXPLICIT) { + // EXPLICIT padding mode and the associated attribute is limited to + // Conv2D. So, fetch attribute by identifier instead of the + // op.explicit_paddings() attribute getter. + explicit_paddings = + op->template getAttrOfType("explicit_paddings").getValue(); + } + + SmallVector spatial_dim_indices; + SmallVector rhs_dilations; + SmallVector window_strides; + SmallVector paddings; + + auto get_int = [](Attribute attr) { + return mlir::cast(attr).getInt(); + }; + + constexpr int num_dims = num_spatial_dims + 2; + for (auto i : llvm::seq(0, num_spatial_dims)) { + const int64_t dim = GetTensorSpatialDimIndex(num_dims, data_format, i); + spatial_dim_indices.push_back(dim); + + const int64_t dilation = get_int(dilations[dim]); + rhs_dilations.push_back(dilation); + const int64_t stride = get_int(strides[dim]); + window_strides.push_back(stride); + + int64_t pad_low, pad_high; + if (padding == tensorflow::Padding::EXPLICIT) { + pad_low = get_int(explicit_paddings[2 * dim]); + pad_high = get_int(explicit_paddings[2 * dim + 1]); + } else { + int64_t output_size; + int64_t pad_low_int64; + int64_t pad_high_int64; + int64_t input_size = input_ty.getDimSize(dim); + if (input_size == ShapedType::kDynamic) return failure(); + absl::Status status = tensorflow::GetWindowedOutputSizeVerbose( + input_size, filter_ty.getDimSize(i), dilation, stride, padding, + &output_size, &pad_low_int64, &pad_high_int64); + if (!status.ok()) return failure(); + pad_low = pad_low_int64; + pad_high = pad_high_int64; + } + paddings.push_back(pad_low); + paddings.push_back(pad_high); + } + + auto rhs_dilations_attr = rewriter.getNamedAttr( + "rhs_dilation", GetI64ArrayAttr(rhs_dilations, &rewriter)); + + auto window_strides_attr = rewriter.getNamedAttr( + "window_strides", GetI64ArrayAttr(window_strides, &rewriter)); + + auto dimension_numbers_attr = GetConvDimensionNumbersAttr( + spatial_dim_indices, data_format, &rewriter); + + const int64_t input_channels = + GetDimSize(input_ty, GetTensorFeatureDimIndex(num_dims, data_format)); + if (input_channels == ShapedType::kDynamic) return failure(); + // Filters data_format is always HWIO so input channels dimension is after + // all spatial dimensions. + const int64_t filter_channels = GetDimSize(filter_ty, num_spatial_dims); + // TensorFlow convolution op verifies that the number of input channels is + // divisible by the number of filter channels. + // For depthwise convolution the feature_group_count argument would be set + // to the input feature dimension. + const int64_t feature_group_count = + depthwise_conv ? input_channels : input_channels / filter_channels; + auto feature_group_count_attr = rewriter.getNamedAttr( + "feature_group_count", rewriter.getI64IntegerAttr(feature_group_count)); + + auto batch_group_count_attr = rewriter.getNamedAttr( + "batch_group_count", rewriter.getI64IntegerAttr(1)); + + RankedTensorType paddings_ty = tensorflow::GetTypeFromTFTensorShape( + {num_spatial_dims, 2}, rewriter.getIntegerType(64)); + auto paddings_attr = rewriter.getNamedAttr( + "padding", DenseElementsAttr::get(paddings_ty, paddings)); + + auto precision_config_attr = rewriter.getNamedAttr( + "precision_config", GetPrecisionConfig(&rewriter)); + + SmallVector operands(op.getOperands()); + // Reshape the filter to {spatial_dims...., 1,in_channels * + // channel_multiplier} + if (depthwise_conv) { + ArrayRef filter_shape = filter_ty.getShape(); + llvm::SmallVector new_shape( + filter_shape.begin(), filter_shape.begin() + num_spatial_dims); + new_shape.push_back(1); + new_shape.push_back(filter_shape[num_spatial_dims] * + filter_shape[num_spatial_dims + 1]); + operands[1] = stablehlo::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(new_shape, + filter_ty.getElementType()), + operands[1]); + } + NamedAttribute attrs[] = {rhs_dilations_attr, window_strides_attr, + dimension_numbers_attr, feature_group_count_attr, + batch_group_count_attr, paddings_attr, + precision_config_attr}; + rewriter.replaceOpWithNewOp( + op, op.getType(), operands, llvm::ArrayRef(attrs)); + return success(); + } +}; + +using ConvertConv2DOp = ConvertConvOp; +using ConvertConv3DOp = ConvertConvOp; +using ConvertDepthConv2DOp = + ConvertConvOp; + +// Converts tf.PadV2Op to stablehlo.DynamicPadOp. Padding values must be const. +class ConvertPadOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + // TODO(disc): To recover static special case's performance with folding and + // canonicalization. + LogicalResult matchAndRewrite(TF::PadV2Op op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + auto input = op.getInput(); + auto paddings = op.getPaddings(); + auto constant_values = op.getConstantValues(); + auto input_type = mlir::dyn_cast(input.getType()); + auto paddings_type = mlir::dyn_cast(paddings.getType()); + if (!input_type || !paddings_type || !paddings_type.hasStaticShape()) + return failure(); + + // TODO(disc): Remove this constraint once fold and canonicalization is + // implemented. + if (input_type.hasStaticShape()) return failure(); + + int input_rank = input_type.getRank(); + // interior padding + std::vector interior_values(input_rank, 0); + auto interior_attr = GetI64ElementsAttr(interior_values, &rewriter); + + Value interior_padding_tensor = + stablehlo::ConstantOp::create(rewriter, loc, interior_attr); + Type paddings_elem_ty = paddings_type.getElementType(); + if (!paddings_elem_ty.isInteger(64)) { + interior_padding_tensor = stablehlo::ConvertOp::create( + rewriter, loc, interior_padding_tensor, paddings_elem_ty); + } + llvm::SmallVector transposed_shape = {2, input_rank}; + auto transpose_attr = GetI64ArrayAttr({1, 0}, &rewriter); + Value transposed_paddings = + stablehlo::TransposeOp::create(rewriter, loc, paddings, transpose_attr); + Value reshaped_paddings = + stablehlo::ReshapeOp::create(rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {input_rank * 2}, paddings_elem_ty), + transposed_paddings); + + auto left_padding_start_attr = GetI64ArrayAttr({0}, &rewriter); + auto left_padding_limit_attr = GetI64ArrayAttr({input_rank}, &rewriter); + auto left_padding_stride_attr = GetI64ArrayAttr({1}, &rewriter); + Value left_padding_tensor = stablehlo::SliceOp::create( + rewriter, loc, reshaped_paddings, left_padding_start_attr, + left_padding_limit_attr, left_padding_stride_attr); + + auto right_padding_start_attr = GetI64ArrayAttr({input_rank}, &rewriter); + auto right_padding_limit_attr = + GetI64ArrayAttr({2 * input_rank}, &rewriter); + auto right_padding_stride_attr = GetI64ArrayAttr({1}, &rewriter); + Value right_padding_tensor = stablehlo::SliceOp::create( + rewriter, loc, reshaped_paddings, right_padding_start_attr, + right_padding_limit_attr, right_padding_stride_attr); + + rewriter.replaceOpWithNewOp( + op, op.getType(), input, constant_values, left_padding_tensor, + right_padding_tensor, interior_padding_tensor); + + return success(); + } +}; + +class ConvertGatherNdOpDynamic : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + // Converts tf.GatherNdOp to stablehlo.DynamicGatherOp. + // Here we leave 'slice_sizes' as an Attr, without defining a new + // DynamicGatherOp, since GatherDimensionNumbers has already provide enough + // information for shape inference and code generation of stablehlo::GatherOp. + // '?' will be filled into slice_sizes for dimensions that are dynamic sized. + // TODO(disc): To recover static special case's performance with folding and + // canonicalization. + LogicalResult matchAndRewrite(TF::GatherNdOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + auto params = op.getParams(); + auto params_ty = mlir::dyn_cast(params.getType()); + auto indices = op.getIndices(); + auto indices_ty = mlir::dyn_cast(indices.getType()); + auto params_rank = params_ty.getRank(); + auto indices_rank = indices_ty.getRank(); + int64_t num_index_dims = indices_ty.getDimSize(indices_rank - 1); + if (!params_ty || !indices_ty) return failure(); + // the last dim of indices of GatherNdOp must be fixed shaped + if (num_index_dims == ShapedType::kDynamic) return failure(); + + SmallVector slice_sizes; + slice_sizes.reserve(params_rank); + for (int64_t i = 0; i < params_rank; ++i) { + if (i < num_index_dims) { + slice_sizes.push_back(1); + } else { + // potentially dynamic + int64_t dim_size = params_ty.getDimSize(i); + slice_sizes.push_back(dim_size); + } + } + SmallVector slice_sizes_vals; + Value slice_sizes_value = nullptr; + for (int64_t i = 0; i < params_rank; ++i) { + if (i < num_index_dims) { + slice_sizes_vals.push_back(arith::ConstantOp::create( + rewriter, loc, + rewriter.getIntegerAttr(indices_ty.getElementType(), 1))); + } else { + int64_t dim_size = params_ty.getDimSize(i); + if (dim_size != ShapedType::kDynamic) { + slice_sizes_vals.push_back(arith::ConstantOp::create( + rewriter, loc, + rewriter.getIntegerAttr(indices_ty.getElementType(), dim_size))); + } else { + slice_sizes_vals.push_back(arith::IndexCastOp::create( + rewriter, loc, indices_ty.getElementType(), + tensor::DimOp::create(rewriter, loc, params, i))); + } + } + } + slice_sizes_value = + tensor::FromElementsOp::create(rewriter, loc, slice_sizes_vals); + + // collapsed_slice_dims + SmallVector collapsed_slice_dims; + collapsed_slice_dims.reserve(num_index_dims); + for (int64_t i = 0; i < num_index_dims; ++i) { + collapsed_slice_dims.push_back(i); + } + // offset_dims + SmallVector offset_dims; + offset_dims.reserve(params_rank - num_index_dims); + for (int64_t i = num_index_dims; i < params_rank; i++) { + offset_dims.push_back(i + indices_rank - 1 - num_index_dims); + } + // start_index_map + SmallVector start_index_map; + offset_dims.reserve(num_index_dims); + for (int64_t i = 0; i < num_index_dims; i++) { + start_index_map.push_back(i); + } + // index_vector_dim + int64_t index_vector_dim = indices_rank - 1; + + auto dims_attr = stablehlo::GatherDimensionNumbersAttr::get( + rewriter.getContext(), offset_dims, collapsed_slice_dims, + /*operandBatchingDims=*/{}, + /*startIndicesBatchingDims=*/{}, start_index_map, index_vector_dim); + // TODO(disc): Remove this if-statement once fold and canonicalization is + // implemented. + if (params_ty.hasStaticShape() && indices_ty.hasStaticShape()) { + rewriter.replaceOpWithNewOp( + op, op.getType(), op.getParams(), op.getIndices(), dims_attr, + GetI64ArrayAttr(slice_sizes, &rewriter)); + } else { + rewriter.replaceOpWithNewOp( + op, op.getType(), op.getParams(), op.getIndices(), slice_sizes_value, + dims_attr); + } + return success(); + } +}; + +// Converts BF16 FloorDiv op to have casting operators on either end as BF16 +// division can result in strange behavior. +// +// floordiv = cast(floordiv(cast(left), cast(right)))) +// +// %left_cast = cast(%left) +// %right_cast = cast(%right) +// %div = div(%left, %left) +// %floored = floor(%div) +// %floored_cast = cast(%floored) +// +// Required to manually specify the intermediate types. +class ConvertBF16FloorDivOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::FloorDivOp op, + PatternRewriter &rewriter) const override { + auto l = mlir::dyn_cast>(op.getX()); + auto r = mlir::dyn_cast>(op.getY()); + if (!l || !r) return failure(); + + auto element_type = getElementTypeOrSelf(l.getType()); + if (!element_type.isBF16()) return failure(); + + auto out_type = op.getZ().getType(); + + l = stablehlo::ConvertOp::create(rewriter, op.getLoc(), l, + rewriter.getF32Type()); + r = stablehlo::ConvertOp::create(rewriter, op.getLoc(), r, + rewriter.getF32Type()); + + auto intermediate = TF::FloorDivOp::create( + rewriter, op.getLoc(), + ChangeTensorElementType(&rewriter, out_type, rewriter.getF32Type()), l, + r); + + auto floor_op = stablehlo::ConvertOp::create(rewriter, op.getLoc(), + out_type, intermediate); + rewriter.replaceOp(op, floor_op.getResult()); + return success(); + } +}; + +class ConvertBroadcastToOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::BroadcastToOp op, + PatternRewriter &rewriter) const override { + auto input_type = mlir::dyn_cast(op.getInput().getType()); + auto output_type = op.getOutput().getType(); + if (!input_type) { + return rewriter.notifyMatchFailure(op, "requires ranked input shape"); + } + llvm::SmallVector broadcast_dimensions; + if (input_type.getRank() > 0) { + auto ranked_output_type = mlir::dyn_cast(output_type); + if (!ranked_output_type) { + return rewriter.notifyMatchFailure(op, "requires ranked output shape"); + } + auto rank_diff = ranked_output_type.getRank() - input_type.getRank(); + // The tf.BroadcastTo op performs "right-aligned" numpy-style + // broadcasting. + broadcast_dimensions = llvm::to_vector<4>( + llvm::seq(rank_diff, ranked_output_type.getRank())); + } + rewriter.replaceOpWithNewOp( + op, output_type, op.getInput(), op.getShape(), + GetI64ArrayAttr(broadcast_dimensions, &rewriter)); + return success(); + } +}; + +/// Converts a TF::RollOp to HLO. Only support 0D axis and shift case, and axis +/// have to be a constant. +class ConvertRollOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::RollOp op, + PatternRewriter &rewriter) const override { + auto shift_ty = mlir::dyn_cast(op.getShift().getType()); + if (!shift_ty || shift_ty.getRank() != 0) { + return rewriter.notifyMatchFailure( + op, "require the type of shift to be 0D tensor"); + } + + APInt val; + if (!matchPattern(op.getAxis(), m_ConstantInt(&val))) { + return rewriter.notifyMatchFailure(op, "require axis to be constant"); + } + int axis = val.getSExtValue(); + + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + if (!input_ty || !input_ty.hasStaticShape()) { + return rewriter.notifyMatchFailure( + op, "require the type of input to have static shapes"); + } + ArrayRef input_shape = input_ty.getShape(); + int input_rank = input_ty.getRank(); + if (axis < 0) axis += input_rank; + + // Adjust large offsets into [0, axis_size). This also makes negative + // offsets positive. + // offset = ((offset % axis_size) + axis_size) % axis_size + ImplicitLocOpBuilder b(op.getLoc(), rewriter); + Value offset = op.getShift(); + auto axis_size = stablehlo::ConstantOp::create( + b, b.getIntegerAttr(getElementTypeOrSelf(offset.getType()), + input_shape[axis])); + offset = stablehlo::RemOp::create( + b, + stablehlo::AddOp::create( + b, stablehlo::RemOp::create(b, offset, axis_size), axis_size), + axis_size); + + // Stack two copies of the dimension, then slice from the calculated + // offset. This also works if shift is not constant. + // DynamicSliceOp requires the sizes being integer, and we can get the + // information from input shape. + auto concat = stablehlo::ConcatenateOp::create( + b, ValueRange{op.getInput(), op.getInput()}, b.getI64IntegerAttr(axis)); + Value zero = stablehlo::ConstantOp::create( + b, b.getIntegerAttr(getElementTypeOrSelf(offset.getType()), 0)); + SmallVector slice_begin_indices(input_rank, zero); + slice_begin_indices[axis] = + stablehlo::SubtractOp::create(b, axis_size, offset); + rewriter.replaceOpWithNewOp( + op, input_ty, concat, slice_begin_indices, + GetI64ArrayAttr(input_shape, &rewriter)); + return success(); + } +}; + +/// Converts a TF::LeakyReluOp to HLO. +/// LeakyRelu(x) = alpha * x if x < 0 else x. +class ConvertLeakyReluOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::LeakyReluOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value features = op.getFeatures(); + + // Use ConstantLike for `alpha` to match the shape of feature. + auto alphaVal = chlo::getConstantLike( + rewriter, loc, op.getAlpha().convertToFloat(), features); + Value zeroVal = chlo::getConstantLike(rewriter, loc, 0.0, features); + + Value leakyActivationVal = + stablehlo::MulOp::create(rewriter, loc, features, alphaVal); + + Value compareGtZero = stablehlo::CompareOp::create( + rewriter, loc, features, zeroVal, stablehlo::ComparisonDirection::GT); + + rewriter.replaceOpWithNewOp( + op, compareGtZero, features, leakyActivationVal); + return success(); + } +}; + +/// Converts a TF::LeakyReluGradOp to HLO. +/// LeakyReluGrad(gradient, inputs) = gradient if input > 0 +/// else alpha * gradient. +class ConvertLeakyReluGradOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::LeakyReluGradOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value gradients = op.getGradients(); + Value features = op.getFeatures(); + auto featureType = features.getType(); + + // Use ConstantLike for `alpha` to match the shape of feature. + auto alphaVal = chlo::getConstantLike( + rewriter, loc, op.getAlpha().convertToFloat(), features); + Value zeroVal = chlo::getConstantLike(rewriter, loc, 0.0, features); + + Value leakyGradientVal = + stablehlo::MulOp::create(rewriter, loc, gradients, alphaVal); + + Value compareGtZero = stablehlo::CompareOp::create( + rewriter, loc, features, zeroVal, stablehlo::ComparisonDirection::GT); + + rewriter.replaceOpWithNewOp( + op, featureType, compareGtZero, gradients, leakyGradientVal); + return success(); + } +}; + +// Converts TensorFlow DiagPartOp to HLO ops using reduction on masked matrix. +// For a Rank-2 input, it creates the following ops: +// %1 = "stablehlo.iota"() {iota_dimension = 0 : i64} +// %2 = "stablehlo.iota"() {iota_dimension = 1 : i64} +// %3 = "stablehlo.compare"(%1, %2) {comparison_direction = "EQ"} +// %4 = stablehlo.constant dense<0.000000e+00> : tensor +// %5 = "stablehlo.broadcast"(%4) +// %6 = "stablehlo.select"(%3, %input, %5) +// %7 = "stablehlo.reduce"(%6, %4) ({ +// ^bb0(%arg1: tensor, %arg2: tensor): +// %9 = stablehlo.add %arg1, %arg2 : tensor +// "stablehlo.return"(%9) : (tensor) -> () +// }) {dimensions = dense<0> : tensor<1xi64>} +// +// If the input's rank N is greater than 2, we will reshape it to R2 first and +// create the above ops, then reshape it back to rank N/2. +class ConvertDiagPartOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::DiagPartOp op, + PatternRewriter &rewriter) const override { + auto input_type = mlir::dyn_cast(op.getInput().getType()); + if (!input_type || !input_type.hasStaticShape()) return failure(); + int64_t num_dims = input_type.getRank(); + if (num_dims < 2 || num_dims % 2 != 0) return failure(); + const int64_t out_dims = num_dims / 2; + + int64_t new_size = 1; + llvm::SmallVector new_dims; + for (int i = 0; i < out_dims; i++) { + if (input_type.getDimSize(i) != input_type.getDimSize(i + out_dims)) + return op.emitOpError("invalid dimensions size"); + new_size *= input_type.getDimSize(i); + new_dims.push_back(input_type.getDimSize(i)); + } + Value reshaped_input = stablehlo::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape({new_size, new_size}, + input_type.getElementType()), + op.getInput()); + auto iota_type = tensorflow::GetTypeFromTFTensorShape( + {new_size, new_size}, rewriter.getIntegerType(32)); + auto iota0 = stablehlo::IotaOp::create(rewriter, op.getLoc(), iota_type, + rewriter.getI64IntegerAttr(0)); + auto iota1 = stablehlo::IotaOp::create(rewriter, op.getLoc(), iota_type, + rewriter.getI64IntegerAttr(1)); + Value compare = + stablehlo::CompareOp::create(rewriter, op.getLoc(), iota0, iota1, + stablehlo::ComparisonDirection::EQ); + Value zero = GetScalarConstOfType(input_type.getElementType(), op.getLoc(), + 0, &rewriter); + Value zero_matrix = stablehlo::BroadcastOp::create( + rewriter, op.getLoc(), reshaped_input.getType(), zero, + GetI64ArrayAttr({new_size, new_size}, &rewriter)); + Value masked = stablehlo::SelectOp::create( + rewriter, op.getLoc(), reshaped_input.getType(), compare, + reshaped_input, zero_matrix); + auto reduce = stablehlo::ReduceOp::create( + rewriter, op.getLoc(), masked, zero, GetI64ArrayAttr({0}, &rewriter), + input_type.getElementType()); + assert(!input_type.getElementType().isInteger(1) && + "data type should not be i1"); + BuildReduceBody(input_type.getElementType(), + &reduce.getBody(), &rewriter); + rewriter.replaceOpWithNewOp( + op, + tensorflow::GetTypeFromTFTensorShape(new_dims, + input_type.getElementType()), + reduce.getResult(0)); + return success(); + } +}; + +// Converts TensorFlow MatrixDiagPartOp to HLO ops. +class ConvertMatrixDiagPartV3Op + : public OpRewritePattern { + using Shape = llvm::SmallVector; + + // Parse the "k" parameter. MatrixDiagPartV3 allows to specify the diagonal(s) + // with k. This can be either a single value (for a single diagonal) or a + // tuple of two values (starting and ending diagonal, for a band). + LogicalResult ExtractK(TF::MatrixDiagPartV3Op op, int64_t (*k)[2]) const { + DenseIntElementsAttr kattr; + if (!matchPattern(op.getK(), m_Constant(&kattr))) { + return failure(); + } + DenseIntElementsAttr::iterator it = kattr.begin(); + (*k)[0] = (*it).getSExtValue(); + it++; + if (it == kattr.end()) { + // Handle input like e.g. "k = 5", in which case we extract a single + // diagonal. + (*k)[1] = (*k)[0]; + } else { + // Handle input like e.g. "k = [-1, 1]", in which case we extract a + // band (multiple diagonals). + (*k)[1] = (*it).getSExtValue(); + } + return success(); + } + + // Utility method for broadcasting integer constants to a given shape. + stablehlo::BroadcastOp BroadcastConstant(Location loc, Shape shape, + int32_t constant, int int_size, + PatternRewriter &rewriter) const { + return stablehlo::BroadcastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(shape, + rewriter.getIntegerType(int_size)), + GetScalarConstOfType(rewriter.getIntegerType(int_size), loc, constant, + &rewriter), + GetI64ArrayAttr(shape, &rewriter)); + } + + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::MatrixDiagPartV3Op op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + ShapedType input_type = mlir::dyn_cast(op.getInput().getType()); + + // Align is a string specifying how superdiagonals and subdiagonals should + // be aligned/padded for diagonals that are shorter than max_diag_len. The + // format is "{super}_{sub}", with {super} the superdiagonal alignment and + // {sub} the subdiagonal alignment. "LEFT" means rows will be padded to the + // left, "RIGHT" means rows will be padded ot the right. The default is + // "RIGHT_LEFT". + StringRef align = op->getAttrOfType("align").getValue(); + enum Alignment { kLeft, kRight }; + + // default is RIGHT_LEFT + Alignment superdiagonal_align = kRight; + Alignment subdiagonal_align = kLeft; + + if (align == "RIGHT_LEFT") { + superdiagonal_align = kRight; + subdiagonal_align = kLeft; + } else if (align == "RIGHT_RIGHT") { + superdiagonal_align = kRight; + subdiagonal_align = kRight; + } else if (align == "LEFT_RIGHT") { + superdiagonal_align = kLeft; + subdiagonal_align = kRight; + } else if (align == "LEFT_LEFT") { + superdiagonal_align = kLeft; + subdiagonal_align = kLeft; + } else { + return failure(); // unsupported alignment + } + + // MatrixDiagPart operates on a matrix of shape [I, J, ..., L, M, N], and + // will extract the diagonal(s) out of [M, N], for all [I, J, ..., L]. + if (!input_type || !input_type.hasStaticShape()) return failure(); + int64_t num_dims = input_type.getRank(); + if (num_dims < 2) return failure(); + int64_t rows = input_type.getDimSize(num_dims - 2); // rows + int64_t cols = input_type.getDimSize(num_dims - 1); // cols + + // We extract the diagonals from k[0] up to and including k[1]. + // Addressing is 0 for the main diagonal. (So k = [0, 0] would just extract + // the main diagonal). It's negative for subdiagonals (under and to the left + // of the main diagonal) and positive for superdiagonals (above and to the + // right of the main diagonal). + int64_t k[2]; + if (failed(ExtractK(op, &k))) return failure(); + int num_diags = k[1] - k[0] + 1; + + // Shifting diagonals away from the main diagonal might shorten them. This + // is the longest diagonal we will see. We make this the last dimension of + // the output shape. + int64_t max_diag_len = + std::min(rows + std::min(k[1], static_cast(0)), + cols + std::min(-k[0], static_cast(0))); + + // The first dimension is the index vector dimension we'll use for gather. + // It's 1 here, but will be 2 once we glue x and y together. + Shape indices_shape({1, num_diags, max_diag_len}); + + RankedTensorType iota_type = tensorflow::GetTypeFromTFTensorShape( + indices_shape, rewriter.getIntegerType(32)); + Value iotaM = stablehlo::IotaOp::create(rewriter, loc, iota_type, + rewriter.getI64IntegerAttr(1)); + Value iotaN = stablehlo::IotaOp::create(rewriter, loc, iota_type, + rewriter.getI64IntegerAttr(2)); + + // Boradcasted constants, of the same shape as iotaM and iotaN. + Value b_zero = BroadcastConstant(loc, indices_shape, 0, 32, rewriter); + Value b_false = BroadcastConstant(loc, indices_shape, 0, 1, rewriter); + Value b_true = BroadcastConstant(loc, indices_shape, -1, 1, rewriter); + Value b_k1 = BroadcastConstant(loc, indices_shape, k[1], 32, rewriter); + Value b_rows = BroadcastConstant(loc, indices_shape, rows, 32, rewriter); + Value b_cols = BroadcastConstant(loc, indices_shape, cols, 32, rewriter); + Value b_max_diag_len = + BroadcastConstant(loc, indices_shape, max_diag_len, 32, rewriter); + + // d = k[1] - m + // (A.k.a. the number of the diagonal, depending on m. Note that we + // subtract m here. This means we start with the superdiagonals and + // move downwards towards the subdiagonals. So the start indices will + // be decreasing.) + Value d = stablehlo::SubtractOp::create(rewriter, loc, b_k1, iotaM); + Value neg_d = stablehlo::NegOp::create(rewriter, loc, d); + + // diag_len_d = min(rows + min(d, 0), cols - max(d, 0)) + // (Length of a diagonal for a given d. Same as max_diag_len for m = 0.) + Value diag_len_d = stablehlo::MinOp::create( + rewriter, loc, + stablehlo::AddOp::create( + rewriter, loc, b_rows, + stablehlo::MinOp::create(rewriter, loc, d, b_zero)), + stablehlo::SubtractOp::create( + rewriter, loc, b_cols, + stablehlo::MaxOp::create(rewriter, loc, d, b_zero))); + + // offset is max_diag_len - diag_len_d if we're padding, 0 otherwise. + Value cmp; + if (subdiagonal_align == kRight && superdiagonal_align == kRight) { + cmp = b_true; + } else if (superdiagonal_align == kRight) { + // offset = d>=0 ? max_diag_len - diag_len_d : 0 + cmp = TF::GreaterEqualOp::create(rewriter, loc, d, b_zero); + } else if (subdiagonal_align == kRight) { + // offset = d<=0 ? max_diag_len - diag_len_d : 0 + cmp = TF::LessEqualOp::create(rewriter, loc, d, b_zero); + } else { + // offset = 0 + cmp = b_false; + } + + // This offset shifts the diagonals to the "left" or "right", depending + // on alignment. + Value offset = stablehlo::SelectOp::create( + rewriter, loc, b_zero.getType(), cmp, + stablehlo::SubtractOp::create(rewriter, loc, b_max_diag_len, + diag_len_d), + b_zero); + + // x = max(d, 0) - offset + // y = max(-d, 0) - offset + Value x = stablehlo::SubtractOp::create( + rewriter, loc, stablehlo::MaxOp::create(rewriter, loc, d, b_zero), + offset); + Value y = stablehlo::SubtractOp::create( + rewriter, loc, stablehlo::MaxOp::create(rewriter, loc, neg_d, b_zero), + offset); + + Value n_plus_x = stablehlo::AddOp::create(rewriter, loc, iotaN, x); + Value n_plus_y = stablehlo::AddOp::create(rewriter, loc, iotaN, y); + + // GatherOp is happy about letting us index out of bounds values, but those + // values will be undefined. So we mask them later. Set up the boolean + // expression that tells us which entries, in the output shape, are out of + // bounds and thus become the padding_value. + Value x_in_bounds = stablehlo::AndOp::create( + rewriter, loc, + TF::GreaterEqualOp::create(rewriter, loc, b_false.getType(), n_plus_x, + b_zero), + TF::LessOp::create(rewriter, loc, b_false.getType(), n_plus_x, b_cols)); + Value y_in_bounds = stablehlo::AndOp::create( + rewriter, loc, + TF::GreaterEqualOp::create(rewriter, loc, b_false.getType(), n_plus_y, + b_zero), + TF::LessOp::create(rewriter, loc, b_false.getType(), n_plus_y, b_rows)); + Value in_bounds = stablehlo::ReshapeOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(Shape({num_diags, max_diag_len}), + rewriter.getIntegerType(1)), + stablehlo::AndOp::create(rewriter, loc, x_in_bounds, y_in_bounds)); + + // Now combine x and y into the index data structure needed for gather. + Shape concat_shape({2, num_diags, max_diag_len}); + Value start_indices = stablehlo::ConcatenateOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(concat_shape, + rewriter.getIntegerType(32)), + mlir::ValueRange({n_plus_y, n_plus_x}), + mlir::IntegerAttr::get(rewriter.getIntegerType(64), 0)); + + // Shape of the final output. (Except for dimension folding in the + // single diagonal case.) + Shape output_shape; + for (int i = 0; i < num_dims - 2; i++) { + output_shape.push_back(input_type.getDimSize(i)); + } + output_shape.push_back(num_diags); + output_shape.push_back(max_diag_len); + + // A slice is the shape of what GatherOp copies per lookup. So the last + // two dimensions (M, N in the matrix-diag-part docs) are where we go + // through entry by entry. + ArrayRef input_shape = input_type.getShape(); + int input_shape_size = input_shape.size(); + Shape slice_sizes(input_shape.begin(), input_shape.end()); + int slice_dimensions = slice_sizes.size(); + slice_sizes[slice_dimensions - 2] = + std::min((int64_t)1, input_shape[input_shape_size - 2]); + slice_sizes[slice_dimensions - 1] = + std::min((int64_t)1, input_shape[input_shape_size - 1]); + + // Dimensions of the input we won't see in the output (M and N). + SmallVector collapsed_dims( + {slice_dimensions - 2, slice_dimensions - 1}); + + // Which dimensions (in the input) the two offset "columns" map to. + SmallVector start_index_map({num_dims - 2, num_dims - 1}); + + // Gather the diagonal entries. + // TODO(kramm): For a single diagonal, this might be slower than the + // mask + sum approach. Special-case num_diags==1? + auto dims_attr = stablehlo::GatherDimensionNumbersAttr::get( + rewriter.getContext(), + /*offsetDims=*/llvm::to_vector<4>(llvm::seq(0, num_dims - 2)), + /*collapsedSliceDims=*/collapsed_dims, + /*operandBatchingDims=*/{}, + /*startIndicesBatchingDims=*/{}, start_index_map, + /*indexVectorDim=*/0); + Value gather = stablehlo::GatherOp::create( + rewriter, loc, op.getInput(), start_indices, dims_attr, + GetI64ArrayAttr(slice_sizes, &rewriter)); + + // We now need to broadcast the "in_bounds" boolean expression, as well as + // the padding value, to do the final select. + Shape broadcast_bounds; + for (int i = 0; i < output_shape.size() - 2; i++) { + broadcast_bounds.push_back(output_shape[i]); + } + Value b_in_bounds = stablehlo::BroadcastOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(output_shape, + rewriter.getIntegerType(1)), + in_bounds, GetI64ArrayAttr(broadcast_bounds, &rewriter)); + Value b_padding = stablehlo::BroadcastOp::create( + rewriter, loc, op.getPaddingValue(), + GetI64ArrayAttr(output_shape, &rewriter)); + + // Replace all out-of-bounds values in the result with padding_value. + Value result = stablehlo::SelectOp::create(rewriter, loc, b_in_bounds, + gather, b_padding); + + if (num_diags == 1) { + // matrix_diag_part folds away the 1-sized band dimension if we only + // extract a single diagonal. + result = + stablehlo::ReshapeOp::create(rewriter, loc, op.getType(), result); + } + + rewriter.replaceOp(op, result); + return success(); + } +}; + +// Converts TensorFlow EinsumOp to HLO EinsumOp +class ConvertEinsumOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::EinsumOp op, + PatternRewriter &rewriter) const override { + // Prepend `,` to equation if unary einsum. + std::string equation_str = op.getEquation().str(); + llvm::SmallVector inputs; + + // Unary einsum prepends `,` to equation and + // creates a scalar constant 1.0 for first operand. + if (op.getN() == 1) { + equation_str = "," + equation_str; + inputs.push_back(stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + hlo::getScalarOfType(mlir::getElementTypeOrSelf(op.getOperand(0)), + 1))); + } + // Insert remaining operands into inputs, TF op verifier requires there be + // 0 or 1 operands. + auto operands = op.getInputs(); + inputs.insert(inputs.end(), operands.begin(), operands.end()); + assert(inputs.size() == 2); + + rewriter.replaceOpWithNewOp( + op, op.getType(), inputs[0], inputs[1], equation_str); + return success(); + } +}; + +// Bypasses IdentityN op. +class ConvertIdentityNOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(TF::IdentityNOp op, + PatternRewriter &rewriter) const override { + rewriter.replaceOp(op, op.getOperands()); + return success(); + } +}; + +template +class ConvertFFTOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + auto input_ty = mlir::cast(op.getInput().getType()); + if (!input_ty.hasRank()) { + return failure(); + } + auto input_shape = input_ty.getShape(); + DenseIntElementsAttr fft_length_attr; + if (!matchPattern(op.getFftLength(), m_Constant(&fft_length_attr))) { + return failure(); + } + int64_t fft_length; + if (fft_length_attr.getNumElements() != 0) { + fft_length = fft_length_attr.getValues()[0].getInt(); + } else { + return failure(); + } + + int64_t expected_dim = fft_length; + std::string fft_string = "RFFT"; + if (typeid(OpTy) == typeid(TF::IRFFTOp)) { + expected_dim = fft_length / 2 + 1; + fft_string = "IRFFT"; + } + Location loc = op.getLoc(); + + // The inner-most dim cannot be dynamic. + if (input_ty.isDynamicDim(input_shape.size() - 1)) { + return failure(); + } + + auto expected_shape = llvm::to_vector<4>(input_shape.drop_back()); + expected_shape.push_back(expected_dim); + + // Zero pad or truncate the last axis + Value reshaped = op.getInput(); + SmallVector begin_indices(input_shape.size(), 0); + SmallVector strides(input_shape.size(), 1); + + // Last dim larger than expected_dim, slice the input + if (input_shape.back() > expected_dim) { + reshaped = stablehlo::SliceOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(expected_shape, + input_ty.getElementType()), + op.getInput(), GetI64ArrayAttr(begin_indices, &rewriter), + GetI64ArrayAttr(expected_shape, &rewriter), + GetI64ArrayAttr(strides, &rewriter)); + + // Last dim smaller than expected_dim, zero-pad the input + } else if (input_ty.getShape().back() < expected_dim) { + SmallVector no_padding(input_shape.size(), 0); + SmallVector padding(input_shape.size() - 1, 0); + padding.push_back(expected_dim - input_shape.back()); + Value zero = + GetScalarConstOfType(input_ty.getElementType(), loc, 0, &rewriter); + reshaped = stablehlo::PadOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(expected_shape, + input_ty.getElementType()), + op.getInput(), zero, GetI64ArrayAttr(no_padding, &rewriter), + GetI64ArrayAttr(padding, &rewriter), + GetI64ArrayAttr(no_padding, &rewriter)); + } + + rewriter.replaceOpWithNewOp( + op, op.getType(), reshaped, + stablehlo::FftTypeAttr::get( + rewriter.getContext(), + stablehlo::symbolizeFftType(fft_string).value()), + GetI64ArrayAttr(fft_length, &rewriter)); + return success(); + } +}; + +using ConvertRFFTOp = ConvertFFTOp; +using ConvertIRFFTOp = ConvertFFTOp; + +// The base class to convert TensorFlow FusedBatchNormGrad*Op to HLO +// BatchNormGradOp for training and a sequence of binary ops for inference. +// TODO(b/145536565): move to legalize_tf_patterns.td if it applies. +template +class ConvertFusedBatchNormGradBase + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(FusedBatchNormGradOpT op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value grad = op.getYBackprop(); + Value act = op.getX(); + Value scale = op.getScale(); + Value mean = op.getReserveSpace_1(); + Value var = op.getReserveSpace_2(); + + // TODO(b/141785544): Update this to not require static shapes. + // activation shape needs to be static to convert negative indices in + // TensorFlow to absolute indices required by HLO. + RankedTensorType act_type = mlir::dyn_cast(act.getType()); + if (!act_type) return failure(); + Type act_ele_type = act_type.getElementType(); + // To support mixed precision, the statistics type, which maybe more + // precise than the input types, are used for this op. + Type kernel_type = mlir::cast(scale.getType()).getElementType(); + grad = stablehlo::ConvertOp::create(rewriter, loc, grad, kernel_type); + act = stablehlo::ConvertOp::create(rewriter, loc, act, kernel_type); + + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + auto feature_dim_attr = getFeatureDimensionAttr(rewriter, data_format, act); + auto feature_dim = feature_dim_attr.getValue().getSExtValue(); + + // Gets the result values. + Value x_backprop, scale_backprop, offset_backprop; + if (op.getIsTraining()) { // training + // TODO(b/145536565): handle GPU logic separately. + // Infers the output type with the converted `act`. + Type feature_type = tensorflow::GetTypeFromTFTensorShape( + {GetDimSize(act_type, feature_dim)}, kernel_type); + + SmallVector operand_types = {act.getType(), feature_type, + feature_type}; + auto training_op = stablehlo::BatchNormGradOp::create( + rewriter, loc, operand_types, act, scale, mean, var, grad, + op.getEpsilon(), feature_dim); + + x_backprop = training_op.getResult(0); + + scale_backprop = training_op.getResult(1); + + offset_backprop = training_op.getResult(2); + } else { // inference + SmallVector non_feature_dims; + for (int64_t i = 0; i < act_type.getRank(); ++i) { + if (i == feature_dim) continue; + non_feature_dims.push_back(i); + } + auto reduce_dims = GetI64ElementsAttr(non_feature_dims, &rewriter); + auto scalar_broadcast_dims = rewriter.getDenseI64ArrayAttr({}); + + // scratch1 = rsqrt(var + epsilon) + RankedTensorType scalar_float = + tensorflow::GetTypeFromTFTensorShape({}, kernel_type); + auto epsilon = stablehlo::ConstantOp::create( + rewriter, loc, + DenseFPElementsAttr::get(scalar_float, {op.getEpsilon()})); + auto add_op = chlo::BroadcastAddOp::create( + rewriter, loc, var, epsilon.getResult(), scalar_broadcast_dims); + + Value scratch1 = stablehlo::RsqrtOp::create(rewriter, loc, add_op); + + // scratch2 = sum(y_backprop * (x - mean)) + auto sub_op = stablehlo::SubtractOp::create( + rewriter, loc, act, + Broadcast1DToFeatureDim(loc, act, mean, feature_dim, rewriter)); + auto weighted_grad = + stablehlo::MulOp::create(rewriter, loc, grad, sub_op); + Value scratch2 = + ApplyReduction(loc, weighted_grad, reduce_dims, &rewriter); + + // x_backprop = y_backprop * (scale * scratch1) + auto scaled_grad = + stablehlo::MulOp::create(rewriter, loc, op.getScale(), scratch1); + x_backprop = stablehlo::MulOp::create( + rewriter, loc, grad, + Broadcast1DToFeatureDim(loc, act, scaled_grad, feature_dim, + rewriter)); + + // scale_backprop = scratch2 * scratch1 + scale_backprop = + stablehlo::MulOp::create(rewriter, loc, scratch1, scratch2); + + // offset_backprop = sum(y_backprop) + offset_backprop = ApplyReduction(loc, grad, reduce_dims, &rewriter); + } + + x_backprop = + stablehlo::ConvertOp::create(rewriter, loc, x_backprop, act_ele_type); + Value last_val[2]; + if (op.getResult(3).use_empty() && op.getResult(4).use_empty()) { + // It doesn't matter what values we provide for the last 2 results. + last_val[0] = last_val[1] = op.getX(); + } else { + auto const_val = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + DenseElementsAttr::get( + tensorflow::GetTypeFromTFTensorShape( + {0}, getElementTypeOrSelf(op.getResult(3))), + 0.0)); + auto maybe_cast = [&](Value val, Type t) -> Value { + if (val.getType() == t) return val; + return tensor::CastOp::create(rewriter, op.getLoc(), t, val); + }; + last_val[0] = maybe_cast(const_val, op.getResult(3).getType()); + last_val[1] = maybe_cast(const_val, op.getResult(4).getType()); + } + rewriter.replaceOp( + op, {/*x_backprop=*/x_backprop, + /*scale_backprop=*/scale_backprop, + /*offset_backprop=*/offset_backprop, last_val[0], last_val[1]}); + return success(); + } +}; + +using ConvertFusedBatchNormGradOp = + ConvertFusedBatchNormGradBase; +using ConvertFusedBatchNormGradV2Op = + ConvertFusedBatchNormGradBase; +using ConvertFusedBatchNormGradV3Op = + ConvertFusedBatchNormGradBase; + +// Converts TensorFlow FusedBatchNormV3Op to either HLO BatchNormTrainingOp or +// HLO BatchNormInferenceOp, depending on the value of the 'is_training' +// parameter. +template +class ConvertFusedBatchNormBase : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(FusedBatchNormOpT op, + PatternRewriter &rewriter) const override { + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + auto feature_dim = + getFeatureDimensionAttr(rewriter, data_format, op.getX()); + + auto input_type_tensor = mlir::cast(op.getX().getType()); + auto input_element_type = input_type_tensor.getElementType(); + + auto scale_type_tensor = mlir::cast(op.getScale().getType()); + auto scale_element_type = scale_type_tensor.getElementType(); + + auto mean_type_tensor = mlir::cast(op.getMean().getType()); + auto mean_element_type = mean_type_tensor.getElementType(); + // In the training case, dimensions of input tensors must be static. + if (op.getIsTraining() && (!input_type_tensor.hasStaticShape() || + !scale_type_tensor.hasStaticShape() || + !mean_type_tensor.hasStaticShape())) + return failure(); + + // TODO(b/69928690): Support mixed precision in the XLA batch + // normalization operators. As a workaround, create a new x with the same + // element type as scale (which may be more precise than the input type). + Value bn_train_input = stablehlo::ConvertOp::create( + rewriter, op.getLoc(), op.getX(), scale_element_type); + TensorType bn_train_input_type_tensor = + mlir::cast(bn_train_input.getType()); + + if (op.getIsTraining()) { + // Training case. + auto operand_shape = bn_train_input_type_tensor.getShape(); + // The mean and variance are each 1 dimensional arrays the size of the + // feature dimension, with the same element type as the operand (x). + // This shape must be constructed manually because the mean and variance + // inputs are empty in the training case. + Type mean_var_type = tensorflow::GetTypeFromTFTensorShape( + {operand_shape[feature_dim.getInt()]}, scale_element_type); + // Op result type is a tuple of 3 values: output with same shape as input; + // batch_mean, and batch_var. + SmallVector operand_types = {bn_train_input_type_tensor, + mean_var_type, mean_var_type}; + auto bn_train_op = stablehlo::BatchNormTrainingOp::create( + rewriter, op.getLoc(), operand_types, bn_train_input, op.getScale(), + op.getOffset(), op.getEpsilon(), feature_dim.getInt()); + // HLO op outputs a tuple of tensors. Extract those results. + Value y_out = bn_train_op.getResult(0); + Value batch_mean = bn_train_op.getResult(1); + Value reserve_space_1 = batch_mean; + Value batch_variance = bn_train_op.getResult(2); + + // Apply Bessel's correction on the variance. + int total_input_size = bn_train_input_type_tensor.getNumElements(); + int total_scale_size = scale_type_tensor.getNumElements(); + int sample_size = + total_scale_size > 0 ? total_input_size / total_scale_size : 0; + int sample_size_minus_one = std::max(1, sample_size - 1); + double factor = static_cast(sample_size) / + static_cast(sample_size_minus_one); + auto factor_const_op = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + rewriter.getFloatAttr(scale_element_type, factor)); + + Value corrected_variance = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), batch_variance.getType(), batch_variance, + factor_const_op, /*broadcast_dimensions=*/DenseI64ArrayAttr()); + + // Convert back to input type to stay aligned with expected output type + // for TF op. + y_out = stablehlo::ConvertOp::create(rewriter, op.getLoc(), y_out, + input_element_type); + + float exponential_avg_factor = + op.getExponentialAvgFactor().convertToFloat(); + if (exponential_avg_factor != 1.0f) { + auto alpha = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + rewriter.getFloatAttr(mean_element_type, + 1.0f - exponential_avg_factor)); + auto beta = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + rewriter.getFloatAttr(mean_element_type, exponential_avg_factor)); + + // new_running_mean = alpha * old_mean + beta * batch_mean. + auto alpha_mul_old_mean = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), op.getMean().getType(), alpha, op.getMean(), + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + auto beta_mul_batch_mean = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), batch_mean.getType(), beta, batch_mean, + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + batch_mean = chlo::BroadcastAddOp::create( + rewriter, op.getLoc(), alpha_mul_old_mean, beta_mul_batch_mean, + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + + // new_running_variance = alpha * old_variance + beta * batch_variance. + auto alpha_mul_old_variance = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), op.getVariance().getType(), alpha, + op.getVariance(), + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + auto beta_mul_batch_variance = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), corrected_variance.getType(), beta, + corrected_variance, + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + corrected_variance = chlo::BroadcastAddOp::create( + rewriter, op.getLoc(), alpha_mul_old_variance, + beta_mul_batch_variance, + /*broadcast_dimensions=*/DenseI64ArrayAttr()); + } + + if (std::is_same::value) { + // FusedBatchNormV2 expects 4 outputs. + // Outputs 3 and 4 are currently marked as "reserved spaces 1 and 2". + // They are used to pass the per-batch mean and variance to the + // gradiant. Here we maintain the same behavior by setting them to the + // mean and variance calculated by BatchNormTraining. + rewriter.replaceOp(op, {y_out, /*batch_mean=*/batch_mean, + /*batch_variance=*/corrected_variance, + /*reserve_space_1=*/reserve_space_1, + /*reserve_space_2=*/batch_variance}); + } else { // TF::FusedBatchNormV3Op + // For FusedBatchNormV3Op, also create a constant tensor to forward to + // last reserve_space_3 output. + auto reserve_space_3_type = + mlir::cast(op.getResult(5).getType()); + int num_elements = reserve_space_3_type.hasStaticShape() + ? reserve_space_3_type.getNumElements() + : 0; + auto const_attr_type = tensorflow::GetTypeFromTFTensorShape( + {num_elements}, getElementTypeOrSelf(reserve_space_3_type)); + Value dummy_const = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + DenseElementsAttr::get(const_attr_type, 0.0)); + if (const_attr_type != reserve_space_3_type) + dummy_const = tensor::CastOp::create( + rewriter, op.getLoc(), reserve_space_3_type, dummy_const); + rewriter.replaceOp(op, {y_out, /*batch_mean=*/batch_mean, + /*batch_variance=*/corrected_variance, + /*reserve_space_1=*/reserve_space_1, + /*reserve_space_2=*/batch_variance, + /*reserve_space_3=*/dummy_const}); + } + } else { // Inference case. + auto bn_train_op = stablehlo::BatchNormInferenceOp::create( + rewriter, op.getLoc(), + /*result_type=*/bn_train_input_type_tensor, bn_train_input, + op.getScale(), op.getOffset(), op.getMean(), op.getVariance(), + op.getEpsilon(), feature_dim.getInt()); + + // Convert back to input type to stay aligned with expected output type + // for TF op. + auto y_out = stablehlo::ConvertOp::create( + rewriter, op.getLoc(), bn_train_op, input_element_type); + + // The mean, variance, and reserved space outputs of the batch norm op are + // not used for inference. It doesn't matter what values we provide for + // the last 5 results as long as they are of the same type. Forward + // input mean and variance to output mean, variance, reserved_space_1 and + // reserved_space_2. + if (std::is_same::value) { + rewriter.replaceOp(op, {/*y=*/y_out, + /*batch_mean=*/op.getMean(), + /*batch_variance=*/op.getVariance(), + /*reserve_space_1=*/op.getMean(), + /*reserve_space_2=*/op.getVariance()}); + } else { + // For FusedBatchNormV3Op, also create a constant tensor to forward to + // last reserve_space_3 output. + auto reserve_space_3_type = + mlir::cast(op.getResult(5).getType()); + int num_elements = reserve_space_3_type.hasStaticShape() + ? reserve_space_3_type.getNumElements() + : 0; + auto const_attr_type = tensorflow::GetTypeFromTFTensorShape( + {num_elements}, getElementTypeOrSelf(reserve_space_3_type)); + Value dummy_const = stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + DenseElementsAttr::get(const_attr_type, 0.0)); + if (const_attr_type != reserve_space_3_type) + dummy_const = tensor::CastOp::create( + rewriter, op.getLoc(), reserve_space_3_type, dummy_const); + rewriter.replaceOp(op, {/*y=*/y_out, + /*batch_mean=*/op.getMean(), + /*batch_variance=*/op.getVariance(), + /*reserve_space_1=*/op.getMean(), + /*reserve_space_2=*/op.getVariance(), + /*reserve_space_3=*/dummy_const}); + } + } + return success(); + } +}; + +using ConvertFusedBatchNormV2Op = + ConvertFusedBatchNormBase; +using ConvertFusedBatchNormV3Op = + ConvertFusedBatchNormBase; + +using PaddingArray = std::vector>; + +// Returns padding values for ReduceWindow op as a vector of pairs. +// +// Requires padding to be either 'SAME' or 'VALID' and the number of input +// dimensions to be equal to the size of window dimensions and window strides. +template +static PaddingArray GetReduceWindowPaddingAsArray( + llvm::ArrayRef input_dims, ArrayAttr window_dims, + ArrayAttr window_strides, StringRef padding, Builder *builder) { + if (padding == "VALID") { + return PaddingArray(num_dims, std::make_pair(0, 0)); + } + assert(padding == "SAME"); + llvm::SmallVector input_shape, window_shape, strides; + input_shape.reserve(input_dims.size()); + window_shape.reserve(window_shape.size()); + strides.reserve(window_strides.size()); + + for (const auto &dim : input_dims) input_shape.push_back(dim); + for (Attribute attr : window_dims) + window_shape.push_back(mlir::cast(attr).getInt()); + for (Attribute attr : window_strides) + strides.push_back(mlir::cast(attr).getInt()); + + PaddingArray paddings = ::xla::MakePadding(input_shape, window_shape, strides, + ::xla::Padding::kSame); + return paddings; +} + +// Same as GetReduceWindowPaddingAsArray but returns padding as +// DenseIntElementsAttr. Returns empty attribute for `VALID` padding. +template +static DenseIntElementsAttr GetReduceWindowPaddingAsAttr( + llvm::ArrayRef input_dims, ArrayAttr window_dims, + ArrayAttr window_strides, StringRef padding, Builder *builder) { + if (padding == "VALID") return {}; + assert(padding == "SAME"); + PaddingArray paddings = GetReduceWindowPaddingAsArray( + input_dims, window_dims, window_strides, padding, builder); + int64_t rank = paddings.size(); + llvm::SmallVector flatten_paddings(rank * 2); + for (int i = 0; i < rank; i++) { + flatten_paddings[2 * i] = paddings[i].first; + flatten_paddings[2 * i + 1] = paddings[i].second; + } + return DenseIntElementsAttr::get(tensorflow::GetTypeFromTFTensorShape( + {rank, 2}, builder->getIntegerType(64)), + flatten_paddings); +} + +// Helper function for dividing each entry of `pooled` by the count of its +// corresponding window, i.e., the number of non-padding entries of the window +// which an `AvgPool` operation performed on an `input_shape`-tensor would map +// to this entry, depending on `ksize` and `strides`. This function is used for +// `AvgPool` and `AvgPoolGrad` legalizations. +// `zero` is passed as a parameter because it can be reused from caller level. +// `pooled` must have `RankedTensorType`. +template +Operation *AvgPoolDivideByCount( + Value pooled, const SmallVector &input_shape, + const SmallVector &ksize, + const SmallVector &strides, OpTy op, Value zero, + PatternRewriter &rewriter) { + Location loc = op.getLoc(); + RankedTensorType pooled_type = mlir::cast(pooled.getType()); + Type element_type = pooled_type.getElementType(); + Operation *result = nullptr; + RankedTensorType orig_input_type = + tensorflow::GetTypeFromTFTensorShape(input_shape, element_type); + + if (op.getPadding() == "VALID") { + // All window counts are equal here because we don't have padding + // (each entry of `pooled` corresponds to a window that consists of + // original input entries only). + int64_t window_count = std::accumulate(ksize.begin(), ksize.end(), 1, + std::multiplies()); + // Divide `pooled` by window counts. + Value divisor = + GetScalarConstOfType(element_type, loc, window_count, &rewriter); + auto scalar_broadcast_dims = rewriter.getDenseI64ArrayAttr({}); + result = chlo::BroadcastDivOp::create(rewriter, loc, pooled_type, pooled, + divisor, scalar_broadcast_dims); + } else { + assert(op.getPadding() == "SAME"); + // For SAME padding, only original entries that contributed to a window + // are counted for the average of this window, not padded entries. + + // Build all-ones tensor of same shape as the original input. + ElementsAttr splat = hlo::getSplat(&rewriter, orig_input_type, 1); + auto all_ones_tensor = stablehlo::ConstantOp::create(rewriter, loc, splat); + + // Get padding for the input. + DenseIntElementsAttr input_padding_attr = + GetReduceWindowPaddingAsAttr(input_shape, op.getKsize(), + op.getStrides(), op.getPadding(), + &rewriter); + + // Count the 1's in each window, using the same padding as for the input, + // which gives us the window counts by which `pooled` needs to be divided. + auto divisor = stablehlo::ReduceWindowOp::create( + rewriter, loc, pooled_type, + /*operand=*/all_ones_tensor, + /*init_value=*/zero, + /*window_dimensions=*/ + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getKsize()), &rewriter), + /*window_strides=*/ + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getStrides()), &rewriter), + /*base_dilations=*/DenseI64ArrayAttr(), + /*window_dilations=*/DenseI64ArrayAttr(), + /*padding=*/input_padding_attr); + BuildReduceBody(element_type, &divisor.getBody(), + &rewriter); + + // Divide `pooled` by window counts. + result = stablehlo::DivOp::create(rewriter, loc, pooled_type, pooled, + divisor.getResult(0)); + } + return result; +} + +Value GetAvgPoolInput(TF::AvgPoolOp op) { return op.getValue(); } +Value GetAvgPoolInput(TF::AvgPool3DOp op) { return op.getInput(); } + +// Converts AvgPool op to HLO ReduceWindow op by setting appropriate window +// dimensions with add as the reduction function. The reduction result is +// then divided by the number of elements in the window. +template +class ConvertAvgPoolOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + Value input_value = GetAvgPoolInput(op); + auto input_type = mlir::dyn_cast(input_value.getType()); + if (!input_type) return failure(); + + // We will do accumulation first; use a larger bitwidth if suitable. + Type input_element_type = input_type.getElementType(); + Type sum_element_type = GetSumAccumulationType(input_element_type); + Type result_type; + + // The result type for reduction and division with the proper element type. + if (auto ranked_type = mlir::dyn_cast(op.getType())) + result_type = tensorflow::GetTypeFromTFTensorShape(ranked_type.getShape(), + sum_element_type); + else + result_type = UnrankedTensorType::get(sum_element_type); + + // Convert if we need enlarge the element type's bitwidth. + if (input_element_type != sum_element_type) + input_value = stablehlo::ConvertOp::create(rewriter, op.getLoc(), + input_value, sum_element_type); + + // Create the ReduceWindow op. + Value init = + GetScalarConstOfType(sum_element_type, op.getLoc(), 0, &rewriter); + DenseIntElementsAttr paddings_attr = GetReduceWindowPaddingAsAttr( + input_type.getShape(), op.getKsize(), op.getStrides(), op.getPadding(), + &rewriter); + auto reduce = stablehlo::ReduceWindowOp::create( + rewriter, op.getLoc(), result_type, input_value, init, + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getKsize()), &rewriter), + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getStrides()), &rewriter), + /*base_dilations=*/DenseI64ArrayAttr(), + /*window_dilations=*/DenseI64ArrayAttr(), paddings_attr); + BuildReduceBody(sum_element_type, &reduce.getBody(), + &rewriter); + + // Count the number of elements in the window. The following calculation + // is only valid for no paddings. + SmallVector input_shape( + llvm::to_vector(input_type.getShape())); + SmallVector ksize, strides; + GetI64ArrayAttrValues(op.getKsize(), &ksize); + GetI64ArrayAttrValues(op.getStrides(), &strides); + + Operation *result_op = AvgPoolDivideByCount( + reduce.getResult(0), input_shape, ksize, strides, op, init, rewriter); + + // Convert back if we enlarged the element type's bitwidth. + Value result = result_op->getOpResult(0); + if (input_element_type != sum_element_type) + result = stablehlo::ConvertOp::create(rewriter, op.getLoc(), result, + input_element_type); + + rewriter.replaceOp(op, result); + return success(); + } +}; + +using ConvertAvgPool2DOp = ConvertAvgPoolOp; +using ConvertAvgPool3DOp = ConvertAvgPoolOp; + +// `AvgPoolGradOp` is converted to the following operations: +// 1. Divide each entry of the output gradient (the gradient for the previous +// layer in backpropagation order) by the count of the corresponding window +// (i.e., the number of non-padding entries of the window which `AvgPool` +// has mapped to this entry in forward propagation). +// 2. Add appropriate interior and exterior padding for step 3 (see example +// below). +// 3. Convolve the result of step 2. with a kernel consisting of 1's (same shape +// as windows) and stride 1 in each dimension. This is implemented as a +// `ReduceWindowOp` with `AddOp` as body. +// +// Example: +// Let f : R^4 -> R^2 be an average pool function with window size 3, stride 2, +// and SAME padding with 0's. It is defined by +// f(x) = [ (x_1 + x_2 + x_3) / 3 ] ( x = (x_1, x_2, x_3, x_4) ) +// [ (x_3 + x_4 + 0) / 2 ] (the 0 results from right padding) +// Note that for SAME padding in `AvgPool` the padded entries are not counted +// for the average, this is why the second denominator is 2 and not 3. +// The Jacobian Df is +// [ 1/3 1/3 1/3 0 ] +// [ 0 0 1/2 1/2 ] +// +// Note that the Jacobian is constant (this is why `ConvertAvgPoolGradOp` only +// needs the original input shape and not the tensor as argument). +// Let v = [ 4 6 ]^T be the output gradient (^T = transposed). Then the +// average pool gradient is given by +// Df^T * v = [ 4/3 4/3 13/3 3 ]^T +// Instead of a matrix-vector-multiplication we can utilize the sparsity and +// structure of Df by using the 3-step approach from above: +// 1. Divide output gradient v by window counts: [ 4/3 6/2 ]^T +// 2. Add appropriate padding: [ 0 0 4/3 0 3 0 ]^T +// 3. Convolve with kernel [ 1 1 1 ]: [ 4/3 4/3 11/3 3 ]^T +// +// Note that the padding in step 2. is chosen in such a way that the subsequent +// convolution produces the gradient. Higher dimensions, different padding, and +// different windows/strides work in a similar way, the main difference is in +// the computation of the paddings in step 2. +// +// For more details on backpropagation for convolution of which `AvgPoolGrad` +// is a special case see `tensorflow/core/kernels/conv_grad_ops.h`. +// `tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir` has more +// examples for different cases. +template +class ConvertAvgPoolGradOp : public OpRewritePattern { + using DimVector = SmallVector; + + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) { + return op.emitOpError("invalid data format"); + } + // `out_grad` is the gradient that was propagated via backpropagation from + // the output layer. + Value out_grad = op.getGrad(); + auto out_grad_type = mlir::dyn_cast(out_grad.getType()); + if (!out_grad_type) { + return failure(); + } + Type element_type = out_grad_type.getElementType(); + DenseIntElementsAttr orig_input_shape_attr; + if (!matchPattern(op.getOrigInputShape(), + m_Constant(&orig_input_shape_attr))) { + return failure(); + } + auto orig_input_shape_values = orig_input_shape_attr.getValues(); + DimVector orig_input_shape(orig_input_shape_values.begin(), + orig_input_shape_values.end()); + DimVector ksize, strides; + GetI64ArrayAttrValues(op.getKsize(), &ksize); + GetI64ArrayAttrValues(op.getStrides(), &strides); + Value zero = GetScalarConstOfType(element_type, loc, 0, &rewriter); + + auto out_grad_divided = AvgPoolDivideByCount( + out_grad, orig_input_shape, ksize, strides, op, zero, rewriter); + + // Get same padding as for original input. + PaddingArray orig_padding = GetReduceWindowPaddingAsArray( + orig_input_shape, op.getKsize(), op.getStrides(), op.getPadding(), + &rewriter); + + // Add padding around `out_grad_divided` values in such a way that the + // subsequent `ReduceWindowOp` produces the gradient. + DimVector out_grad_shape( + llvm::to_vector(out_grad_type.getShape())); + DimVector low_padding(num_dims, 0); + DimVector high_padding(num_dims, 0); + DimVector interior_padding(num_dims, 0); + constexpr int num_spatial_dims = num_dims - 2; + for (int i = 0; i < num_spatial_dims; ++i) { + int dim = tensorflow::GetTensorSpatialDimIndex(num_dims, data_format, i); + int orig_input_shape_padded_in_dim = orig_input_shape[dim] + + orig_padding[dim].first + + orig_padding[dim].second; + // Set interior padding such that neighboring entries from + // `out_grad_divided` have distance `strides[dim]` from each other in + // every dimension. + interior_padding[dim] = strides[dim] - 1; + // Set exterior padding in the same way as for convolution gradient + // computation. + auto status = ::xla::ConvGradExtractAndVerifyDimension( + /*input_size=*/orig_input_shape_padded_in_dim, + /*filter_size=*/ksize[dim], + /*output_size=*/out_grad_shape[dim], + /*dilation=*/1, + /*stride=*/strides[dim], + /*padding=*/::xla::Padding::kValid); + if (!status.ok()) { + return failure(); + } + ::xla::SpatialDimensionOutputSizeAndPadding &conv_grad_spatial_dim = + status.value(); + // Subtract the original exterior padding since it doesn't contribute to + // the gradient. Note that we save one `PadOp` and some unnecessary kernel + // computations, compared to the `xla::AvgPoolGrad` implementation, by + // subtracting the original exterior padding before `ReduceWindowOp` + // instead of trimming the result of `ReduceWindowOp` (the final result is + // the same because all strides are 1). + low_padding[dim] = + conv_grad_spatial_dim.pad_before - orig_padding[dim].first; + high_padding[dim] = + conv_grad_spatial_dim.pad_after - orig_padding[dim].second; + + // Update `out_grad_shape` to result shape of following `PadOp`. + out_grad_shape[dim] = low_padding[dim] + high_padding[dim] + + (out_grad_shape[dim] - 1) * strides[dim] + 1; + } + Value reduce_window_input = stablehlo::PadOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(out_grad_shape, element_type), + /*operand=*/out_grad_divided->getOpResult(0), + /*padding_value=*/zero, + /*edge_padding_low=*/GetI64ArrayAttr(low_padding, &rewriter), + /*edge_padding_high=*/GetI64ArrayAttr(high_padding, &rewriter), + /*interior_padding=*/GetI64ArrayAttr(interior_padding, &rewriter)); + + // Compute result by convolving `reduce_window_input` with an all-ones + // kernel, using `ReduceWindowOp` with `AddOp` body. + + Type sum_element_type = GetSumAccumulationType(element_type); + if (element_type != sum_element_type) { + // Convert to appropriate sum accumulation type to avoid precision loss. + reduce_window_input = stablehlo::ConvertOp::create( + rewriter, loc, reduce_window_input, sum_element_type); + zero = GetScalarConstOfType(sum_element_type, loc, 0, &rewriter); + } + auto ones = GetI64ArrayAttr(DimVector(num_dims, 1), &rewriter); + auto reduce_window_op = stablehlo::ReduceWindowOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(orig_input_shape, + sum_element_type), + /*operand=*/reduce_window_input, + /*init_value=*/zero, + /*window_dimensions=*/ + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getKsize()), &rewriter), + /*window_strides=*/ones, + /*base_dilations=*/DenseI64ArrayAttr(), + /*window_dilations=*/DenseI64ArrayAttr(), + /*padding=*/DenseIntElementsAttr()); + BuildReduceBody(sum_element_type, + &reduce_window_op.getBody(), &rewriter); + Value result = reduce_window_op.getResult(0); + + if (element_type != sum_element_type) { + // Convert back to original element type. + result = stablehlo::ConvertOp::create(rewriter, op.getLoc(), result, + element_type); + } + rewriter.replaceOp(op, {result}); + return success(); + } +}; + +using ConvertAvgPool2DGradOp = + ConvertAvgPoolGradOp; +using ConvertAvgPool3DGradOp = + ConvertAvgPoolGradOp; + +// Converts MaxPool op to HLO ReduceWindow op by setting appropriate window +// dimensions with max as the reduction function. +// +// Sample result for VALID padding mode: +// +// %init = arith.constant dense<...> : tensor +// %max_pool = "stablehlo.reduce"(%inp, %init) ["stablehlo.maximum"] +// {window_dimensions = ..., window_strides = ... } +// +template +class ConvertMaxPoolOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + Type element_type = + mlir::cast(op.getInput().getType()).getElementType(); + if (!element_type.isSignlessIntOrFloat()) return failure(); + tensorflow::Padding padding; + if (!GetPaddingFromString(op.getPadding().str(), &padding).ok()) + return failure(); + if (padding == tensorflow::Padding::EXPLICIT) { + return failure(); + } + Location loc = op.getLoc(); + stablehlo::ConstantOp init = GetScalarLimitConstOfType( + element_type, loc, hlo::kInfinityLowest, &rewriter); + + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + if (!input_ty) return failure(); + DenseIntElementsAttr paddings_attr = GetReduceWindowPaddingAsAttr( + input_ty.getShape(), op.getKsize(), op.getStrides(), op.getPadding(), + &rewriter); + auto reduce = stablehlo::ReduceWindowOp::create( + rewriter, loc, op.getType(), op.getInput(), init, + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getKsize()), &rewriter), + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getStrides()), &rewriter), + /*base_dilations=*/DenseI64ArrayAttr(), + /*window_dilations=*/DenseI64ArrayAttr(), paddings_attr); + BuildReduceBody(element_type, &reduce.getBody(), + &rewriter); + + rewriter.replaceOp(op, reduce.getResult(0)); + return success(); + } +}; + +using ConvertMaxPool2DOp = ConvertMaxPoolOp; +using ConvertMaxPool3DOp = ConvertMaxPoolOp; + +// Converts tf.Select (SelectV1) to stablehlo.select. It has optional +// broadcasting on the condition only. +class ConvertSelectOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SelectOp op, + PatternRewriter &rewriter) const override { + // This lowering only works on ranked types. + auto cond_type = + mlir::dyn_cast(op.getCondition().getType()); + auto then_type = + mlir::dyn_cast(op.getThenValue().getType()); + auto else_type = + mlir::dyn_cast(op.getElseValue().getType()); + if (!cond_type || !then_type || !else_type) { + return failure(); + } + + ImplicitLocOpBuilder b(op.getLoc(), rewriter); + Value cond_shape = b.createOrFold(op.getCondition()); + Value then_shape = b.createOrFold(op.getThenValue()); + Value else_shape = b.createOrFold(op.getElseValue()); + + // First check that the `then` and `else` shapes are the equal. + Value assumption = + b.createOrFold(ValueRange{then_shape, else_shape}); + // For a vector cond we also verify that the majormost dim of `then` matches + // the vector size. To do that split off the first dim of `then`. + bool needs_broadcast = cond_type.getRank() == 1 && then_type.getRank() != 1; + Value then_shape_split = then_shape; + if (needs_broadcast) { + Value const_one = arith::ConstantIndexOp::create(b, 1); + Type extent_first = shape::getExtentTensorType(b.getContext(), 1); + Type extent_second = + shape::getExtentTensorType(b.getContext(), then_type.getRank() - 1); + SmallVector then_split; + b.createOrFold(then_split, + TypeRange{extent_first, extent_second}, + then_shape, const_one); + then_shape_split = then_split[0]; + } + // If the condition is not a scalar, check that it matches the other shapes. + if (cond_type.getRank() > 0) { + Value eq_cstr = b.createOrFold( + ValueRange{cond_shape, then_shape_split}); + auto witness = shape::WitnessType::get(b.getContext()); + assumption = b.createOrFold( + witness, ValueRange{assumption, eq_cstr}); + } + auto result_type = mlir::cast(op.getResult().getType()); + auto assuming_op = + shape::AssumingOp::create(b, ArrayRef{result_type}, assumption); + + OpBuilder::InsertionGuard guard(b); + b.createBlock(&assuming_op.getDoRegion()); + + // Broadcast the cond if necessary. + Value cond = op.getCondition(); + if (needs_broadcast) { + Value result_extents = shape::ToExtentTensorOp::create( + b, GetExtentsTensorTypeFor(result_type), then_shape); + cond = stablehlo::DynamicBroadcastInDimOp::create( + b, + tensorflow::GetTypeFromTFTensorShape(result_type.getShape(), + b.getI1Type()), + cond, result_extents, + GetI64ArrayAttrForSeq(0, cond_type.getRank(), &b)); + } + Value select = stablehlo::SelectOp::create( + b, result_type, cond, op.getThenValue(), op.getElseValue()); + shape::AssumingYieldOp::create(b, select); + rewriter.replaceOp(op, {assuming_op.getResult(0)}); + return success(); + } +}; + +// Converts the tf.Slice op into stablehlo.real_dynamic_slice +// TODO(disc): To recover static special case's performance with folding and +// canonicalization. +class ConvertSliceOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SliceOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value input = op.getInput(); + Value begin_indices = op.getBegin(); + Value sizes = op.getSize(); + + auto input_ty = mlir::dyn_cast(input.getType()); + auto begin_type = mlir::dyn_cast(begin_indices.getType()); + auto size_type = mlir::dyn_cast(sizes.getType()); + + if (!input_ty || !begin_type || !size_type || + !begin_type.hasStaticShape() || !size_type.hasStaticShape() || + begin_type.getRank() != 1 || size_type.getRank() != 1) { + return failure(); + } + // TODO(disc): remove static shape check once folding/canonicalization func + // added + DenseIntElementsAttr size_attr; + if (matchPattern(op.getSize(), m_Constant(&size_attr))) { + return failure(); + } + + int rank = begin_type.getDimSize(0); + auto shape_scalar_type = begin_type.getElementType(); + Value one = arith::ConstantIndexOp::create(rewriter, loc, 1); + SmallVector stride_values(rank, one); + SmallVector end_values; + SmallVector begin_values; + end_values.reserve(rank); + for (int i = 0; i < rank; ++i) { + SmallVector indices; + indices.push_back(arith::ConstantIndexOp::create(rewriter, loc, i)); + auto begin_value = + tensor::ExtractOp::create(rewriter, loc, begin_indices, indices); + auto size_value = + tensor::ExtractOp::create(rewriter, loc, sizes, indices); + Value minus_one = arith::IndexCastOp::create( + rewriter, loc, shape_scalar_type, + arith::ConstantIndexOp::create(rewriter, loc, -1)); + auto is_minus_one = arith::CmpIOp::create( + rewriter, loc, arith::CmpIPredicate::eq, size_value, minus_one); + Value end_value = + arith::AddIOp::create(rewriter, loc, begin_value, size_value); + auto dim_value = arith::IndexCastOp::create( + rewriter, loc, shape_scalar_type, + tensor::DimOp::create(rewriter, loc, input, i)); + end_value = mlir::arith::SelectOp::create(rewriter, loc, is_minus_one, + dim_value, end_value); + auto end_value_casted = arith::IndexCastOp::create( + rewriter, loc, rewriter.getIndexType(), end_value); + end_values.push_back(end_value_casted); + + auto begin_value_casted = arith::IndexCastOp::create( + rewriter, loc, rewriter.getIndexType(), begin_value); + begin_values.push_back(begin_value_casted); + } + auto index_ty = rewriter.getIndexType(); + auto start_indices = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(begin_values.size())}, index_ty), + begin_values); + auto end_indices = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(end_values.size())}, index_ty), + end_values); + auto stride_indices = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(stride_values.size())}, index_ty), + stride_values); + + auto d_slice = stablehlo::RealDynamicSliceOp::create( + rewriter, loc, op.getOperation()->getResult(0).getType(), input, + start_indices, end_indices, stride_indices); + rewriter.replaceOp(op, d_slice.getOperation()->getResults()); + return success(); + } +}; + +static void BroadcastBatchMatMulV2Operands(Value lhs, Value rhs, Location loc, + Value *out_lhs, Value *out_rhs, + PatternRewriter *rewriter) { + // The dimension structure of the relevant operands to a tf.BatchMatMulV2 is: + // - lhs: [LHSBATCHDIMS..., LHSROWS, LHSCOLS] + // - rhs: [RHSBATCHDIMS..., RHSROWS, RHSCOLS] + // - result: [broadcast(LHSBATCHDIMS, RHSBATCHDIMS)..., LHSROWS, RHSCOLS] + // To perform the matmul, we need to first broadcast lhs and rhs to a common + // set of leading dimensions before doing the actual matmul. + // That's what the code below does. + // In particular, we populate out_lhs and out_rhs to have dimension structure: + // - out_lhs: [broadcast(LHSBATCHDIMS, RHSBATCHDIMS)..., LHSROWS, LHSCOLS] + // - out_rhs: [broadcast(LHSBATCHDIMS, RHSBATCHDIMS)..., RHSROWS, RHSCOLS] + // To do this, we need to calculate those output shapes, which involves + // slicing off the leading batch dims of each operand, broadcasting them, + // then concatenating the broadcasted leading dims back to the row/col dims. + // Finally, we create a TF::BroadcastTo op that does the actual broadcast. + + // TODO(silvasean): Reduce duplication across reified shape calculations and + // the static computation of output types needed to create ops. + Value lhs_shape = shape::ShapeOfOp::create(*rewriter, loc, lhs); + Value rhs_shape = shape::ShapeOfOp::create(*rewriter, loc, rhs); + Value const_neg2 = + arith::ConstantOp::create(*rewriter, loc, rewriter->getIndexAttr(-2)); + auto shape_type = shape::ShapeType::get(rewriter->getContext()); + auto lhs_splitted = shape::SplitAtOp::create( + *rewriter, loc, TypeRange{shape_type, shape_type}, lhs_shape, const_neg2); + auto rhs_splitted = shape::SplitAtOp::create( + *rewriter, loc, TypeRange{shape_type, shape_type}, rhs_shape, const_neg2); + auto lhs_type = mlir::cast(lhs.getType()); + auto rhs_type = mlir::cast(rhs.getType()); + // The last two dimensions are the matrix row/col dimensions. Don't broadcast + // them. + SmallVector result_batch_shape_compile_time_extents; + mlir::OpTrait::util::getBroadcastedShape( + lhs_type.getShape().drop_back(2), rhs_type.getShape().drop_back(2), + result_batch_shape_compile_time_extents); + auto result_batch_shape = + shape::BroadcastOp::create(*rewriter, loc, shape_type, + lhs_splitted.getHead(), rhs_splitted.getHead(), + /*error=*/nullptr); + // Lambda which handles the broadcasting of one side to the common + // leading-batch dimensions. + auto broadcast_one_side = [&](Value side, RankedTensorType type, + Value tail_shape, Value *out_side) { + ArrayRef matrix_dims = type.getShape().take_back(2); + auto result_shape = result_batch_shape_compile_time_extents; + result_shape.append(matrix_dims.begin(), matrix_dims.end()); + auto result_type = tensorflow::GetTypeFromTFTensorShape( + result_shape, type.getElementType()); + auto shape = shape::ConcatOp::create(*rewriter, loc, shape_type, + result_batch_shape, tail_shape); + auto shape_tensor = shape::ToExtentTensorOp::create( + *rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(result_shape.size())}, + rewriter->getIndexType()), + shape); + *out_side = TF::BroadcastToOp::create(*rewriter, loc, result_type, side, + shape_tensor); + }; + broadcast_one_side(lhs, lhs_type, lhs_splitted.getTail(), out_lhs); + broadcast_one_side(rhs, rhs_type, rhs_splitted.getTail(), out_rhs); +} + +class ConvertBatchMatMulV2Op : public OpRewritePattern { + public: + // TODO(hinsu): Legalize this op to Einsum op. HLO Einsum op needs to be moved + // to CHLO and it is missing legalization to StableHLO. Once that is done, + // this pattern's benefit can be changed back to one as well as the fallback + // lowering pattern for the op can be removed. + // + // Set benefit of this pattern to zero to prefer the fallback pattern when + // available and applicable. That pattern avoids broadcast on operands and is + // therefore faster. + // + // Native legalization for BatchMatMulV3 needs to be added as well. + explicit ConvertBatchMatMulV2Op(MLIRContext *context) + : OpRewritePattern(context, /*benefit=*/0) {} + + LogicalResult matchAndRewrite(TF::BatchMatMulV2Op op, + PatternRewriter &rewriter) const override { + Value lhs = op.getX(); + Value rhs = op.getY(); + auto lhs_type = mlir::dyn_cast(lhs.getType()); + auto rhs_type = mlir::dyn_cast(rhs.getType()); + if (!lhs_type || !rhs_type) return failure(); + if (mlir::isa(lhs_type.getElementType()) && op.getAdjX()) { + lhs = TF::ConjOp::create(rewriter, op.getLoc(), lhs_type, lhs); + } + if (mlir::isa(rhs_type.getElementType()) && op.getAdjY()) { + rhs = TF::ConjOp::create(rewriter, op.getLoc(), rhs_type, rhs); + } + + // Broadcast both operands. + BroadcastBatchMatMulV2Operands(lhs, rhs, op.getLoc(), &lhs, &rhs, + &rewriter); + lhs_type = mlir::cast(lhs.getType()); + rhs_type = mlir::cast(rhs.getType()); + assert(lhs_type.getRank() == rhs_type.getRank()); + int64_t rank = lhs_type.getRank(); + auto batch_dimensions = llvm::to_vector<4>(llvm::seq(0, rank - 2)); + auto lhs_contracting_dimensions = llvm::to_vector<4>( + llvm::ArrayRef({op.getAdjX() ? rank - 2 : rank - 1})); + auto rhs_contracting_dimensions = llvm::to_vector<4>( + llvm::ArrayRef({op.getAdjY() ? rank - 1 : rank - 2})); + auto dimension_numbers = stablehlo::DotDimensionNumbersAttr::get( + rewriter.getContext(), + /*lhs_batching_dimensions=*/batch_dimensions, + /*rhs_batching_dimensions=*/batch_dimensions, + /*lhs_contracting_dimensions=*/lhs_contracting_dimensions, + /*rhs_contracting_dimensions=*/rhs_contracting_dimensions); + // TODO(silvasean): Emit shape checks for contracting dimensions. + // (The batch dimensions are checked by the broadcasting logic) + rewriter.replaceOpWithNewOp( + op, op.getType(), lhs, rhs, dimension_numbers, + /*precision_config=*/GetPrecisionConfig(&rewriter), + /*algorithm=*/stablehlo::DotAlgorithmAttr{}); + return success(); + } +}; + +// Converts the tf.Split op into a series of HLO slice ops when the tensor to be +// split has fully static shape and the dimension to split is a constant. +// +// The main logic of this pattern is to calculate the index start and end range +// for each slice. And this happens only on the dimension to be split; for all +// other dimensions, all resultant slices' index start and end range covers the +// input tensor's full range. Strides for all resultant slices are all one. +// +// For example, the following source IR: +// +// %dim = "tf.Const"() {value = dense<1> : tensor} : () -> tensor +// %0:3 = "tf.Split"(%dim, %input) : (tensor, tensor<4x6xf32>) -> +// (tensor<4x2xf32>, tensor<4x2xf32>, tensor<4x2xf32>) +// +// will be converted into: +// +// %0 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x2xf32> +// %1 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x2xf32> +// %2 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x2xf32> +// TODO(antiagainst): consider lowering into TF ops so the pattern can be more +// applicable. +class ConvertSplitOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SplitOp op, + PatternRewriter &rewriter) const override { + // We can only split inputs that have fully static shape. + auto input_type = mlir::dyn_cast(op.getValue().getType()); + if (!input_type || !input_type.hasStaticShape()) return failure(); + + // We can only match when the split dimension is a constant scalar. + DenseIntElementsAttr split_dim_attr; + if (!matchPattern(op.getSplitDim(), m_Constant(&split_dim_attr))) + return failure(); + + // Get the dimension we are splitting at. Offset properly if it's negative. + int64_t input_rank = input_type.getRank(); + int64_t dim_index = (*split_dim_attr.begin()).getSExtValue(); + if (dim_index < 0) dim_index += input_rank; + + // Calculate the dimension size for each slice along the split dimension. + int64_t input_dim_size = input_type.getDimSize(dim_index); + + int64_t num_splits = op.getNumResults(); + int64_t slice_size = input_dim_size / num_splits; + + // Get each slice's type. + auto slice_shape = llvm::to_vector<4>(input_type.getShape()); + slice_shape[dim_index] = slice_size; + Type slice_type = tensorflow::GetTypeFromTFTensorShape( + slice_shape, input_type.getElementType()); + + // Parameters for constructing each slice. + SmallVector begin_indices(input_rank, 0); + auto end_indices = llvm::to_vector<4>(input_type.getShape()); + SmallVector strides(input_rank, 1); + + // All HLO slice results used to replace the original tf.Split op. + SmallVector slices; + slices.reserve(num_splits); + + for (int i = 0; i < num_splits; ++i) { + begin_indices[dim_index] = i * slice_size; + end_indices[dim_index] = (i + 1) * slice_size; + slices.push_back(stablehlo::SliceOp::create( + rewriter, op.getLoc(), slice_type, op.getValue(), + GetI64ArrayAttr(begin_indices, &rewriter), + GetI64ArrayAttr(end_indices, &rewriter), + GetI64ArrayAttr(strides, &rewriter))); + } + + rewriter.replaceOp(op, slices); + return success(); + } +}; + +// Converts the tf.Split op into a series of stablehlo.real_dynamic_slice ops +// the dimension to split is a constant. +// TODO(disc): To recover static special case's performance with folding and +// canonicalization. delete ConvertSplitOp +class ConvertSplitOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SplitOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value input = op.getValue(); + auto input_type = mlir::dyn_cast(input.getType()); + if (!input_type) return failure(); + + // TODO(disc): remove static shape check once folding/canonicalization func + // added and ConvertSplitOp deleted. Calculate the dimension size for each + // slice along the split dimension. We are splitting along the dynamic + // dimension, or using static pattern transform + if (input_type.hasStaticShape()) return failure(); + + // We can only match when the split dimension is a constant scalar. + DenseIntElementsAttr split_dim_attr; + if (!matchPattern(op.getSplitDim(), m_Constant(&split_dim_attr))) + return failure(); + + // Get the dimension we are splitting at. Offset properly if it's negative. + int64_t input_rank = input_type.getRank(); + int64_t dim_index = (*split_dim_attr.begin()).getSExtValue(); + if (dim_index < 0) dim_index += input_rank; + + Value input_dim_size = + tensor::DimOp::create(rewriter, loc, input, dim_index); + // Calculate the dimension size for each slice along the split dimension. + int num_splits = op.getNumResults(); + Value num_splits_value = arith::ConstantOp::create( + rewriter, loc, rewriter.getIndexAttr(num_splits)); + Value slice_size = + arith::DivSIOp::create(rewriter, loc, input_dim_size, num_splits_value); + + Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0); + Value one = arith::ConstantIndexOp::create(rewriter, loc, 1); + + SmallVector begin_indices(input_rank, zero); + SmallVector end_indices; + end_indices.reserve(input_rank); + SmallVector strides(input_rank, one); + for (int i = 0; i < input_rank; ++i) { + end_indices.push_back(tensor::DimOp::create(rewriter, loc, input, i)); + } + + // All HLO d_slice results used to replace the original tf.Split op. + SmallVector slices; + slices.reserve(num_splits); + + for (int i = 0; i < num_splits; ++i) { + begin_indices[dim_index] = arith::MulIOp::create( + rewriter, loc, slice_size, + arith::ConstantIndexOp::create(rewriter, loc, i)); + end_indices[dim_index] = arith::MulIOp::create( + rewriter, loc, slice_size, + arith::ConstantIndexOp::create(rewriter, loc, i + 1)); + + Type index_ty = rewriter.getIndexType(); + auto begin_value = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(begin_indices.size())}, index_ty), + begin_indices); + auto end_value = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(end_indices.size())}, index_ty), + end_indices); + auto stride_value = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(strides.size())}, index_ty), + strides); + slices.push_back(stablehlo::RealDynamicSliceOp::create( + rewriter, loc, op.getOperation()->getResult(i).getType(), input, + begin_value, end_value, stride_value)); + } + + rewriter.replaceOp(op, slices); + return success(); + } +}; + +// Converts the tf.SplitV op into a series of HLO slice ops when the tensor to +// be split has fully static shape and the dimension to split and split sizes +// are constants. +// +// This is similar to the conversion for tf.Split op other than that the size of +// each chunk on the dimension to split is explicitly given as an op operand +// and they are not necessarily the same. +// +// For example, given the following IR: +// +// %split_sizes = "tf.Const"() {value = dense<[1, -1, 3]> : tensor<3xi32>} +// %split_dim = "tf.Const"() {value = dense<1> : tensor} +// %0:3 = "tf.SplitV"(%input, %split_sizes, %split_dim) : +// (tensor<4x6xf32>, tensor<3xi32>, tensor) -> +// (tensor<4x1xf32>, tensor<4x2xf32>, tensor<4x3xf32>) +// +// We will generate slices following slices: +// %0 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x1xf32> +// %1 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x2xf32> +// %2 = "stablehlo.slice"(%input) { +// limit_indices = array, +// start_indices = array, +// strides = array} : +// (tensor<4x6xf32>) -> tensor<4x3xf32> +class ConvertSplitVOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SplitVOp op, + PatternRewriter &rewriter) const override { + // We can only split inputs that have fully static shape. + // TODO(b/145731001): enhance to support dynamic-shaped inputs. + auto input_type = mlir::dyn_cast(op.getValue().getType()); + if (!input_type || !input_type.hasStaticShape()) return failure(); + + // We can only match when the split dimension is a constant scalar. + DenseIntElementsAttr split_dim_attr; + if (!matchPattern(op.getSplitDim(), m_Constant(&split_dim_attr))) + return failure(); + + // We can only match when the split sizes is a constant int vector. + DenseIntElementsAttr split_sizes_attr; + if (!matchPattern(op.getSizeSplits(), m_Constant(&split_sizes_attr))) + return failure(); + + // Get each chunck's size along the dimension to split. It may contain + // dynamic sizes and we need to update it if so. + SmallVector split_sizes; + int64_t total_dim_size = 0; // Total dimension size assigned to splits + std::optional dynamic_dim_index; + split_sizes.reserve( + mlir::cast(split_sizes_attr.getType()).getNumElements()); + for (const auto &dim : llvm::enumerate(split_sizes_attr)) { + int64_t dim_val = dim.value().getSExtValue(); + split_sizes.push_back(dim_val); + if (dim_val == -1) { + // We cannot have more than one dynamic dimension. + assert(!dynamic_dim_index && "invalid split sizes"); + dynamic_dim_index = dim.index(); + } else { + total_dim_size += dim_val; + } + } + + // Get the dimension we are splitting at. Offset properly if it's negative. + int64_t input_rank = input_type.getRank(); + int64_t dim_index = (*split_dim_attr.begin()).getSExtValue(); + if (dim_index < 0) dim_index += input_rank; + + int64_t input_dim_size = input_type.getDimSize(dim_index); + assert(((dynamic_dim_index && total_dim_size <= input_dim_size) || + (!dynamic_dim_index && total_dim_size == input_dim_size)) && + "invalid split sizes"); + + // Update the dynamic dimension with calculated concrete size. + if (dynamic_dim_index) + split_sizes[*dynamic_dim_index] = input_dim_size - total_dim_size; + + // Parameters for constructing each slice. + SmallVector begin_indices(input_rank, 0); + auto end_indices = llvm::to_vector<4>(input_type.getShape()); + SmallVector strides(input_rank, 1); + + // All HLO slice results used to replace the original tf.Split op. + SmallVector slices; + slices.reserve(op.getNumResults()); + + for (int i = 0, end = op.getNumResults(); i < end; ++i) { + end_indices[dim_index] = begin_indices[dim_index] + split_sizes[i]; + slices.push_back( + stablehlo::SliceOp::create(rewriter, op.getLoc(), op.getValue(), + GetI64ArrayAttr(begin_indices, &rewriter), + GetI64ArrayAttr(end_indices, &rewriter), + GetI64ArrayAttr(strides, &rewriter))); + // Prepare the begin indice for the next slice. + begin_indices[dim_index] = end_indices[dim_index]; + } + + rewriter.replaceOp(op, slices); + return success(); + } +}; + +// Converts StridedSlice op to HLO Slice op along with Reverse op to handle +// negative strides and Reshape op to update the output shape. Indices and +// strides operands are converted to attributes with non-negative indexing. +// +// If the begin input is not a compile time constant, the begin input needs to +// be sliced and the slice needs to be lowered to stablehlo.DynamicSlice. In +// this case, strides must have a known value of 1 (otherwise we have +// insufficient information to conform to XLA's op semantics). +// +// For example with an op like following, +// tf.StridedSlice(%input, %begin, %end, %strides) {shrink_axis_mask = 1} +// : tensor -> tensor +// +// If the %begin input is constant, output would be: +// %reversed = "stablehlo.Reverse" (%input) {dimensions = ...} +// %sliced = "stablehlo.Slice" (%input) +// {start_indices = ..., limit_indices = ..., strides = ...} +// %output = "stablehlo.Reshape" (%sliced) : tensor<1xPxf32> -> tensor +// +class ConvertStridedSliceOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult rewriteWithConstantBegin(TF::StridedSliceOp op, + ArrayRef begin_indices, + ArrayRef end_indices, + ArrayRef strides, + RankedTensorType input_ty, + PatternRewriter &rewriter) const { + SmallVector hlo_begin_indices, hlo_end_indices, hlo_strides, + dims_to_reverse; + int64_t input_rank = input_ty.getRank(); + ArrayRef input_shape = input_ty.getShape(); + hlo_begin_indices.reserve(input_rank); + hlo_end_indices.reserve(input_rank); + hlo_strides.reserve(input_rank); + + int64_t indices_elements = begin_indices.size(); + if (input_rank < indices_elements) return failure(); + + // Convert from TensorFlow negative or out of range indices and strides + // values to legal HLO Slice attributes. + for (int i = 0, e = indices_elements; i != e; i++) { + int64_t begin = begin_indices[i]; + int64_t end = end_indices[i]; + int64_t stride = strides[i]; + + if (stride < 0) { + // Negative stride means that the output values are computed starting + // from end until begin. Mark the dimension for reversal before slice + // and compute indices for the reversed input. + dims_to_reverse.push_back(i); + begin = (input_shape[i] - 1) - begin; + end = (input_shape[i] - 1) - end; + stride = -stride; + } + + // Unlike TensorFlow, HLO requires begin and end values to be within + // range. + begin = std::max(int64_t(0), begin); + end = std::max(begin, end); + end = std::min(end, input_shape[i]); + + hlo_begin_indices.push_back(begin); + hlo_end_indices.push_back(end); + hlo_strides.push_back(stride); + } + + Location loc = op.getLoc(); + Value input = op.getInput(); + if (!dims_to_reverse.empty()) + input = stablehlo::ReverseOp::create( + rewriter, loc, input_ty, op.getInput(), + GetI64ArrayAttr(dims_to_reverse, &rewriter)); + auto sliced = stablehlo::SliceOp::create( + rewriter, loc, input, GetI64ArrayAttr(hlo_begin_indices, &rewriter), + GetI64ArrayAttr(hlo_end_indices, &rewriter), + GetI64ArrayAttr(hlo_strides, &rewriter)); + + // Reshape slice result so that the shape is updated depending on + // 'new_axis_mask' or 'shrink_axis_mask' attributes. + rewriter.replaceOpWithNewOp(op, op.getType(), sliced); + return success(); + } + + LogicalResult rewriteWithUnknownBegin(TF::StridedSliceOp op, + RankedTensorType input_ty, + RankedTensorType result_ty, + PatternRewriter &rewriter) const { + // If begin and end values are dynamic, we can only support this lowering + // if strides are a known value of 1. + DenseIntElementsAttr sparse_strides_attr; + if (!matchPattern(op.getStrides(), m_Constant(&sparse_strides_attr))) { + return rewriter.notifyMatchFailure( + op, + "requires that strides are known when begin/end values are dynamic"); + } + SmallVector strides; + int64_t stride_value; + for (const APInt &stride : sparse_strides_attr) { + if ((stride_value = stride.getSExtValue()) != 1) { + return rewriter.notifyMatchFailure(op, + "requires that strides are all 1 " + "when begin/end values are dynamic"); + } + strides.push_back(stride_value); + } + + ArrayRef input_shape = input_ty.getShape(); + int last_dim = std::max(static_cast(input_shape.size()) - 1, 0); + + // When begin/end values are dynamic, the ellipsis mask, if set, must refer + // to the last dimension. + int ellipsis_mask = op.getEllipsisMask(); + if (!(ellipsis_mask == 0 || ellipsis_mask == (1 << last_dim))) + return rewriter.notifyMatchFailure( + op, + "requires that ellipsis_mask, if set, refer to the last dimension of " + "input (when begin/end values are dynamic)"); + + // In this case where the begin and end values are dynamic, we only support + // cases where the number of output elements has to be equal to the number + // of input elements that are sliced. Each dimension is either sliced fully + // or sliced with a size of one. + int output_elements = result_ty.getNumElements(); + int input_elements_sliced = 1; + + // Begin must be a ranked, 1-dimensional tensor: This is checked by the + // verifier. + int64_t slicing_dim_size = + mlir::cast(op.getBegin().getType()).getDimSize(0); + uint64_t begin_mask = op.getBeginMask(); + uint64_t end_mask = op.getEndMask(); + const int input_rank = input_shape.size(); + for (int d = 0; d < input_rank; ++d) { + // Each dimension is either sliced fully or has size of one. + if ((((begin_mask >> d) & 1) && ((end_mask >> d) & 1)) || + (d >= slicing_dim_size)) { + input_elements_sliced *= input_shape[d]; + } + } + if (input_elements_sliced != output_elements) { + return rewriter.notifyMatchFailure( + op, + "requires the number of output elements to be equal to the number of " + "input elements sliced (when begin/end values are dynamic)"); + } + + SmallVector slice_begin_indices; + // For the dimensions that are to be sliced, all have slice sizes of 1. + SmallVector slice_sizes; + auto begin_element_ty = + mlir::cast(op.getBegin().getType()).getElementType(); + // Scalar tensor type. + TensorType type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, begin_element_ty); + Location loc = op.getLoc(); + auto zero = GetScalarConstOfType(begin_element_ty, loc, 0, &rewriter); + for (int d = 0; d < input_rank; ++d) { + if ((((begin_mask >> d) & 1) && ((end_mask >> d) & 1)) || + (d >= slicing_dim_size)) { + slice_begin_indices.push_back(zero); + slice_sizes.push_back(input_shape[d]); + continue; + } + + auto index = stablehlo::SliceOp::create( + rewriter, loc, op.getBegin(), GetI64ArrayAttr({d}, &rewriter), + GetI64ArrayAttr({d + 1}, &rewriter), GetI64ArrayAttr({1}, &rewriter)); + // Convert index to scalar. + auto reshaped_index = + stablehlo::ReshapeOp::create(rewriter, loc, type, index); + // If the index is negative, wrap it around with dimension size. + auto index_negative = + TF::LessOp::create(rewriter, loc, reshaped_index, zero); + auto input_val = GetScalarConstOfType(begin_element_ty, loc, + input_shape[d], &rewriter); + auto wrapped_index = + TF::AddV2Op::create(rewriter, loc, input_val, reshaped_index); + auto final_index = stablehlo::SelectOp::create( + rewriter, loc, type, index_negative, wrapped_index, reshaped_index); + slice_begin_indices.push_back(final_index); + slice_sizes.push_back(1); + } + + auto slice_sizes_attr = GetI64ArrayAttr(slice_sizes, &rewriter); + auto sliced_type = tensorflow::GetTypeFromTFTensorShape( + slice_sizes, op.getType().getElementType()); + // This must be an xla DynamicSlice op due to the inputs that aren't + // constant. + auto sliced = stablehlo::DynamicSliceOp::create( + rewriter, loc, sliced_type, op.getInput(), slice_begin_indices, + slice_sizes_attr); + + // Reshape slice result so that the shape is updated depending on + // 'new_axis_mask' or 'shrink_axis_mask' attributes. + rewriter.replaceOpWithNewOp(op, op.getType(), sliced); + return success(); + } + + LogicalResult matchAndRewrite(TF::StridedSliceOp op, + PatternRewriter &rewriter) const override { + // Input shape needs to be static to convert negative indices in TensorFlow + // to absolute indices required by HLO. + // + // TODO(hinsu): Relax this constraint for ops without negative indices and + // strides. + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + if (!input_ty || !input_ty.hasStaticShape()) return failure(); + + // Output shape needs to be static to apply 'new_axis_mask' or + // 'shrink_axis_mask' by reshaping tensor after slice. + // + // TODO(hinsu): Relax this constraint for ops without the above masks. + auto result_ty = mlir::dyn_cast(op.getType()); + if (!result_ty || !result_ty.hasStaticShape()) return failure(); + + DenseIntElementsAttr sparse_begin_attr, sparse_end_attr; + if (!matchPattern(op.getBegin(), m_Constant(&sparse_begin_attr)) || + !matchPattern(op.getEnd(), m_Constant(&sparse_end_attr))) { + return rewriteWithUnknownBegin(op, input_ty, result_ty, rewriter); + } + + SmallVector begin_indices, end_indices, strides; + if (!op.GetSlicedBoundRanges(&begin_indices, &end_indices, &strides)) { + return failure(); + } + return rewriteWithConstantBegin(op, begin_indices, end_indices, strides, + input_ty, rewriter); + } +}; + +// Converts tf.StridedSliceGrad to HLO reshape, reverse and padding ops. +// +// tf.StridedSlice is taking slice of the input tensor. tf.StridedSliceGrad does +// the reverse: it propagates the graident for the sliced tensor to the original +// input tensor by doing padding with zeros. The main logic is calculating the +// indices and strides for padding. +class ConvertStridedSliceGradOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::StridedSliceGradOp op, + PatternRewriter &rewriter) const override { + // We need constant input shape to perform padding calculations later. + DenseIntElementsAttr input_shape_attr; + if (!matchPattern(op.getShape(), m_Constant(&input_shape_attr))) + return failure(); + + // We also need constant begin/end indices and strides to perform padding + // calculations. + // Bounded shape after performing strided slice + SmallVector shape; + // Bounded begin, end, and strides for strided slice + SmallVector begin_indices, end_indices, strides; + if (!op.GetSlicedShapeAndBoundRanges(&shape, &begin_indices, &end_indices, + &strides)) + return failure(); + + Value grad = op.getDy(); + Type element_type = mlir::cast(grad.getType()).getElementType(); + + // Perform reshape to undo any new/shrink axes done by strided slice. + grad = stablehlo::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape(shape, element_type), grad); + + SmallVector padding_low, padding_high, padding_interm; + SmallVector dims_to_reverse; + padding_low.reserve(shape.size()); + padding_high.reserve(shape.size()); + padding_interm.reserve(shape.size()); + + // Prepare padding parameters for each dimension. + for (int i = 0, e = shape.size(); i < e; ++i) { + int64_t input_dim = (*(input_shape_attr.begin() + i)).getSExtValue(); + if (strides[i] > 0) { + padding_low.push_back(begin_indices[i]); + padding_interm.push_back(strides[i] - 1); + + // Pad the upper dimension up to the expected input shape. It's not + // sufficient simply to use end_indices[i] to compute the padding in + // cases where the stride does not divide evenly into the interval + // between begin_indices[i] and end_indices[i]. + int64_t size = + padding_low[i] + shape[i] + (shape[i] - 1) * padding_interm[i]; + padding_high.push_back(input_dim - size); + } else { + dims_to_reverse.push_back(i); + padding_high.push_back(input_dim - begin_indices[i] - 1); + padding_interm.push_back(-strides[i] - 1); + + // Pad the lower dimension up to the expected input shape. + int64_t size = + padding_high[i] + shape[i] + (shape[i] - 1) * padding_interm[i]; + padding_low.push_back(input_dim - size); + } + } + + if (!dims_to_reverse.empty()) { + grad = stablehlo::ReverseOp::create( + rewriter, op.getLoc(), grad.getType(), grad, + GetI64ArrayAttr(dims_to_reverse, &rewriter)); + } + + auto zero = GetScalarConstOfType(element_type, op.getLoc(), 0, &rewriter); + rewriter.replaceOpWithNewOp( + op, op.getType(), grad, zero, GetI64ArrayAttr(padding_low, &rewriter), + GetI64ArrayAttr(padding_high, &rewriter), + GetI64ArrayAttr(padding_interm, &rewriter)); + return success(); + } +}; + +/// Converts the RangeOp tensorflow op to a stablehlo.iota op with a scaling and +/// offset applied to generate the range values. The output tensor needs to +/// have a static shape. +/// +/// For example an op like the following: +/// %result = "tf.Range"(%start, %limit, %delta) {Tidx = "tfdtype$DT_FLOAT"} +/// : (tensor, tensor, tensor) -> tensor<5xf32> +/// +/// Output would be: +/// %iota = "stablehlo.iota"() {iota_dimension = 0 : i64} : () -> +/// tensor<5xf32> %scaled = "stablehlo.multiply"(%iota, %delta) +/// {broadcast_dimensions = dense<[]> : tensor<0xi64>} : +/// (tensor<5xf32>, tensor) -> tensor<5xf32> +/// %result = "stablehlo.add"(%scaled, %offset) +/// {broadcast_dimensions = dense<[]> : tensor<0xi64>} : +/// (tensor<5xf32>, tensor) -> tensor<5xf32> +/// +/// Implementation is defined in C++ due to no type interface for the iota op. +class ConvertRangeOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::RangeOp op, + PatternRewriter &rewriter) const override { + auto result = op.getResult(); + auto result_type = result.getType(); + if (!mlir::cast(result_type).hasStaticShape()) { + return failure(); + } + + auto iota = stablehlo::IotaOp::create(rewriter, op.getLoc(), result_type, + rewriter.getI64IntegerAttr(0)); + auto scaled = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), result_type, iota, op.getDelta(), + hlo::getBroadcastDimensionsAttr(&rewriter, iota, op.getDelta())); + rewriter.replaceOpWithNewOp( + op, result_type, scaled, op.getStart(), + hlo::getBroadcastDimensionsAttr(&rewriter, scaled, op.getStart())); + return success(); + } +}; + +// Converts RangeOp for cases with the length is a dynamic value. The shape of +// the resulting tensor computed, then the start and delta is used with the +// dynamic_iota value to compute the final range value. +// +// For example, the resulting range op value: +// %range = "tf.range"(%start, %limit, %delta) +// +// Is converted to the following. +// %start + %delta * iota(ceil(abs((%limit - %start) / %delta)) +// +// Implementation is defined in C++ due to the complicated type behavior. +class ConvertDynamicRangeOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::RangeOp op, + PatternRewriter &rewriter) const override { + auto result = op.getResult(); + auto result_type = mlir::cast(result.getType()); + if (result_type.hasStaticShape()) { + return failure(); + } + + Value start = op.getStart(); + Value delta = op.getDelta(); + Value limit = op.getLimit(); + + // To compute the length we need to use floating point calculations so that + // ceil can be computed for the number of steps. + auto compute_element_type = + mlir::isa(getElementTypeOrSelf(start.getType())) + ? getElementTypeOrSelf(start.getType()) + : rewriter.getF64Type(); + auto compute_type = tensorflow::GetTypeFromTFTensorShape( + mlir::cast(limit.getType()).getShape(), + compute_element_type); + + // Compute the length of the sequence we are going to need. This includes + // some conversion to float for the operations. + // + // %size = ceil(abs((%limit - %start) / %delta)) + auto range = + stablehlo::SubtractOp::create(rewriter, op.getLoc(), limit, start); + auto abs = stablehlo::AbsOp::create(rewriter, op.getLoc(), range); + + // Delta is not necessarily the same type as start and limit. + auto abs_cast = + stablehlo::ConvertOp::create(rewriter, op.getLoc(), compute_type, abs); + auto delta_cast = stablehlo::ConvertOp::create(rewriter, op.getLoc(), + compute_type, delta); + + // Compute the total number of integer steps and convert to the HLO + // dimension tensor. + auto normalized = + stablehlo::DivOp::create(rewriter, op.getLoc(), abs_cast, delta_cast); + auto ceil = stablehlo::CeilOp::create(rewriter, op.getLoc(), normalized); + auto steps = stablehlo::ConvertOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getI64Type()), ceil); + auto reshape = stablehlo::ReshapeOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape({1}, rewriter.getI64Type()), + steps); + + // Using the resulting length compute the correct range value: + // + // %range = %start + %delta * iota(%size) + auto out_scalar_type = tensorflow::GetTypeFromTFTensorShape( + {}, getElementTypeOrSelf(result_type)); + auto start_out_cast = stablehlo::ConvertOp::create(rewriter, op.getLoc(), + out_scalar_type, start); + auto delta_out_cast = stablehlo::ConvertOp::create(rewriter, op.getLoc(), + out_scalar_type, delta); + + auto iota = stablehlo::DynamicIotaOp::create(rewriter, op.getLoc(), + result_type, reshape, + rewriter.getI64IntegerAttr(0)); + auto scaled = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), result_type, iota, delta_out_cast, + hlo::getBroadcastDimensionsAttr(&rewriter, iota, delta_cast)); + rewriter.replaceOpWithNewOp( + op, result_type, scaled, start_out_cast, + hlo::getBroadcastDimensionsAttr(&rewriter, scaled, start_out_cast)); + return success(); + } +}; + +DenseI64ArrayAttr ConvertAxisAttr(Value val, ElementsAttr attr, + Builder *builder) { + auto int_attr = mlir::cast(attr); + auto type = mlir::cast(val.getType()); + + SmallVector axis; + axis.reserve(int_attr.getNumElements()); + + int64_t rank = type.getRank(); + for (auto val : int_attr.getValues()) { + axis.push_back((val.getSExtValue() + rank) % rank); + } + + return builder->getDenseI64ArrayAttr(axis); +} + +/// Converts the LinSpace tensorflow op to a stablehlo.iota op with a scaling +/// and offset applied to generate the linspace values. The output tensor needs +/// to have a static shape. The implementation is defined in C++ because there +/// is no type inference for the iota op. +class ConvertLinSpaceOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::LinSpaceOp op, + PatternRewriter &rewriter) const override { + auto result = op.getResult(); + auto result_type = mlir::dyn_cast(result.getType()); + if (!result_type || !result_type.hasStaticShape()) { + return failure(); + } + + DenseIntElementsAttr num_attr; + if (!matchPattern(op.getNum(), m_Constant(&num_attr))) { + return rewriter.notifyMatchFailure(op, "Num must be a constant scalar"); + } + + if (num_attr.begin() == num_attr.end()) { + return rewriter.notifyMatchFailure(op, "Num must not be empty"); + } + int64_t num = (*num_attr.begin()).getSExtValue(); + + // Calculate the scaling that needs to be applied to the iota. + auto step_numerator = chlo::BroadcastSubOp::create( + rewriter, op.getLoc(), op.getStart().getType(), op.getStop(), + op.getStart(), + hlo::getBroadcastDimensionsAttr(&rewriter, op.getStop(), + op.getStart())); + Value step_denominator = stablehlo::ConvertOp::create( + rewriter, op.getLoc(), op.getNum(), result_type.getElementType()); + if (num > 1) { + Value one = GetScalarConstOfType(result_type.getElementType(), + op.getLoc(), 1, &rewriter); + step_denominator = chlo::BroadcastSubOp::create( + rewriter, op.getLoc(), step_denominator.getType(), step_denominator, + one, + hlo::getBroadcastDimensionsAttr(&rewriter, step_denominator, one)); + } + auto step = chlo::BroadcastDivOp::create( + rewriter, op.getLoc(), step_numerator.getType(), step_numerator, + step_denominator, + hlo::getBroadcastDimensionsAttr(&rewriter, step_numerator, + step_denominator)); + + // Scale the iota and add the offset. + auto iota = stablehlo::IotaOp::create(rewriter, op.getLoc(), result_type, + rewriter.getI64IntegerAttr(0)); + auto scaled = chlo::BroadcastMulOp::create( + rewriter, op.getLoc(), result_type, iota, step, + hlo::getBroadcastDimensionsAttr(&rewriter, iota, step)); + rewriter.replaceOpWithNewOp( + op, result_type, scaled, op.getStart(), + hlo::getBroadcastDimensionsAttr(&rewriter, scaled, op.getStart())); + return success(); + } +}; + +/// Converts a generic OpTy tensorflow op to a stablehlo.reduce op over +/// ReductionOp. +/// `is_accumulation` controls whether it uses higher precision for the actual +/// reduction. This is set to false for ops like max where there is no precision +/// concerns. +// +// The Derived class should have a static method to return the initial value to +// use for reduction: +// static Value GetInitialValue(Type reduce_element_type, Location loc, +// PatternRewriter *rewriter); +// The reduce_element_type is guaranteed to be a float, int, or complex type +// suitable for use with GetScalarConstOfType or GetScalarLimitConstOfType. +template +class GenericConvertReductionOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + // TODO(b/141785544): Update this to not require ranked shapes. + // Input shape needs to be ranked to convert negative indices in TensorFlow + // to absolute indices required by HLO. + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + if (!input_ty) return failure(); + ArrayRef input_shape = input_ty.getShape(); + + DenseIntElementsAttr dimensions; + if (!matchPattern(op.getReductionIndices(), m_Constant(&dimensions))) + return failure(); + + // Build the final shape from input_shape and dimensions using a bitmap + // to mark the reduced dimensions. + SmallVector reduced_dimensions_bitmap(input_shape.size(), false); + SmallVector xla_dimensions; + for (const APInt &index_raw : dimensions.getValues()) { + int64_t index = index_raw.getSExtValue(); + int64_t rank = input_shape.size(); + if ((index < -rank || index >= rank)) return failure(); + index = (index + rank) % rank; + reduced_dimensions_bitmap[index] = true; + xla_dimensions.push_back(index); + } + + Location loc = op.getLoc(); + Type element_type = input_ty.getElementType(); + + // Only float, int, and complex types are currently supported. + if (!mlir::isa(element_type) && + !mlir::isa(element_type) && + !mlir::isa(element_type)) { + return rewriter.notifyMatchFailure( + op, "element type must be float, int, or complex type"); + } + + // Convert to an accumulation type to not lose precision when doing + // repeated arithmetic operations. + Type reduce_element_type = + is_accumulation ? GetAccumulationType(element_type) : element_type; + auto casted_input = stablehlo::ConvertOp::create( + rewriter, loc, op.getInput(), reduce_element_type); + + // Each reduction op can have a different initial value. + Value init = Derived::GetInitialValue(reduce_element_type, loc, &rewriter); + + auto reduction = stablehlo::ReduceOp::create( + rewriter, loc, casted_input.getResult(), init, + GetI64ArrayAttr(xla_dimensions, &rewriter), reduce_element_type); + BuildReduceBody(reduce_element_type, &reduction.getBody(), + &rewriter); + Value result = reduction.getResult(0); + + // The mean op needs to divide by the product of the reduced dimensions. + if (std::is_same::value) { + Value in_shape = shape::ShapeOfOp::create(rewriter, loc, op.getInput()); + Value divisor_count = arith::ConstantIndexOp::create(rewriter, loc, 1); + for (size_t i = 0; i < input_shape.size(); ++i) { + if (reduced_dimensions_bitmap[i]) { + Value index = arith::ConstantIndexOp::create(rewriter, loc, i); + auto dim = tensor::ExtractOp::create(rewriter, loc, in_shape, index); + divisor_count = + arith::MulIOp::create(rewriter, loc, divisor_count, dim); + } + } + // HLO ops are only defined on tensors, so we cast the divisor from + // index -> i64 -> tensor<1xi64> -> tensor -> tensor + Value divisor_casted = arith::IndexCastOp::create( + rewriter, loc, rewriter.getI64Type(), divisor_count); + Value divisor_tensor = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, rewriter.getI64Type()), + divisor_casted); + Value divisor = stablehlo::ConvertOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({}, reduce_element_type), + divisor_tensor); + auto broadcast_dims = rewriter.getDenseI64ArrayAttr({}); + result = chlo::BroadcastDivOp::create(rewriter, loc, result, divisor, + broadcast_dims); + } + + result = stablehlo::ConvertOp::create(rewriter, loc, result, element_type); + + // Need to reshape back after the reduction if we're keeping the reduced + // dimensions. Note that we do this through successive (nominally 1) + // applications of the TF ExpandDims op vs a more labor intensive + // reshape. Various code generation techniques benefit from the knowledge + // that this is a restricted form of shape manipulation that is just adding + // unit dims. + if (op.getKeepDims()) { + for (const auto &dim_is_reduced : + llvm::enumerate(reduced_dimensions_bitmap)) { + if (dim_is_reduced.value()) { + auto index_attr = GetI32ElementsAttr( + {static_cast(dim_is_reduced.index())}, &rewriter); + Value index = arith::ConstantOp::create(rewriter, loc, index_attr); + result = TF::ExpandDimsOp::create(rewriter, loc, result, index); + } + } + } + rewriter.replaceOp(op, {result}); + + return success(); + } +}; + +// Converts Mean op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %sum = "stablehlo.reduce"(%inp, %init) ["stablehlo.add"] +// {dimensions = ...} +// %divisor = arith.constant dense<...> : tensor +// %mean = "stablehlo.divide"(%sum, %divisor) +class ConvertMeanOp + : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarNegZeroOfType(reduce_element_type, loc, rewriter); + } +}; + +// Converts Sum op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %sum = "stablehlo.reduce"(%inp, %init) ["stablehlo.add"] +// {dimensions = ...} +class ConvertSumOp : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + // The neutral element of fp addition is -0.0, not 0.0: '0.0 + -0.0 = 0.0'. + return GetScalarNegZeroOfType(reduce_element_type, loc, rewriter); + } +}; + +// Converts Max op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %max = "stablehlo.reduce"(%inp, %init) ["stablehlo.maximum"] +// {dimensions = ...} +class ConvertMaxOp + : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, + hlo::kInfinityLowest, rewriter); + } +}; + +// Converts Min op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %min = "stablehlo.reduce"(%inp, %init) ["stablehlo.minimum"] +// {dimensions = ...} +class ConvertMinOp + : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, + hlo::kInfinityMax, rewriter); + } +}; + +// Converts Prod op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %prod = "stablehlo.reduce"(%inp, %init) ["stablehlo.multiply"] +// {dimensions = ...} +class ConvertProdOp + : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarConstOfType(reduce_element_type, loc, 1, rewriter); + } +}; + +// Converts All op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %max = "stablehlo.reduce"(%inp, %init) ["stablehlo.and"] +// {dimensions = ...} +class ConvertAllOp : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarConstOfType(reduce_element_type, loc, 1, rewriter); + } +}; + +// Converts Any op to HLO Reduce op. +// +// %init = arith.constant dense<...> : tensor +// %max = "stablehlo.reduce"(%inp, %init) ["stablehlo.or"] +// {dimensions = ...} +class ConvertAnyOp : public GenericConvertReductionOp { + public: + using GenericConvertReductionOp::GenericConvertReductionOp; + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarConstOfType(reduce_element_type, loc, 0, rewriter); + } +}; + +// Converts tensorflow ArgMin or ArgMax op to mhlo operations that perform +// a reduction on the original input and the corresponding index. The reduction +// sub-computation selects the max (or min) value and the index for the value. +// Derived: is the resulting derived class of this class. +// OpTy: is TF::ArgMaxOp or TF::ArgMinOp. +template +class ConvertArgMinMaxOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + RankedTensorType input_type = + mlir::dyn_cast(op.getInput().getType()); + if (!input_type) { + return failure(); + } + + Type input_element_type = input_type.getElementType(); + // TODO(bixia): Clarify whether tf.ArgMax supports complex data types. If + // tf.ArgMax doesn't support complex data types, this check can be removed. + if (!input_element_type.isSignlessIntOrFloat()) return failure(); + + Location loc = op.getLoc(); + Value init_value = + Derived::GetInitialValue(input_element_type, loc, rewriter); + + RankedTensorType output_type = + mlir::dyn_cast(op.getOutput().getType()); + if (!output_type) { + return rewriter.notifyMatchFailure(op, "requires known rank"); + } + + Type index_element_type = output_type.getElementType(); + Value index_init_value = + GetScalarConstOfType(index_element_type, loc, 0, &rewriter); + + RankedTensorType index_type = tensorflow::GetTypeFromTFTensorShape( + input_type.getShape(), index_element_type); + + std::optional optional_axis = + GetIntegerHLOAxisFromTFAxis(op.getDimension(), input_type.getRank()); + if (!optional_axis.has_value()) + return rewriter.notifyMatchFailure(op, "required axis"); + int64_t axis = optional_axis.value(); + + IntegerAttr iota_dimension = + IntegerAttr::get(rewriter.getIntegerType(64), axis); + Value input_shape = shape::ShapeOfOp::create(rewriter, loc, op.getInput()); + Value index_values = stablehlo::DynamicIotaOp::create( + rewriter, loc, index_type, input_shape, iota_dimension); + + Value operands[] = {op.getInput(), index_values}; + Value init_values[] = {init_value, index_init_value}; + + auto reduction = stablehlo::ReduceOp::create( + rewriter, loc, llvm::ArrayRef(operands), + llvm::ArrayRef(init_values), GetI64ArrayAttr({axis}, &rewriter), + TypeRange({input_element_type, index_element_type})); + auto direction = Derived::GetDirection(); + BuildArgMinMaxReductionBody(input_element_type, index_element_type, + direction, &reduction.getBody(), &rewriter); + + rewriter.replaceOp(op, {reduction.getResult(1)}); + return success(); + } +}; + +// Converts tensorflow ArgMax op to mhlo operations. The actual +// implementation is in class ConvertArgMinMaxOp: +// +// %init_index = arith.constant dense<...> : tensor +// %init = arith.constant dense<...> : tensor +// %reduce = "stablehlo.reduce"(%selected_input, %select_index, %init, +// %init_index) ["stablehlo.arg_max"] +class ConvertArgMaxOp + : public ConvertArgMinMaxOp { + public: + using ConvertArgMinMaxOp::ConvertArgMinMaxOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter &rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, + hlo::kInfinityLowest, &rewriter); + } + + static stablehlo::ComparisonDirection GetDirection() { + return stablehlo::ComparisonDirection::GE; + } +}; + +// Converts tensorflow ArgMin op to mhlo operations. The actual +// implementation is in class ConvertArgMinMaxOp: +// +// %init_index = arith.constant dense<...> : tensor +// %init = arith.constant dense<...> : tensor +// %reduce = "stablehlo.reduce"(%selected_input, %select_index, %init, +// %init_index) ["stablehlo.arg_min"] +class ConvertArgMinOp + : public ConvertArgMinMaxOp { + public: + using ConvertArgMinMaxOp::ConvertArgMinMaxOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter &rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, + hlo::kInfinityMax, &rewriter); + } + + static stablehlo::ComparisonDirection GetDirection() { + return stablehlo::ComparisonDirection::LE; + } +}; + +// Converts TF TensorScatterUpdate/Min/Max/Add/Sub op into Scatter Op with +// assignment: +// +// %result = "stablehlo.scatter"(%tensor, %indices, %updates) +// { dimensions = ... } +// +template +class ConvertTensorScatterOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + auto tensor_ty = mlir::dyn_cast(op.getTensor().getType()); + auto indices_ty = + mlir::dyn_cast(op.getIndices().getType()); + auto updates_ty = + mlir::dyn_cast(op.getUpdates().getType()); + + if (!tensor_ty || !indices_ty || !updates_ty) return failure(); + // Last dimension of the indices needs to known at compile time for + // computation of the 'update_window_dims' attribute in the dimensions + // struct. + int64_t num_index_dims = indices_ty.getShape().back(); + if (ShapedType::isDynamic(num_index_dims)) return failure(); + + auto updates = op.getUpdates(); + + // Broadcast scalar `updates` in into expected shape as following shape: + // updates.shape == indices.shape[:-1] + tensor.shape[indices.shape[-1]:] + if (updates_ty.getRank() == 0 && + (std::is_same::value || + std::is_same::value)) { + if (!tensor_ty.hasStaticShape()) { + return failure(); + } + + if (!indices_ty.hasStaticShape()) { + return failure(); + } + + auto tensor_shape = tensor_ty.getShape(); + auto indices_shape = indices_ty.getShape(); + auto index_depth = indices_shape.back(); + llvm::SmallVector expected_update_shape; + + // create the expected update shape which scalar update is broadcasted to + expected_update_shape.append(indices_shape.begin(), + std::prev(indices_shape.end())); + + expected_update_shape.append(std::next(tensor_shape.begin(), index_depth), + tensor_shape.end()); + + auto const_type = tensorflow::GetTypeFromTFTensorShape( + {static_cast(expected_update_shape.size())}, + rewriter.getIntegerType(64)); + + auto const_attr = GetI64ElementsAttr(expected_update_shape, &rewriter); + + auto const_op = + TF::ConstOp::create(rewriter, op->getLoc(), const_type, const_attr); + + auto broadcast_to_type = tensorflow::GetTypeFromTFTensorShape( + llvm::ArrayRef(expected_update_shape), + updates_ty.getElementType()); + + updates = TF::BroadcastToOp::create( + rewriter, op->getLoc(), broadcast_to_type, op.getUpdates(), const_op); + + updates_ty = mlir::dyn_cast(updates.getType()); + } + + int64_t tensor_rank = tensor_ty.getRank(); + int64_t indices_rank = indices_ty.getRank(); + int64_t updates_rank = + mlir::dyn_cast(updates.getType()).getRank(); + + int64_t window_dims = tensor_rank - num_index_dims; + auto dims_attr = stablehlo::ScatterDimensionNumbersAttr::get( + rewriter.getContext(), + llvm::to_vector<4>( + llvm::seq(updates_rank - window_dims, updates_rank)), + llvm::to_vector<4>(llvm::seq(0, num_index_dims)), + /*inputBatchingDims=*/{}, + /*scatterIndicesBatchingDims=*/{}, + llvm::to_vector<4>(llvm::seq(0, num_index_dims)), + indices_rank - 1); + + Location loc = op.getLoc(); + auto scatter = stablehlo::ScatterOp::create( + rewriter, loc, op.getType(), ValueRange(Value(op.getTensor())), + op.getIndices(), updates, dims_attr); + Derived::BuildScatterBody(tensor_ty.getElementType(), + &scatter.getUpdateComputation(), loc, rewriter); + + rewriter.replaceOp(op, scatter.getResult(0)); + return success(); + } +}; + +class ConvertTensorScatterUpdateOp + : public ConvertTensorScatterOp { + public: + using ConvertTensorScatterOp::ConvertTensorScatterOp; + + static void BuildScatterBody(Type element_type, Region *region, Location loc, + OpBuilder &builder) { + OpBuilder::InsertionGuard guard(builder); + Block *block = builder.createBlock(region); + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + stablehlo::ReturnOp::create(builder, loc, block->getArgument(1)); + } +}; + +class ConvertTensorScatterAddOp + : public ConvertTensorScatterOp { + public: + using ConvertTensorScatterOp::ConvertTensorScatterOp; + + static void BuildScatterBody(Type element_type, Region *region, Location loc, + OpBuilder &builder) { + OpBuilder::InsertionGuard guard(builder); + Block *block = builder.createBlock(region); + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + auto add_op = stablehlo::AddOp::create(builder, loc, block->getArgument(0), + block->getArgument(1)); + stablehlo::ReturnOp::create(builder, loc, add_op.getResult()); + } +}; + +class ConvertTensorScatterSubOp + : public ConvertTensorScatterOp { + public: + using ConvertTensorScatterOp::ConvertTensorScatterOp; + + static void BuildScatterBody(Type element_type, Region *region, Location loc, + OpBuilder &builder) { + OpBuilder::InsertionGuard guard(builder); + Block *block = builder.createBlock(region); + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + auto sub_op = stablehlo::SubtractOp::create( + builder, loc, block->getArgument(0), block->getArgument(1)); + stablehlo::ReturnOp::create(builder, loc, sub_op.getResult()); + } +}; + +class ConvertTensorScatterMinOp + : public ConvertTensorScatterOp { + public: + using ConvertTensorScatterOp::ConvertTensorScatterOp; + + static void BuildScatterBody(Type element_type, Region *region, Location loc, + OpBuilder &builder) { + OpBuilder::InsertionGuard guard(builder); + Block *block = builder.createBlock(region); + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + auto min_op = stablehlo::MinOp::create(builder, loc, block->getArgument(0), + block->getArgument(1)); + stablehlo::ReturnOp::create(builder, loc, min_op.getResult()); + } +}; + +class ConvertTensorScatterMaxOp + : public ConvertTensorScatterOp { + public: + using ConvertTensorScatterOp::ConvertTensorScatterOp; + + static void BuildScatterBody(Type element_type, Region *region, Location loc, + OpBuilder &builder) { + OpBuilder::InsertionGuard guard(builder); + Block *block = builder.createBlock(region); + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + auto max_op = stablehlo::MaxOp::create(builder, loc, block->getArgument(0), + block->getArgument(1)); + stablehlo::ReturnOp::create(builder, loc, max_op.getResult()); + } +}; + +// Converts Tile op to HLO BroadcastInDim and Reshape ops. +// For shape [S1, S2] and multiples [M1, M2], +// MS1 = M1 * S1; MS2 = M2 * S2 +// +// %broadcast = stablehlo.broadcast_in_dim(%input) { +// broadcast_dimensions = [0, 2] +// } +// %result = "stablehlo.reshape"(%broadcast) : (tensor) +// -> tensor +class ConvertTileOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::TileOp op, + PatternRewriter &rewriter) const override { + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + if (!input_ty || !input_ty.hasStaticShape()) return failure(); + ArrayRef input_shape = input_ty.getShape(); + Type element_type = input_ty.getElementType(); + + DenseIntElementsAttr multiples; + if (!matchPattern(op.getMultiples(), m_Constant(&multiples)) || + multiples.getType().getRank() != 1) + return failure(); + + const int64_t input_shape_size = input_shape.size(); + if (multiples.getNumElements() != input_shape_size) return failure(); + + SmallVector broadcasted_shape; + SmallVector broadcast_dimensions; + broadcasted_shape.reserve(input_shape.size() * 2); + broadcast_dimensions.reserve(input_shape.size()); + for (auto multiple_and_input : + llvm::zip(multiples.getValues(), input_shape)) { + int64_t multiple = std::get<0>(multiple_and_input).getSExtValue(); + int64_t input_size = std::get<1>(multiple_and_input); + + if (multiple < 0) return failure(); + + // Line input up with the next dimension in broadcasted_shape + // when broadcasting. + int64_t broadcast_dim; + int64_t output_size = input_size * multiple; + if (input_size == 1 || multiple == 1) { + // Special case for when normal broadcasting will just work. + broadcast_dim = broadcasted_shape.size(); + broadcasted_shape.push_back(output_size); + } else { + // Tiling will happen for this dimension during the ReshapeOp below. + broadcasted_shape.push_back(multiple); + broadcast_dim = broadcasted_shape.size(); + broadcasted_shape.push_back(input_size); + } + broadcast_dimensions.push_back(broadcast_dim); + } + Location loc = op.getLoc(); + Type broadcasted_type = + tensorflow::GetTypeFromTFTensorShape(broadcasted_shape, element_type); + Type output_type = op.getType(); + + Value result = stablehlo::BroadcastInDimOp::create( + rewriter, loc, broadcasted_type, op.getInput(), + GetI64ArrayAttr(broadcast_dimensions, &rewriter)); + + if (output_type != broadcasted_type) { + result = stablehlo::ReshapeOp::create(rewriter, loc, output_type, result); + } + + rewriter.replaceOp(op, {result}); + + return success(); + } +}; + +// Converts the tf.TileOp op into stablehlo.dynamic_reshape +// TODO(disc): To recover static special case's performance with folding and +// canonicalization. +class ConvertTileOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + // clang-format off + // Converts Tile op to HLO DBroadcastInDim and DReshape ops. + // For shape [S1, S2] and multiples [M1, M2], + // MS1 = M1 * S1; MS2 = M2 * S2 + // + // %out_dim_size = [S1, M1, S2, M2] + // %broadcast_dimensions = [1, 3]; + // %broadcast = stablehlo.d_broadcast_in_dim( + // %input, %out_dim_size, %braodcast_dimensions); + // %shape = [MS1, MS2] + // %result = "stablehlo.d_reshape"(%broadcast, %shape) + // : (tensor) -> tensor + // clang-format on + LogicalResult matchAndRewrite(TF::TileOp op, + PatternRewriter &rewriter) const final { + Location loc = op.getLoc(); + Value input = op.getInput(); + Value multiples = op.getMultiples(); + auto input_ty = mlir::dyn_cast(input.getType()); + if (!input_ty) return failure(); + // TODO(disc): Remove this constraint once fold and canonicalization + // implemented. + if (input_ty.hasStaticShape()) return failure(); + + Type element_type = input_ty.getElementType(); + int64_t input_rank = input_ty.getRank(); + SmallVector input_shape_values; + for (int64_t i = 0; i < input_rank; ++i) { + auto dim_size = input_ty.getDimSize(i); + if (dim_size == ShapedType::kDynamic) { + input_shape_values.push_back( + tensor::DimOp::create(rewriter, loc, input, i)); + } else { + input_shape_values.push_back(arith::ConstantOp::create( + rewriter, loc, rewriter.getIndexAttr(dim_size))); + } + } + + auto multiples_ty = mlir::dyn_cast(multiples.getType()); + int64_t multiples_rank = multiples_ty.getRank(); + // rank of multiples input of tf.TileOp must be 1 + if (multiples_rank != 1) return failure(); + // multiples input of tf.TileOp must be fixed shaped + if ((!multiples_ty.hasStaticShape()) || + (multiples_ty.getDimSize(0) != input_rank)) { + return failure(); + } + Type index_ty = rewriter.getIndexType(); + // %out_dim_size + SmallVector out_dim_size; + out_dim_size.reserve(input_rank * 2); + for (int64_t dim_idx = 0; dim_idx < input_rank; ++dim_idx) { + Value index = arith::ConstantOp::create(rewriter, loc, + rewriter.getIndexAttr(dim_idx)); + Value multiples_size = tensor::ExtractOp::create(rewriter, loc, multiples, + ValueRange{index}); + Value multiples_size_casted = + arith::IndexCastOp::create(rewriter, loc, index_ty, multiples_size); + out_dim_size.push_back(multiples_size_casted); + out_dim_size.push_back(input_shape_values[dim_idx]); + } + SmallVector broadcast_dimensions; + broadcast_dimensions.reserve(input_rank); + for (int64_t dim_idx = 0; dim_idx < input_rank; ++dim_idx) { + broadcast_dimensions.push_back(1 + 2 * dim_idx); + } + auto broadcast_dims_attr = GetI64ArrayAttr(broadcast_dimensions, &rewriter); + + Value out_dim_size_tensor = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(out_dim_size.size())}, index_ty), + out_dim_size); + SmallVector broadcast_shape(input_rank * 2, + ShapedType::kDynamic); + RankedTensorType broadcast_type = + tensorflow::GetTypeFromTFTensorShape(broadcast_shape, element_type); + Value broadcast = stablehlo::DynamicBroadcastInDimOp::create( + rewriter, loc, broadcast_type, input, out_dim_size_tensor, + broadcast_dims_attr); + + // %shape = [MS1, MS2] + SmallVector shape_values; + shape_values.reserve(input_rank); + for (int64_t i = 0; i < input_rank; ++i) { + Value dim_size_value = mlir::arith::MulIOp::create( + rewriter, loc, out_dim_size[2 * i], out_dim_size[2 * i + 1]); + shape_values.push_back(dim_size_value); + } + Value shape = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape({input_rank}, index_ty), + shape_values); + rewriter.replaceOpWithNewOp(op, op.getType(), + broadcast, shape); + return success(); + } +}; + +template +class ConvertMaxPoolGradOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + + Type element_type = + mlir::cast(op.getOrigInput().getType()).getElementType(); + + // Compute paddings using the original input and kernel shape and strides. + // Here, ReduceWindow op as used as the MaxPool op is lowered to the + // ReduceWindow op. + auto input_ty = + mlir::dyn_cast(op.getOrigInput().getType()); + if (!input_ty) return failure(); + DenseIntElementsAttr paddings_attr = GetReduceWindowPaddingAsAttr( + input_ty.getShape(), op.getKsize(), op.getStrides(), op.getPadding(), + &rewriter); + + auto result = stablehlo::SelectAndScatterOp::create( + rewriter, loc, op.getType(), op.getOrigInput(), op.getGrad(), + GetScalarConstOfType(element_type, loc, 0, &rewriter), + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getKsize()), &rewriter), + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getStrides()), &rewriter), + paddings_attr); + + BuildReduceBody(element_type, &result.getScatter(), + &rewriter); + { + OpBuilder::InsertionGuard guard(rewriter); + Block *block = rewriter.createBlock(&result.getSelect()); + + // Block arguments are scalars of the given element type. + Type type = + tensorflow::GetTypeFromTFTensorShape(/*shape=*/{}, element_type); + block->addArguments({type, type}, SmallVector(2, loc)); + + auto reducer = stablehlo::CompareOp::create( + rewriter, loc, block->getArgument(0), block->getArgument(1), + stablehlo::ComparisonDirection::GE); + stablehlo::ReturnOp::create(rewriter, loc, reducer.getResult()); + } + + rewriter.replaceOp(op, result); + + return success(); + } +}; + +using ConvertMaxPool2DGradOp = + ConvertMaxPoolGradOp; +using ConvertMaxPool3DGradOp = + ConvertMaxPoolGradOp; + +// Converts tf.Conv?DBackpropInputOp into: +// %rev_filter = "stablehlo.reverse"(%filter) +// %result = "stablehlo.convolution"(%out_backprop, %rev_filter) +template +class ConvertConvBackpropInputOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + // Unpack all of the attributes. + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + constexpr int num_dims = num_spatial_dims + 2; + int batch_dim = GetTensorBatchDimIndex(num_dims, data_format); + + tensorflow::Padding padding; + if (!GetPaddingFromString(op.getPadding().str(), &padding).ok()) + return failure(); + + auto out_backprop_ty = + mlir::dyn_cast(op.getOutBackprop().getType()); + auto filter_ty = mlir::dyn_cast(op.getFilter().getType()); + + // With the exception of out_backprop's batch dimension, out_backprop and + // filter need to have static shape. Filter is validated here, out_backprop + // is mostly validated at use. + if (!out_backprop_ty || !filter_ty || !filter_ty.hasStaticShape()) + return failure(); + + // Compute input_shape by supporting either: + // 1) Fully static shapes, represented as constants. + // 2) Static shapes with a dynamic batch dimension, represented as + // 1D tf.Pack of a batch dimension (can be static or dynamic) + // and other dimensions (can only be static), for example: + // "tf.Pack"(%142, %cst_301, %cst_301, %cst_300) {axis = 0 : i64, ...} + std::vector input_shape; + DenseIntElementsAttr input_shape_attr; + if (matchPattern(op.getInputSizes(), m_Constant(&input_shape_attr)) && + input_shape_attr.getType().getRank() == 1) { + input_shape.insert(input_shape.end(), + input_shape_attr.getValues().begin(), + input_shape_attr.getValues().end()); + } else { + auto pack = op.getInputSizes().template getDefiningOp(); + if (!pack || pack.getAxis() != 0) return failure(); + auto pack_ty = mlir::dyn_cast(pack.getType()); + if (!pack_ty || pack_ty.getRank() != 1) return failure(); + for (auto i = 0; i < pack_ty.getDimSize(0); ++i) { + if (i == batch_dim) { + // We don't use the batch dimension below, so we don't care about + // its size. Might as well populate it with -1. + input_shape.push_back(ShapedType::kDynamic); + } else { + DenseIntElementsAttr input_dims_attr; + if (matchPattern(pack.getValues()[i], m_Constant(&input_dims_attr)) && + input_dims_attr.getType().getRank() == 0) { + input_shape.push_back(input_dims_attr.getSplatValue()); + } else { + return failure(); + } + } + } + } + + auto dilations_attr = GetI64ElementsAttr(op.getDilations()); + std::vector dilations{ + dilations_attr.template getValues().begin(), + dilations_attr.template getValues().end()}; + auto strides_attr = GetI64ElementsAttr(op.getStrides()); + std::vector strides{ + strides_attr.template getValues().begin(), + strides_attr.template getValues().end()}; + + std::vector explicit_paddings; + if (padding == tensorflow::Padding::EXPLICIT) { + // EXPLICIT padding mode and the associated attribute is limited to + // Conv2DBackpropInput. So, fetch attribute by identifier instead of the + // op.explicit_paddings() attribute getter. + ArrayRef explicit_paddings_attr = + op->template getAttrOfType("explicit_paddings").getValue(); + explicit_paddings.reserve(explicit_paddings_attr.size()); + for (Attribute explicit_padding : explicit_paddings_attr) + explicit_paddings.push_back( + mlir::cast(explicit_padding).getInt()); + } + + ArrayRef filter_shape = filter_ty.getShape(); + + // Compute ConvDimensionNumbers, dilation, and padding. + SmallVector spatial_dims; + SmallVector lhs_dilation; + SmallVector rhs_dilation; + SmallVector paddings; + + for (int i : llvm::seq(0, num_spatial_dims)) { + const int64_t spatial_dim = + GetTensorSpatialDimIndex(num_dims, data_format, i); + spatial_dims.push_back(spatial_dim); + + // Prepare metadata indexed by spatial_dim for computing pad_before + // and pad_after. + int64_t input_size = input_shape[spatial_dim]; + if (input_size == ShapedType::kDynamic) return failure(); + int64_t output_size = out_backprop_ty.getDimSize(spatial_dim); + if (output_size == ShapedType::kDynamic) return failure(); + int64_t filter_size = filter_ty.getDimSize(i); + int64_t stride = strides[spatial_dim]; + int64_t dilation = dilations[spatial_dim]; + + // Compute pad_before and pad_after following the logic from + // ConvBackpropComputeDimensionsV2. (Unfortunately, we cannot call + // the function in question because it doesn't work with dynamic dims). + int64_t padding_before = -1, padding_after = -1; + if (padding == tensorflow::Padding::EXPLICIT) { + padding_before = explicit_paddings[2 * spatial_dim]; + padding_after = explicit_paddings[2 * spatial_dim + 1]; + } + int64_t expected_output_size = 0; + auto status = GetWindowedOutputSizeVerbose( + input_size, filter_size, dilation, stride, padding, + &expected_output_size, &padding_before, &padding_after); + if (!status.ok()) return failure(); + if (output_size != expected_output_size) return failure(); + int64_t effective_filter_size = (filter_size - 1) * dilation + 1; + int64_t pad_before = effective_filter_size - 1 - padding_before; + int64_t padded_out_size = input_size + effective_filter_size - 1; + int64_t expanded_output_size = (output_size - 1) * stride + 1; + int64_t pad_after = padded_out_size - expanded_output_size - pad_before; + + // Populate metadata for the upcoming stablehlo.conv op using the result + // of the computations performed above. + lhs_dilation.push_back(stride); + rhs_dilation.push_back(dilation); + paddings.push_back(pad_before); + paddings.push_back(pad_after); + } + + RankedTensorType paddings_ty = tensorflow::GetTypeFromTFTensorShape( + {num_spatial_dims, 2}, rewriter.getIntegerType(64)); + auto paddings_attr = DenseIntElementsAttr::get(paddings_ty, paddings); + + Value filter = op.getFilter(); + + const int feature_dim = + tensorflow::GetTensorFeatureDimIndex(num_dims, data_format); + const int64_t in_depth = *(input_shape.begin() + feature_dim); + if (in_depth == ShapedType::kDynamic) return failure(); + const int64_t filter_in_depth = filter_shape[num_spatial_dims]; + const int64_t feature_group_count = in_depth / filter_in_depth; + + if (feature_group_count != 1) { + // 1. Reshape filter from + // [H, W, ..., filter_in_depth, out_depth] to + // [H, W, ..., filter_in_depth, G, out_depth / G]. + auto new_shape = llvm::to_vector<6>(filter_shape); + new_shape.back() = feature_group_count; + new_shape.push_back(filter_shape.back() / feature_group_count); + Type filter_element_ty = filter_ty.getElementType(); + auto ty = + tensorflow::GetTypeFromTFTensorShape(new_shape, filter_element_ty); + filter = stablehlo::ReshapeOp::create(rewriter, op.getLoc(), ty, filter); + + // 2. Transpose to [H, W, ..., G, filter_in_depth, out_depth / G]. + llvm::SmallVector perm(num_dims + 1); + std::iota(perm.begin(), perm.end(), 0); + std::swap(perm[num_spatial_dims], perm[num_spatial_dims + 1]); + std::swap(new_shape[num_spatial_dims], new_shape[num_spatial_dims + 1]); + ty = tensorflow::GetTypeFromTFTensorShape(new_shape, filter_element_ty); + filter = stablehlo::TransposeOp::create(rewriter, op.getLoc(), ty, filter, + GetI64ArrayAttr(perm, &rewriter)); + + // 3. Reshape to [H, W, ..., in_depth, out_depth / G]. + new_shape[num_spatial_dims] *= new_shape[num_spatial_dims + 1]; + new_shape[num_spatial_dims + 1] = new_shape.back(); + new_shape.pop_back(); + ty = tensorflow::GetTypeFromTFTensorShape(new_shape, filter_element_ty); + filter = stablehlo::ReshapeOp::create(rewriter, op.getLoc(), ty, filter); + } + + SmallVector kernel_spatial_dims; + kernel_spatial_dims.resize(num_spatial_dims); + std::iota(kernel_spatial_dims.begin(), kernel_spatial_dims.end(), 0); + + // Mirror the filter in the spatial dimensions. + filter = stablehlo::ReverseOp::create( + rewriter, op.getLoc(), filter, + GetI64ArrayAttr(kernel_spatial_dims, &rewriter)); + + // activation gradients + // = gradients (with padding and dilation) mirrored_weights + Value result = stablehlo::ConvolutionOp::create( + rewriter, op.getLoc(), op.getType(), op.getOutBackprop(), filter, + /*window_strides=*/ + GetI64ArrayAttrForValue(/*size=*/num_spatial_dims, /*val=*/1, + &rewriter), + /*padding=*/paddings_attr, + /*lhs_dilation=*/GetI64ArrayAttr(lhs_dilation, &rewriter), + /*rhs_dilation=*/GetI64ArrayAttr(rhs_dilation, &rewriter), + /*window_reversal=*/nullptr, + stablehlo::ConvDimensionNumbersAttr::get( + rewriter.getContext(), + /*inputBatchDimension=*/batch_dim, + /*inputFeatureDimension=*/feature_dim, + /*inputSpatialDimensions=*/spatial_dims, + // TF filter shape is [ H, W, ..., inC, outC ] + // Transpose the input and output features for computing the + // gradient. + /*kernelInputFeatureDimension=*/ + num_spatial_dims + 1, + /*kernelOutputFeatureDimension=*/ + num_spatial_dims, + /*kernelSpatialDimensions=*/kernel_spatial_dims, + /*outputBatchDimension=*/batch_dim, + /*outputFeatureDimension=*/feature_dim, + /*outputSpatialDimensions=*/spatial_dims), + rewriter.getI64IntegerAttr(feature_group_count), + /*batch_group_count=*/rewriter.getI64IntegerAttr(1), + /*precision_config=*/GetPrecisionConfig(&rewriter)); + + rewriter.replaceOp(op, {result}); + + return success(); + } +}; + +using ConvertConv2DBackpropInputOp = + ConvertConvBackpropInputOp; +using ConvertConv3DBackpropInputOp = + ConvertConvBackpropInputOp; + +// Converts tf.Conv?DBackpropFilterOp into: +// %result = "stablehlo.convolution"(%input, %out_backprop) +template +class ConvertConvBackpropFilterOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + // Unpack all of the attributes. + tensorflow::TensorFormat data_format; + if (!FormatFromString(op.getDataFormat().str(), &data_format)) + return op.emitOpError("invalid data format"); + + tensorflow::Padding padding; + if (!GetPaddingFromString(op.getPadding().str(), &padding).ok()) + return failure(); + + auto out_backprop_ty = + mlir::dyn_cast(op.getOutBackprop().getType()); + auto input_ty = mlir::dyn_cast(op.getInput().getType()); + + for (RankedTensorType ty : {out_backprop_ty, input_ty}) + if (!ty || !ty.hasStaticShape()) return failure(); + + ArrayRef out_backprop_shape = out_backprop_ty.getShape(); + ArrayRef input_shape = input_ty.getShape(); + + DenseIntElementsAttr filter_shape_attr; + if (!matchPattern(op.getFilterSizes(), m_Constant(&filter_shape_attr)) || + filter_shape_attr.getType().getRank() != 1) + return failure(); + + auto dilations_attr = GetI64ElementsAttr(op.getDilations()); + std::vector dilations{ + dilations_attr.template getValues().begin(), + dilations_attr.template getValues().end()}; + auto strides_attr = GetI64ElementsAttr(op.getStrides()); + std::vector strides{ + strides_attr.template getValues().begin(), + strides_attr.template getValues().end()}; + + std::vector explicit_paddings; + if (padding == tensorflow::Padding::EXPLICIT) { + // EXPLICIT padding mode and the associated attribute is limited to + // Conv2DBackpropFilter. So, fetch attribute by identifier instead of the + // op.explicit_paddings() attribute getter. + ArrayRef explicit_paddings_attr = + op->template getAttrOfType("explicit_paddings").getValue(); + explicit_paddings.reserve(explicit_paddings_attr.size()); + for (Attribute explicit_padding : explicit_paddings_attr) + explicit_paddings.push_back( + mlir::cast(explicit_padding).getInt()); + } + + constexpr int num_dims = num_spatial_dims + 2; + auto filter_shape = filter_shape_attr.getValues(); + + // Reuse dimension computation logic from conv_grad_shape_utils.cc. + tensorflow::ConvBackpropDimensions dims; + if (!tensorflow::ConvBackpropComputeDimensionsV2( + /*label=*/"", num_spatial_dims, + ToTensorShape(input_shape), + ToTensorShape(filter_shape), + ToTensorShape(out_backprop_shape), dilations, + strides, padding, explicit_paddings, data_format, &dims) + .ok()) { + return failure(); + } + + // The activations (inputs) form the LHS of the convolution. + // Activations have shape: [batch, in_rows, in_cols, ..., in_depth] + // For the gradient computation, we need to: + // 1. In the case of group convolution, move the num_groups dimension before + // the batch dimension + // 2. Swap the roles of the batch and feature dimensions. + const int feature_dim = + tensorflow::GetTensorFeatureDimIndex(num_dims, data_format); + const int64_t in_depth = input_shape[feature_dim]; + const int64_t filter_in_depth = *(filter_shape.begin() + num_spatial_dims); + const int64_t batch_group_count = in_depth / filter_in_depth; + + // Compute ConvDimensionNumbers, dilation, and padding. + SmallVector spatial_dims; + SmallVector kernel_spatial_dims; + SmallVector rhs_dilation; + SmallVector paddings; + SmallVector window_strides; + + // The filter gradients are computed by a convolution of the input + // activations and the output gradients, with some appropriate padding. + // See the comment at the top of conv_grad_ops.h for details. + + for (int i : llvm::seq(0, num_spatial_dims)) { + const int64_t dim = + tensorflow::GetTensorSpatialDimIndex(num_dims, data_format, i); + kernel_spatial_dims.push_back(dim); + // Besides padding the input, we will also expand output_rows to + // expanded_out_rows = (output_rows - 1) * stride + 1 + // with zeros in between: + // + // a . . . b . . . c . . . d . . . e + // + // This is done by specifying the window dilation factors in the + // convolution HLO below. + const auto &spatial_dim_i = dims.spatial_dims[i]; + rhs_dilation.push_back(spatial_dim_i.stride); + window_strides.push_back(dilations[dim]); + + // We will also need to pad the input with zeros such that after the + // convolution, we get the right size for the filter. + // The padded_in_rows should be such that when we convolve this with the + // expanded_out_rows as a filter, we should get filter_rows back. + + const int64_t padded_in_size = + spatial_dim_i.expanded_output_size + + (spatial_dim_i.filter_size - 1) * dilations[dim]; + + // However it can be smaller than input_rows: in this + // case it means some of the inputs are not used. + // + // An example is to have input_cols = 3, filter_cols = 2 and stride = 2: + // + // INPUT = [ A B C ] + // + // FILTER = [ x y ] + // + // and the output will only have one column: a = A * x + B * y + // + // and input "C" is not used at all. + // + // We apply negative padding in this case. + const int64_t pad_total = padded_in_size - spatial_dim_i.input_size; + + // + For the EXPLICIT padding, we pad the top/left side with the explicit + // padding and pad the bottom/right side with the remaining space. + // + For the VALID padding, we don't pad anything on the top/left side + // and pad the bottom/right side with the remaining space. + // + For the SAME padding, we pad top/left side the same as bottom/right + // side. + // + // In addition, if the padded input size is smaller than the input size, + // we need to ignore some training elements of the input. We do this by + // applying negative padding on the right/bottom. + const int64_t pad_before = padding == tensorflow::Padding::EXPLICIT + ? explicit_paddings[2 * dim] + : padding == tensorflow::Padding::SAME + ? std::max(pad_total / 2, 0) + : 0; + paddings.push_back(pad_before); + paddings.push_back(pad_total - pad_before); + } + + RankedTensorType paddings_ty = tensorflow::GetTypeFromTFTensorShape( + {num_spatial_dims, 2}, rewriter.getIntegerType(64)); + auto paddings_attr = DenseIntElementsAttr::get(paddings_ty, paddings); + + SmallVector output_spatial_dimensions; + output_spatial_dimensions.resize(num_spatial_dims); + std::iota(output_spatial_dimensions.begin(), + output_spatial_dimensions.end(), 0); + + const int batch_dim = + tensorflow::GetTensorBatchDimIndex(num_dims, data_format); + + Value result = stablehlo::ConvolutionOp::create( + rewriter, op.getLoc(), op.getType(), op.getInput(), op.getOutBackprop(), + /*window_strides=*/GetI64ArrayAttr(window_strides, &rewriter), + /*padding=*/paddings_attr, /*lhs_dilation=*/ + GetI64ArrayAttrForValue(/*size=*/num_spatial_dims, /*val=*/1, + &rewriter), + GetI64ArrayAttr(rhs_dilation, &rewriter), + /*window_reversal=*/nullptr, + stablehlo::ConvDimensionNumbersAttr::get( + rewriter.getContext(), + // Swap batch_dim and feature_dim in the activations. + /*inputBatchDimension=*/feature_dim, + /*inputFeatureDimension=*/batch_dim, + /*inputSpatialDimensions=*/kernel_spatial_dims, + // The gradients become the RHS of the convolution. + // The gradients have shape [batch, out_rows, out_cols, ..., + // out_depth] where the batch becomes the input feature for the + // convolution. + /*kernelInputFeatureDimension=*/batch_dim, + /*kernelOutputFeatureDimension=*/feature_dim, + /*kernelSpatialDimensions=*/kernel_spatial_dims, + /*outputBatchDimension=*/num_spatial_dims, + /*outputFeatureDimension=*/num_spatial_dims + 1, + /*outputSpatialDimensions=*/output_spatial_dimensions), + /*feature_group_count=*/rewriter.getI64IntegerAttr(1), + rewriter.getI64IntegerAttr(batch_group_count), + /*precision_config=*/GetPrecisionConfig(&rewriter)); + + rewriter.replaceOp(op, {result}); + + return success(); + } +}; + +using ConvertConv2DBackpropFilterOp = + ConvertConvBackpropFilterOp; +using ConvertConv3DBackpropFilterOp = + ConvertConvBackpropFilterOp; + +class ConvertOneHotOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::OneHotOp op, + PatternRewriter &rewriter) const override { + auto indices_ty = + mlir::dyn_cast(op.getIndices().getType()); + if (!indices_ty || !indices_ty.hasStaticShape()) return failure(); + ArrayRef indices_shape = indices_ty.getShape(); + Type element_type = indices_ty.getElementType(); + + DenseIntElementsAttr depth_attr; + if (!matchPattern(op.getDepth(), m_Constant(&depth_attr))) { + return failure(); + } + + int64_t depth = depth_attr.getValues()[0].getSExtValue(); + int64_t axis = op.getAxis(); + if (axis == -1) axis = indices_shape.size(); + + llvm::SmallVector broadcast_dims(indices_shape.size()); + std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0); + std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1); + + llvm::SmallVector output_dims = + llvm::to_vector<4>(indices_shape); + output_dims.insert(output_dims.begin() + axis, depth); + + Location loc = op.getLoc(); + + // The iota result is the effective output shape of the computation, + // and indices must be broadcast into it. At this point, this computation + // would need to be reworked quite a bit to support dynamic shapes, so + // just using static broadcasting. + auto index_type = + tensorflow::GetTypeFromTFTensorShape(output_dims, element_type); + auto iota = stablehlo::IotaOp::create( + rewriter, loc, index_type, + IntegerAttr::get(rewriter.getIntegerType(64), axis)); + auto broadcast_indices = stablehlo::BroadcastInDimOp::create( + rewriter, loc, index_type, op.getIndices(), + GetI64ArrayAttr(broadcast_dims, &rewriter)); + + Value compare = + stablehlo::CompareOp::create(rewriter, loc, broadcast_indices, iota, + stablehlo::ComparisonDirection::EQ); + Value on_value = stablehlo::BroadcastOp::create( + rewriter, loc, op.getType(), op.getOnValue(), + GetI64ArrayAttr(output_dims, &rewriter)); + Value off_value = stablehlo::BroadcastOp::create( + rewriter, loc, op.getType(), op.getOffValue(), + GetI64ArrayAttr(output_dims, &rewriter)); + Value result = stablehlo::SelectOp::create(rewriter, loc, op.getType(), + compare, on_value, off_value); + + rewriter.replaceOp(op, {result}); + + return success(); + } +}; + +// Converts InfeedDequeueTuple to XLA HLO create_token, infeed and +// get_tuple_element ops. +// +// All HLO infeed ops expect a HLO token type operand and produce a tuple +// containing a token. This HLO token type is used to order multiple infeed +// operations within a computation. The token type can come from other +// infeed/outfeed/send/recv ops or can be generated using create_token op with +// no operands. Here we emit a create_token op to generate the token type +// operand of infeed. The stablehlo.InfeedOp can produce multiple results and +// later will be exported to XLA infeed op with single tuple return type. +// +// For example the following IR: +// %0:2 = "tf.InfeedDequeueTuple"() : () -> (tensor<3xi32>, tensor<4xf32>) +// +// would be lowered to +// +// %token = "stablehlo.create_token"() : () -> !stablehlo.token +// %data_and_token = "stablehlo.infeed"(%token) {infeed_config = ""} : +// (!stablehlo.token) -> tensor<3xi32>, tensor<4xf32>, !stablehlo.token> +// +class ConvertInfeedDequeueTupleOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::InfeedDequeueTupleOp op, + PatternRewriter &rewriter) const override { + SmallVector result_types; + result_types.reserve(op.getOutputs().size() + 1); + for (const auto &output : op.getOutputs()) { + Type ty = output.getType(); + if (auto tensor_ty = mlir::dyn_cast(ty)) { + if (!tensor_ty.hasStaticShape()) return failure(); + } + result_types.push_back(ty); + } + + // Infeed takes a single token operand. Generate the token using + // create_token op to pass to the infeed op. + auto token = stablehlo::CreateTokenOp::create( + rewriter, op.getLoc(), + stablehlo::TokenType::get(rewriter.getContext())); + + result_types.push_back(token.getType()); + + ArrayAttr layout; // filled in during the xla-adjust-layout pass + auto data_and_token = stablehlo::InfeedOp::create( + rewriter, op.getLoc(), result_types, token, + /*infeed_config=*/rewriter.getStringAttr(""), + /*layout=*/layout); + + result_types.pop_back(); // remove the token type. + + if (op.get_XlaSharding().has_value()) { + // _XlaSharding attribute in TF is a serialized string of the OpSharding + // proto, so convert to a text form here. + ::xla::OpSharding sharding_proto; + if (tensorflow::DecodeShardingAttribute( + op.get_XlaSharding().value().str(), sharding_proto) + .failed()) { + return failure(); + } + // Token is a control signal and not a real data, so arbitrarily assign + // the token to device 0. + if (sharding_proto.type() == ::xla::OpSharding::TUPLE) { + *sharding_proto.add_tuple_shardings() = + ::xla::sharding_builder::SingleDevice(0); + data_and_token->setAttr( + kShardingAttr, + rewriter.getStringAttr(sharding_proto.SerializeAsString())); + } else { + data_and_token->setAttr(kShardingAttr, op.get_XlaShardingAttr()); + } + } + + if (op->hasAttr("layouts")) { + // Append a UnitAttr for the "token" operand of the stablehlo.infeed op + // here to avoid compilation failure when exporting "layouts" attribute of + // the corresponding InfeedDequeueTupleOp to a graph node. + data_and_token->setAttr("layout", op->getAttr("layouts")); + } + llvm::SmallVector results; + results.reserve(result_types.size()); + for (const auto &idx_and_type : llvm::enumerate(result_types)) { + results.push_back(data_and_token.getResult(idx_and_type.index())); + } + rewriter.replaceOp(op, ValueRange(results)); + return success(); + } +}; + +// Converts tf.OutfeedEnqueueTuple to XLA HLO tuple, create_token and outfeed +// ops. +// +// XLA HLO outfeed op expects a token, which we generate by emitting an +// create_token op. +// +// For example the following IR: +// "tf.OutfeedEnqueueTuple"(%val_1, %val_2) : (tensor<3xi32>, tensor<4xf32>) -> +// () +// +// would be lowered to +// +// %token = "stablehlo.create_token"() : () -> !stablehlo.token +// %outfeed_token = "stablehlo.outfeed"(%val_1, %val_2, %token) {outfeed_config +// = ""} +// : +// (tensor<3xi32>, tensor<4xf32>, !stablehlo.token) -> !stablehlo.token +// +class ConvertOutfeedEnqueueTupleOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::OutfeedEnqueueTupleOp op, + PatternRewriter &rewriter) const override { + auto token_type = stablehlo::TokenType::get(rewriter.getContext()); + auto token = + stablehlo::CreateTokenOp::create(rewriter, op.getLoc(), token_type); + + stablehlo::OutfeedOp::create(rewriter, op.getLoc(), token_type, + op.getInputs(), token, + /*outfeed_config=*/rewriter.getStringAttr("")); + rewriter.eraseOp(op); + return success(); + } +}; + +// Converts tf.TopKV2 to chlo.top_k. +class ConvertTopKV2Op : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::TopKV2Op op, + PatternRewriter &rewriter) const override { + // We can only match when the `k` operand is a constant scalar. + DenseIntElementsAttr k_attr; + if (!matchPattern(op.getK(), m_Constant(&k_attr))) return failure(); + int64_t k = (*k_attr.begin()).getSExtValue(); + + TensorType input_type = mlir::cast(op.getInput().getType()); + if (!input_type.hasRank()) return failure(); + int64_t input_rank = input_type.getRank(); + int64_t last_dim_index = input_rank - 1; + int64_t last_dim_size = input_type.getDimSize(last_dim_index); + if (last_dim_size == ShapedType::kDynamic) return failure(); + + rewriter.replaceOpWithNewOp(op, op.getInput(), k); + return success(); + } +}; + +// Converts tf.Unpack to a series of XLA HLO slice ops. +// +// Each slice takes one element along the dimension to unpack and takes the full +// range for all other dimensions. Each slice is then reshaped to drop the +// dimension to unpack (which is always of size 1). +// TODO(antiagainst): consider changing this into a TF internal lowering pass. +class ConvertUnpackOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::UnpackOp op, + PatternRewriter &rewriter) const override { + auto value_type = mlir::dyn_cast(op.getValue().getType()); + if (!value_type) return failure(); + + int64_t value_rank = value_type.getRank(); + int64_t axis = op.getAxis(); + if (axis < 0) axis += value_rank; + + // Parameters for constructing each slice. + SmallVector begin_indices(value_rank, 0); + auto end_indices = llvm::to_vector<4>(value_type.getShape()); + SmallVector strides(value_rank, 1); + + // All HLO slice+squeeze results used to replace the original tf.Unpack op. + SmallVector results; + results.reserve(op.getNumResults()); + + for (int i = 0, end = op.getNumResults(); i < end; ++i) { + begin_indices[axis] = i; + end_indices[axis] = i + 1; + + auto slice_op = + stablehlo::SliceOp::create(rewriter, op.getLoc(), op.getValue(), + GetI64ArrayAttr(begin_indices, &rewriter), + GetI64ArrayAttr(end_indices, &rewriter), + GetI64ArrayAttr(strides, &rewriter)); + // Reshape to drop the axis dimension. + auto result = + TF::SqueezeOp::create(rewriter, op.getLoc(), op.getType(i), slice_op, + rewriter.getI64ArrayAttr(op.getAxis())); + results.push_back(result); + } + + rewriter.replaceOp(op, results); + return success(); + } +}; + +// Converts tf.Unpack to a series of XLA HLO Slice ops. +// TODO(disc): To recover static special case's performance with folding and +// canonicalization. +class ConvertUnpackOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::UnpackOp op, + PatternRewriter &rewriter) const override { + auto value_type = mlir::dyn_cast(op.getValue().getType()); + if (!value_type) return failure(); + // TODO(disc): Remove this constraint once fold and canonicalization + // implemented. + if (value_type.hasStaticShape()) return failure(); + + int64_t value_rank = value_type.getRank(); + int64_t axis = op.getAxis(); + if (axis < 0) axis += value_rank; + Location loc = op.getLoc(); + + auto shape_scalar_type = rewriter.getIntegerType(32); + // Parameters for constructing each slice. + SmallVector begin_indices, end_indices, strides; + begin_indices.reserve(value_rank); + end_indices.reserve(value_rank); + strides.reserve(value_rank); + // final output shape + SmallVector shape_values; + shape_values.reserve(value_rank - 1); + // slice shape before reshape, should be like{?, 1, ?, ?} if axis = 1 + SmallVector slice_shape(value_rank, ShapedType::kDynamic); + for (int64_t dim_idx = 0; dim_idx < value_rank; ++dim_idx) { + int64_t dim_size = value_type.getDimSize(dim_idx); + if (dim_size == ShapedType::kDynamic) { + Value dim_i = arith::IndexCastOp::create( + rewriter, loc, shape_scalar_type, + tensor::DimOp::create(rewriter, loc, op.getOperand(), dim_idx)); + end_indices.push_back(dim_i); + if (dim_idx != axis) { + shape_values.push_back(dim_i); + } + } else { + Value dim_i = arith::ConstantOp::create( + rewriter, loc, shape_scalar_type, + rewriter.getIntegerAttr(shape_scalar_type, dim_size)); + end_indices.push_back(dim_i); + if (dim_idx != axis) { + shape_values.push_back(dim_i); + slice_shape[dim_idx] = dim_size; + } else { + slice_shape[dim_idx] = 1; + } + } + begin_indices.push_back( + arith::ConstantIntOp::create(rewriter, loc, 0, 32)); + strides.push_back(arith::ConstantIntOp::create(rewriter, loc, 1, 32)); + } + + SmallVector results; + results.reserve(op.getNumResults()); + Type i32_ty = rewriter.getI32Type(); + for (int64_t i = 0; i < op.getNumResults(); ++i) { + begin_indices[axis] = arith::ConstantIntOp::create(rewriter, loc, i, 32); + end_indices[axis] = + arith::ConstantIntOp::create(rewriter, loc, i + 1, 32); + Value slice_op = stablehlo::RealDynamicSliceOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape(slice_shape, + value_type.getElementType()), + op.getValue(), + tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(begin_indices.size())}, i32_ty), + begin_indices), + tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(end_indices.size())}, i32_ty), + end_indices), + tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(strides.size())}, i32_ty), + strides)); + // Reshape to drop the axis dimension. + Value new_shape = tensor::FromElementsOp::create( + rewriter, loc, + tensorflow::GetTypeFromTFTensorShape( + {static_cast(shape_values.size())}, i32_ty), + shape_values); + Value reshape_op = stablehlo::DynamicReshapeOp::create( + rewriter, loc, op.getType(i), slice_op, new_shape); + results.push_back(reshape_op); + } + + rewriter.replaceOp(op, results); + return success(); + } +}; + +// Converts the tf.SigmoidGradOp +// TODO(disc): To recover static special case's performance with folding and +// canonicalization. +class ConvertSigmoidGradOpDynamic : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SigmoidGradOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value y = op.getY(); + Value dy = op.getDy(); + auto tp_y = mlir::dyn_cast(y.getType()); + auto tp_dy = mlir::dyn_cast(dy.getType()); + if (!tp_y || !tp_dy) return failure(); + + // TODO(disc): Remove this constraint once fold and canonicalization + // implemented. + if (tp_y.hasStaticShape() || tp_dy.hasStaticShape()) return failure(); + + Attribute attr; + Type elem_tp = tp_y.getElementType(); + if (elem_tp.isSignlessInteger()) { + attr = rewriter.getIntegerAttr(elem_tp, 1); + } else { + assert(mlir::isa(elem_tp)); + attr = rewriter.getFloatAttr(elem_tp, 1); + } + Value one = stablehlo::ConstantOp::create( + rewriter, loc, + DenseElementsAttr::get( + tensorflow::GetTypeFromTFTensorShape({}, elem_tp), attr)); + + auto v0 = chlo::BroadcastMulOp::create( + rewriter, loc, dy, y, + hlo::getBroadcastDimensionsAttr(&rewriter, dy, y)); + auto v1 = chlo::BroadcastSubOp::create( + rewriter, loc, one, y, + hlo::getBroadcastDimensionsAttr(&rewriter, one, y)); + auto result = chlo::BroadcastMulOp::create( + rewriter, loc, v0, v1, + hlo::getBroadcastDimensionsAttr(&rewriter, v0, v1)); + + rewriter.replaceOp(op, result.getOperation()->getResults()); + return success(); + } +}; + +// Converts TF unsorted segment reduction ops to XLA HLO scatter op. +// +// TF unsorted segment reduction op peforms the following calculation: +// +// Assume segment ids' shape is [SI0, SI1, ..., SIm] and data's shape is +// [D0, D1, ..., Dn]. Note that segment ids' shape must be a prefix of data's +// shape, so we can have data's shape represented as [SI0, SI1, ..., SIm, +// Dm+1, ..., Dn]. Then +// output[segment_ids[SI_i0, SI_i1, ..., SI_im], D_im+1, ..., D_in] = +// over data[SI_i0, SI_i1, ..., SI_im, D_im+1, ..., D_in] +// where SI_iN is in the range of [0, SIN) and D_iN is in the range of [0, DN). +// +// The op will be translated to XLA HLO scatter with the following parameters: +// * Update window dims is [segment_id_rank, data_rank). +// * Inserted window dims is {0}. +// * Scatter dims to operand dims mapping is {0}. +// * Index vector dim is segment_id_rank. +template +class GenericConvertUnsortedSegmentReductionOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpTy op, + PatternRewriter &rewriter) const override { + auto data_type = mlir::dyn_cast(op.getData().getType()); + if (!data_type) return failure(); + int64_t data_rank = data_type.getRank(); + + auto segment_ids_type = + mlir::dyn_cast(op.getSegmentIds().getType()); + if (!segment_ids_type) return failure(); + int64_t segment_ids_rank = segment_ids_type.getRank(); + + DenseIntElementsAttr num_segments_attr; + if (!matchPattern(op.getNumSegments(), m_Constant(&num_segments_attr))) + return failure(); + + // The final shape for TF unsorted segment reduction op is [num_segments] + + // data_shape[segment_ids_rank:]. + SmallVector output_shape; + output_shape.push_back((*num_segments_attr.begin()).getSExtValue()); + auto suffix = data_type.getShape().drop_front(segment_ids_rank); + output_shape.append(suffix.begin(), suffix.end()); + auto output_type = tensorflow::GetTypeFromTFTensorShape( + output_shape, data_type.getElementType()); + + // Broadcast the initial value for reduction. This will become the + // 'operand' parameter to scatter to for the final scatter op. + Value init = ConcreteClass::GetInitialValue(data_type.getElementType(), + op.getLoc(), &rewriter); + auto broadcasted_init = stablehlo::BroadcastOp::create( + rewriter, op.getLoc(), output_type, init, + GetI64ArrayAttr(output_shape, &rewriter)); + + // Parameters for the generated scatter op. + SmallVector inserted_window_dims(1, 0); + SmallVector scatter_dims_to_operand_dims(1, 0); + int64_t index_vector_dim = segment_ids_rank; + + // Put all parameters in a StructAttr. + auto dims_attr = stablehlo::ScatterDimensionNumbersAttr::get( + rewriter.getContext(), + llvm::to_vector<4>(llvm::seq(segment_ids_rank, data_rank)), + inserted_window_dims, + /*inputBatchingDims=*/{}, + /*scatterIndicesBatchingDims=*/{}, scatter_dims_to_operand_dims, + index_vector_dim); + + auto scatter = stablehlo::ScatterOp::create( + rewriter, op.getLoc(), op.getType(), + ValueRange(Value(broadcasted_init)), op.getSegmentIds(), op.getData(), + dims_attr); + BuildReduceBody(data_type.getElementType(), + &scatter.getUpdateComputation(), &rewriter); + + rewriter.replaceOp(op, scatter.getResult(0)); + return success(); + } +}; + +class ConvertUnsortedSegmentMaxOp + : public GenericConvertUnsortedSegmentReductionOp< + ConvertUnsortedSegmentMaxOp, TF::UnsortedSegmentMaxOp, + stablehlo::MaxOp> { + public: + using GenericConvertUnsortedSegmentReductionOp:: + GenericConvertUnsortedSegmentReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, hlo::kLowest, + rewriter); + } +}; + +class ConvertUnsortedSegmentMinOp + : public GenericConvertUnsortedSegmentReductionOp< + ConvertUnsortedSegmentMinOp, TF::UnsortedSegmentMinOp, + stablehlo::MinOp> { + public: + using GenericConvertUnsortedSegmentReductionOp:: + GenericConvertUnsortedSegmentReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarLimitConstOfType(reduce_element_type, loc, hlo::kMax, + rewriter); + } +}; + +class ConvertUnsortedSegmentProdOp + : public GenericConvertUnsortedSegmentReductionOp< + ConvertUnsortedSegmentProdOp, TF::UnsortedSegmentProdOp, + stablehlo::MulOp> { + public: + using GenericConvertUnsortedSegmentReductionOp:: + GenericConvertUnsortedSegmentReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarConstOfType(reduce_element_type, loc, 1, rewriter); + } +}; + +class ConvertUnsortedSegmentSumOp + : public GenericConvertUnsortedSegmentReductionOp< + ConvertUnsortedSegmentSumOp, TF::UnsortedSegmentSumOp, + stablehlo::AddOp> { + public: + using GenericConvertUnsortedSegmentReductionOp:: + GenericConvertUnsortedSegmentReductionOp; + + static Value GetInitialValue(Type reduce_element_type, Location loc, + PatternRewriter *rewriter) { + return GetScalarConstOfType(reduce_element_type, loc, 0, rewriter); + } +}; + +// Converts tf.RandomShuffle op into a series of XLA HLO ops. +// +// tf.RandomShuffle shuffles tensors along the first dimension. If the input +// tensor's rank is 1, then it is translated into HLO sort op(s) according to +// indices randomly generated via HLO rng_uniform ops. Otherwise, it is +// translated into an HLO while op to first emulate shuffling indices using +// HLO dynamic_slice and dynamic_update_slice ops, then finally HLO gather +// with the shuffled indices. +class ConvertRandomShuffleOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::RandomShuffleOp op, + PatternRewriter &rewriter) const override { + auto no_op = [&]() { + rewriter.replaceOp(op, op.getValue()); + return success(); + }; + + auto input_type = mlir::dyn_cast(op.getValue().getType()); + if (!input_type) return failure(); + if (input_type.hasStaticShape() && input_type.getNumElements() <= 1) + // No shuffling is required, so copy input directly to output. + return no_op(); + + int64_t input_rank = input_type.getRank(); + int64_t first_dim_size = input_type.getDimSize(0); + if (ShapedType::isDynamic(first_dim_size)) return failure(); + + if (first_dim_size <= 1) + // No shuffling is required, so copy input directly to output. + return no_op(); + + // For vectors, shuffle values by sorting instead of the obvious + // Fisher-Yates algorithm. Fisher-Yates is simple to implement and correct, + // but not easily parallelizable. For a sufficiently parallel architecture, + // it is faster to sort many times, than Fisher-Yates shuffle once. + if (input_rank == 1) { + // Shuffle values by assigning each value a random key and sorting the + // keys. Keys can collide causing detectable patterns in the shuffled + // output. Collisions translates into more ascending sub-sequences in the + // shuffled output than would be expected by chance. To avoid collisions, + // the number of possible key values must be sufficiently large. + + // How are more than 2^32 keys created? In each loop iteration, the + // algorithm sorts by random keys. Conceptually, the earlier iterations + // are sorting on the lower-order bits of larger keys that are never + // actually assembled. + + // The expected number of collisions is n - d + d(1 - 1/d)^n, where d is + // the number of possible keys and n is the number of values. If d = n^2, + // then the limit as n goes to infinity is 1/2. If d = n^3, then the limit + // as n goes to infinity is zero. + + // This implementation ensures that the key-space is greater than or equal + // to the cube of the number of values. The risk of collisions can be + // further reduced by increasing Exponent at the expense of + // performance. + + // For Exponent = 2, the expected number of collisions per shuffle is + // maximized at n = floor((2^32-1)^(1/2)) = 65535 where the expectation is + // about 1/2. + + // For Exponent = 3, the expected number of collisions per shuffle is + // maximized at n = floor((2^32-1)^(1/3)) = 1625 where the expectation is + // about 1/3255. + + // For Exponent = 4, the expected number of collisions per shuffle is + // maximized at n = floor((2^32-1)^(1/4)) = 255 where the expectation is + // about 1/132622. + constexpr int exponent = 3; + int64_t num_elements = input_type.getNumElements(); + uint32_t u32_max = std::numeric_limits::max(); + int rounds = + std::ceil(exponent * std::log(num_elements) / std::log(u32_max)); + + Value current = op.getValue(); + for (int i = 0; i < rounds; ++i) { + auto keys = + CreateRngUniform32(op.getLoc(), num_elements, /*lower_limit=*/0, + /*upper_limit=*/u32_max, &rewriter); + auto sorted = stablehlo::createSortOp( + &rewriter, op.getLoc(), {keys, current}, + {rewriter.getIntegerType(32), input_type.getElementType()}, + /*dimension=*/-1, /*isStable=*/false, + /*direction=*/stablehlo::ComparisonDirection::LT); + current = sorted.getResult(1); + } + rewriter.replaceOp(op, current); + return success(); + } + + // The Fisher-Yates algorithm. + + // Generate range(n) as the initial value for the indices to be swapped. + auto indices_type = tensorflow::GetTypeFromTFTensorShape( + {first_dim_size}, rewriter.getIntegerType(32)); + Value indices = stablehlo::IotaOp::create( + rewriter, op.getLoc(), indices_type, rewriter.getI64IntegerAttr(0)); + + // Generate random numbers to be used as swaps for the indices. + Value swaps = CreateRngUniform32(op.getLoc(), first_dim_size, 0, + first_dim_size, &rewriter); + + // While loop body to perform index swaps. + auto swap_body_fn = [&](Location loc, Value i, ArrayRef old_values, + SmallVectorImpl *new_values, + OpBuilder *builder) { + Value swaps = old_values[0]; + Value indices = old_values[1]; + + auto scalar_i32_type = + tensorflow::GetTypeFromTFTensorShape({}, builder->getIntegerType(32)); + + auto scalar_one = builder->getDenseI64ArrayAttr({1}); + + // We need to swap the indices[i] with indices[swaps[i]]. First get + // these index values. + Value source_index = stablehlo::DynamicSliceOp::create( + *builder, loc, indices, i, scalar_one); + Value swap_index = stablehlo::ReshapeOp::create( + *builder, loc, scalar_i32_type, + stablehlo::DynamicSliceOp::create(*builder, loc, swaps, i, + scalar_one)); + Value target_index = stablehlo::DynamicSliceOp::create( + *builder, loc, indices, swap_index, scalar_one); + + // Then perform the swap. + // indices[i] <- indices[swaps[i]] + indices = stablehlo::DynamicUpdateSliceOp::create( + *builder, loc, indices.getType(), indices, target_index, + llvm::ArrayRef(i)); + // indices[swaps[i]] <- indices[i] + indices = stablehlo::DynamicUpdateSliceOp::create( + *builder, loc, indices.getType(), indices, source_index, + llvm::ArrayRef(swap_index)); + + // Update new values. + new_values->assign({swaps, indices}); + }; + + // Create a while op to swap indices. + SmallVector while_output; + CreateWhile32(op.getLoc(), first_dim_size, swap_body_fn, {swaps, indices}, + &while_output, &rewriter); + Value swaped_indices = while_output[1]; + + // Gather the data using the swapped indices as the shuffled order. + auto slice_sizes = tensorflow::ConvertMlirShapeToTF(input_type.getShape()); + slice_sizes[0] = 1; + auto dims_attr = stablehlo::GatherDimensionNumbersAttr::get( + rewriter.getContext(), + /*offsetDims=*/llvm::to_vector<4>(llvm::seq(1, input_rank)), + /*collapsedSliceDims=*/{0}, + /*operandBatchingDims=*/{}, + /*startIndicesBatchingDims=*/{}, + /*startIndexMap=*/{0}, + /*indexVectorDim=*/1); + + SmallVector slice_sizes_values; + for (auto i = 0; i < slice_sizes.size(); ++i) { + if (slice_sizes[i] == tensorflow::kTFDynamicSize) { + Value i_const = arith::ConstantOp::create(rewriter, op.getLoc(), + rewriter.getIndexAttr(i)); + Value slice_size_index = + shape::DimOp::create(rewriter, op.getLoc(), op.getValue(), i_const); + Value index_to_i64 = arith::IndexCastOp::create( + rewriter, op.getLoc(), rewriter.getI64Type(), slice_size_index); + Value i64_to_tensor = tensor::FromElementsOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape({1}, rewriter.getI64Type()), + index_to_i64); + slice_sizes_values.push_back(i64_to_tensor); + } else { + slice_sizes_values.push_back(stablehlo::ConstantOp::create( + rewriter, op.getLoc(), + GetI64ElementsAttr({slice_sizes[i]}, &rewriter))); + } + } + + auto slice_sizes_concat = stablehlo::ConcatenateOp::create( + rewriter, op.getLoc(), slice_sizes_values, + rewriter.getI64IntegerAttr(0)); + rewriter.replaceOpWithNewOp( + op, op.getType(), op.getValue(), swaped_indices, slice_sizes_concat, + dims_attr); + + return success(); + } +}; + +// Converts an XlaSharding op to a XLA HLO shard op with sharding attributes. +class ConvertXlaShardingOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaShardingOp op, + PatternRewriter &rewriter) const override { + // TODO(b/148313088): define sharding attribute struct in MLIR intead of + // using a string. + if (!op.get_XlaSharding().has_value() && + !op.get_XlaShardingV2().has_value()) { + return failure(); + } + + auto sharding = tensorflow::GetXlaShardingAttrFromShardingOp(op); + if (!sharding.ok()) { + return failure(); + } + + NamedAttribute call_target_name = rewriter.getNamedAttr( + "call_target_name", rewriter.getStringAttr("Sharding")); + + auto custom_call = stablehlo::CustomCallOp::create( + rewriter, op.getLoc(), op.getType(), op.getInput(), + ArrayRef{call_target_name}); + custom_call->setAttr(kShardingAttr, *sharding); + rewriter.replaceOp(op, custom_call.getResult(0)); + return success(); + } +}; + +// Converts a TF InplaceUpdate op to DynamicUpdateSlice HLO. +class ConvertInplaceUpdateOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::InplaceUpdateOp op, + PatternRewriter &rewriter) const override { + auto input = mlir::dyn_cast>(op.getX()); + if (!input) return failure(); + auto indices = op.getI(); + auto updates = op.getV(); + + // Slice each row of `i` and `v` to perform a separate dynamic-update-slice + // on the contents of `x`. + auto input_type = mlir::cast(input.getType()); + auto updates_type = mlir::cast(updates.getType()); + auto indices_type = mlir::cast(indices.getType()); + if (!input_type.hasRank()) return failure(); + if (!updates_type.hasRank() || updates_type.isDynamicDim(0)) + return failure(); + if (!indices_type.hasStaticShape()) return failure(); + + if (indices_type.getRank() != 1) return failure(); + + SmallVector unpacked_indices_type( + indices_type.getDimSize(0), tensorflow::GetTypeFromTFTensorShape( + {}, indices_type.getElementType())); + // Note on zero_attr integer type: DynamicUpdateSlice op start_indices are + // required to have matching types. This rewrite rule creates + // DynamicUpdateSlice ops where the first "start index" is always i32 and + // subsequent ones are constructed based on zero_attr. Thus the type + // for zero_attr needs to be i32 as well. + auto zero_attr = IntegerAttr::get(rewriter.getIntegerType(32), 0); + auto unpacked_indices = TF::UnpackOp::create( + rewriter, op.getLoc(), unpacked_indices_type, indices, zero_attr); + + SmallVector split_updates_shape; + split_updates_shape.append(updates_type.getShape().begin(), + updates_type.getShape().end()); + split_updates_shape.front() = 1; + SmallVector split_updates_type; + split_updates_type.resize( + updates_type.getShape().front(), + tensorflow::GetTypeFromTFTensorShape(split_updates_shape, + updates_type.getElementType())); + + auto cst = stablehlo::ConstantOp::create(rewriter, op.getLoc(), zero_attr) + .getResult(); + auto split_updates = TF::SplitOp::create(rewriter, op.getLoc(), + split_updates_type, cst, updates); + + SmallVector input_indices; + input_indices.resize(input_type.getRank(), cst); + + for (auto pair : + llvm::zip(unpacked_indices.getOutput(), split_updates.getOutput())) { + input_indices.front() = std::get<0>(pair); + input = stablehlo::DynamicUpdateSliceOp::create( + rewriter, op.getLoc(), op.getType(), input, std::get<1>(pair), + input_indices); + } + + rewriter.replaceOp(op, input); + return success(); + } +}; + +// Converts a TF XlaDynamicUpdateSlice op to DynamicUpdateSlice HLO. +class ConvertXlaDynamicUpdateSliceOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaDynamicUpdateSliceOp op, + PatternRewriter &rewriter) const override { + auto indices_type = + mlir::dyn_cast(op.getIndices().getType()); + if (!indices_type || !indices_type.hasStaticShape() || + indices_type.getShape().size() != 1) + return failure(); + + SmallVector unpacked_indices_type( + indices_type.getDimSize(0), tensorflow::GetTypeFromTFTensorShape( + {}, indices_type.getElementType())); + auto unpacked_indices = TF::UnpackOp::create( + rewriter, op.getLoc(), unpacked_indices_type, op.getIndices(), + IntegerAttr::get(rewriter.getIntegerType(64), 0)); + rewriter.replaceOpWithNewOp( + op, op.getType(), op.getInput(), op.getUpdate(), + unpacked_indices.getOutput()); + return success(); + } +}; + +// Converts a TF XlaReduceScatter op to ReduceScatter HLO. +class ConvertXlaReduceScatterOp + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaReduceScatterOp op, + PatternRewriter &rewriter) const override { + DenseIntElementsAttr group_assignment; + if (!matchPattern(op.getGroupAssignment(), m_Constant(&group_assignment))) + return failure(); + auto replica_groups = + mlir::cast(hlo::convertElementsAttr( + group_assignment, rewriter.getIntegerType(64))); + if (replica_groups.getType().getRank() != 2) return failure(); + + APInt scatter_dimension; + if (!matchPattern(op.getScatterDimension(), + m_ConstantInt(&scatter_dimension))) + return failure(); + + Location loc = op.getLoc(); + Type element_type = getElementTypeOrSelf(op.getInput().getType()); + + auto reduce_scatter = stablehlo::ReduceScatterOp::create( + rewriter, loc, op.getType(), op.getInput(), + rewriter.getIntegerAttr(rewriter.getIntegerType(64), + scatter_dimension.getSExtValue()), + replica_groups, stablehlo::ChannelHandleAttr()); + StringRef reduce_op = op.getReduceOp(); + if (reduce_op == "Add") { + BuildReduceBody( + element_type, &reduce_scatter.getComputation(), &rewriter); + } else if (reduce_op == "Mul") { + BuildReduceBody( + element_type, &reduce_scatter.getComputation(), &rewriter); + } else if (reduce_op == "Min") { + BuildReduceBody( + element_type, &reduce_scatter.getComputation(), &rewriter); + } else if (reduce_op == "Max") { + BuildReduceBody( + element_type, &reduce_scatter.getComputation(), &rewriter); + } else { + // For mean, add replicas in the same group. Then divide the sum by the + // number of replicas in each group below. + assert(reduce_op == "Mean"); + BuildReduceBody( + element_type, &reduce_scatter.getComputation(), &rewriter); + } + Value result = reduce_scatter.getResult(); + + // For mean, divide the merge result by group size. + if (reduce_op == "Mean") { + int64_t replica_group_size = replica_groups.getType().getDimSize(1); + if (replica_group_size == 0) return failure(); + auto divisor = GetScalarConstOfType(element_type, loc, replica_group_size, + &rewriter); + auto broadcast_dims = rewriter.getDenseI64ArrayAttr({}); + result = chlo::BroadcastDivOp::create( + rewriter, loc, result, divisor.getResult(), broadcast_dims); + } + + rewriter.replaceOp(op, {result}); + return success(); + } +}; + +// Converts tf.XlaReduceWindow to stablehlo.ReduceWindow +class ConvertXlaReduceWindowOp + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaReduceWindowOp op, + PatternRewriter &rewriter) const override { + DenseElementsAttr window_dimensions, window_strides, base_dilations, + window_dilations, padding; + if (!(matchPattern(op.getWindowDimensions(), + m_Constant(&window_dimensions)) && + matchPattern(op.getWindowStrides(), m_Constant(&window_strides)) && + matchPattern(op.getBaseDilations(), m_Constant(&base_dilations)) && + matchPattern(op.getWindowDilations(), + m_Constant(&window_dilations)) && + matchPattern(op.getPadding(), m_Constant(&padding)))) + return failure(); + + Location loc = op.getLoc(); + + SmallVector result_types{op.getResult().getType()}; + // Create the stablehlo.SelectAndScatter op. + auto reduce_window_op = stablehlo::ReduceWindowOp::create( + rewriter, loc, result_types, op.getInput(), op.getInitValue(), + ToDenseI64ArrayAttr(window_dimensions, &rewriter), + ToDenseI64ArrayAttr(window_strides, &rewriter), + ToDenseI64ArrayAttr(base_dilations, &rewriter), + ToDenseI64ArrayAttr(window_dilations, &rewriter), + mlir::cast( + hlo::convertElementsAttr(padding, rewriter.getIntegerType(64)))); + // Insert a call to the reducer in the region of the mhlo op. + mlir::SymbolRefAttr func = op.getComputation(); + auto func_op = cast(SymbolTable::lookupSymbolIn( + op->getParentOfType(), func)); + auto func_ty = func_op.getFunctionType(); + BuildBodyWithCall(rewriter, loc, func, func_ty, + &reduce_window_op.getBody()); + + rewriter.replaceOp(op, reduce_window_op.getResults()); + + return success(); + } +}; + +// Converts ClipByValue to XLA's clamp operation. Includes the broadcasting +// semantics for static and dynamic cases. +class ConvertClipByValueOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::ClipByValueOp op, + PatternRewriter &rewriter) const override { + Value input = op.getX(); + Value min = op.getClipValueMin(); + Value max = op.getClipValueMax(); + + auto input_ty = mlir::cast(input.getType()); + auto min_ty = mlir::cast(min.getType()); + auto max_ty = mlir::cast(max.getType()); + + if (!input_ty.hasRank() || !min_ty.hasRank() || !max_ty.hasRank()) { + return failure(); + } + + auto shape = + TF::ShapeOp::create(rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape( + {input_ty.getRank()}, rewriter.getI32Type()), + input); + + if (min_ty != input_ty) { + min = TF::BroadcastToOp::create(rewriter, op.getLoc(), input_ty, min, + shape); + } + + if (max_ty != input_ty) { + max = TF::BroadcastToOp::create(rewriter, op.getLoc(), input_ty, max, + shape); + } + + rewriter.replaceOpWithNewOp(op, input_ty, min, input, + max); + return success(); + } +}; + +// Converts ConstOp to XLA's constant operation and introduces a tensor cast if +// needed. +class ConvertConstOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::ConstOp op, + PatternRewriter &rewriter) const override { + // Convert only for valid HLO tensors. + auto ty = mlir::dyn_cast(op.getType()); + if (!ty || + !mlir::isa(ty.getElementType())) + return failure(); + + Location loc = op.getLoc(); + Value result = stablehlo::ConstantOp::create(rewriter, loc, op.getValue()); + if (result.getType() != op.getType()) + result = tensor::CastOp::create(rewriter, loc, op.getType(), result); + rewriter.replaceOp(op, result); + return success(); + } +}; + +// Converts the Cumsum or Cumprod TensorFlow op to the HLO ReduceWindow op by +// setting appropriate window dimensions, with the given aggregation op as the +// reduction function. The input tensor needs to have a static shape, and 'axis' +// must be const. The TableGen pattern is not used for this rewrite because it +// involves regions. +template +class ConvertCumOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(OpT op, + PatternRewriter &rewriter) const override { + auto input = mlir::dyn_cast>(op.getX()); + if (!input) { + return rewriter.notifyMatchFailure(op, "input X not ranked tensor"); + } + auto input_type = mlir::dyn_cast(input.getType()); + if (!input_type || !input_type.hasStaticShape()) { + return rewriter.notifyMatchFailure(op, "input not static shape"); + } + + ArrayRef input_shape = input_type.getShape(); + int64_t rank = input_shape.size(); + + // We can only match when the axis is a constant scalar. + DenseIntElementsAttr axis_attr; + if (!matchPattern(op.getAxis(), m_Constant(&axis_attr))) { + return rewriter.notifyMatchFailure(op, "axis not constant"); + } + + // Get the dimension to apply the reduction on, and offset properly if it is + // negative. + int64_t axis = (*axis_attr.begin()).getSExtValue(); + if (axis < 0) { + axis += rank; + } + + // If we're supposed to sum things up in the reverse direction, we reverse + // the input and then later reverse the output. + if (op.getReverse()) { + llvm::SmallVector dims_to_reverse({axis}); + input = stablehlo::ReverseOp::create( + rewriter, op.getLoc(), input, + GetI64ArrayAttr(dims_to_reverse, &rewriter)); + } + + // Convert if we need to enlarge the element type's bitwidth to avoid + // precision loss. + Type input_element_type = input_type.getElementType(); + + // TODO(hinsu): Handle complex element types. + if (!input_element_type.isIntOrFloat()) { + return rewriter.notifyMatchFailure(op, + "input element type not int or float"); + } + + Type sum_element_type = GetSumAccumulationType(input_element_type); + input = stablehlo::ConvertOp::create(rewriter, op.getLoc(), input, + sum_element_type); + + SmallVector window_dims(rank, 1); + SmallVector window_strides(rank, 1); + window_dims[axis] = input_shape[axis]; + + SmallVector paddings(rank * 2, 0); + paddings[axis * 2] = + std::max(input_shape[axis] - 1, static_cast(0)); + auto paddings_attr = + DenseIntElementsAttr::get(tensorflow::GetTypeFromTFTensorShape( + {rank, 2}, rewriter.getIntegerType(64)), + paddings); + + int64_t init_value = + (std::is_same::value) ? 0 : 1; + Value init = GetScalarConstOfType(sum_element_type, op.getLoc(), init_value, + &rewriter); + + auto reduce = stablehlo::ReduceWindowOp::create( + rewriter, op.getLoc(), input.getType(), input, init, + GetI64ArrayAttr(window_dims, &rewriter), + GetI64ArrayAttr(window_strides, &rewriter), + /*base_dilations=*/DenseI64ArrayAttr(), + /*window_dilations=*/DenseI64ArrayAttr(), paddings_attr); + BuildReduceBody(sum_element_type, &reduce.getBody(), + &rewriter); + Value result = reduce.getResult(0); + + if (op.getExclusive()) { + // In "exclusive" operation, the output will start with the "init" (0) + // values. There is no way to express that as a ReduceWindowOp, so run the + // normal operation, and then use a PadOp to add the 0 "column" on the + // left and cut away the last column on the right. + llvm::SmallVector low_padding(rank, 0); + llvm::SmallVector high_padding(rank, 0); + llvm::SmallVector interior_padding(rank, 0); + low_padding[axis] = 1; + high_padding[axis] = -1; + result = stablehlo::PadOp::create( + rewriter, op.getLoc(), result, init, + GetI64ArrayAttr(low_padding, &rewriter), + GetI64ArrayAttr(high_padding, &rewriter), + GetI64ArrayAttr(interior_padding, &rewriter)); + } + + // Convert back if we enlarged the element type's bitwidth. + result = stablehlo::ConvertOp::create(rewriter, op.getLoc(), result, + input_element_type); + + if (op.getReverse()) { + llvm::SmallVector dims_to_reverse({axis}); + result = stablehlo::ReverseOp::create( + rewriter, op.getLoc(), result, + GetI64ArrayAttr(dims_to_reverse, &rewriter)); + } + + rewriter.replaceOp(op, result); + return success(); + } +}; + +using ConvertCumsumOp = ConvertCumOp; +using ConvertCumprodOp = ConvertCumOp; + +// Converts the Tensorflow ShapeOp to a sequence of Shape dialect and Standard +// dialect lowerings. This involves extracting the shape type, extracting and +// converting each dimension to a known integer type, and repacking into a final +// tensor. +class ConvertShapeOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::ShapeOp op, + PatternRewriter &rewriter) const override { + Value input = op.getInput(); + + auto result_ty = mlir::dyn_cast(op.getResult().getType()); + if (!result_ty) { + return failure(); + } + + auto index_tensor = tensorflow::GetTypeFromTFTensorShape( + result_ty.getShape(), rewriter.getIndexType()); + auto shape_op = + shape::ShapeOfOp::create(rewriter, op.getLoc(), index_tensor, input); + rewriter.replaceOpWithNewOp(op, result_ty, shape_op); + return success(); + } +}; + +class ConvertDynamicExpandDimsOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::ExpandDimsOp op, + PatternRewriter &rewriter) const override { + auto input = op.getInput(); + auto input_ty = mlir::cast(input.getType()); + auto result_ty = mlir::cast(op.getType()); + if (!result_ty.hasRank() || !input_ty.hasRank() || + result_ty.hasStaticShape()) { + return failure(); + } + + DenseIntElementsAttr expand_dims_attr; + if (!matchPattern(op.getDim(), m_Constant(&expand_dims_attr))) { + return failure(); + } + + auto shape = shape::ShapeOfOp::create( + rewriter, op.getLoc(), + tensorflow::GetTypeFromTFTensorShape({input_ty.getRank()}, + rewriter.getIndexType()), + input); + auto expand_dims = llvm::to_vector<6>(expand_dims_attr.getValues()); + + llvm::SmallVector dims; + dims.resize(result_ty.getRank()); + + auto inserted_dim = expand_dims[0].getSExtValue(); + + // Handle the negative value use case. + if (inserted_dim < 0) { + inserted_dim += result_ty.getRank(); + // This means the value is completely incorrect, just return. + if (inserted_dim < 0) { + return failure(); + } + } + + dims[inserted_dim] = + arith::ConstantIndexOp::create(rewriter, op.getLoc(), 1); + + for (int i = 0; i < dims.size() - 1; i++) { + // Add the extracted dim. + Value index = arith::ConstantIndexOp::create(rewriter, op.getLoc(), i); + Value dim = + tensor::ExtractOp::create(rewriter, op.getLoc(), shape, index); + dims[i >= inserted_dim ? i + 1 : i] = dim; + } + + auto from_extents = + tensor::FromElementsOp::create(rewriter, op.getLoc(), dims); + rewriter.replaceOpWithNewOp( + op, result_ty, input, from_extents); + return success(); + } +}; + +class ConvertDynamicSqueezeOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::SqueezeOp op, + PatternRewriter &rewriter) const override { + auto input = op.getInput(); + auto input_ty = mlir::cast(input.getType()); + auto result_ty = mlir::cast(op.getType()); + if (!result_ty.hasRank() || !input_ty.hasRank() || + result_ty.hasStaticShape()) { + return failure(); + } + + // The fully dynamic case is unsupported. + if (op.getSqueezeDims().empty()) { + return failure(); + } + + SmallVector squeeze_dims; + int64_t input_rank = input_ty.getRank(); + for (const auto &squeeze_dim_apint : + op.getSqueezeDims().getAsValueRange()) { + int64_t squeeze_dim = squeeze_dim_apint.getSExtValue(); + // Handle negative inputs. + if (squeeze_dim < 0) squeeze_dim += input_rank; + assert(squeeze_dim >= 0 && squeeze_dim < input_rank && + "squeeze dim out of bounds"); + + squeeze_dims.push_back(squeeze_dim); + } + + // Collect the unsqueezed dimensions. + llvm::SmallVector dims; + for (int64_t i = 0; i != input_rank; ++i) { + if (llvm::is_contained(squeeze_dims, i)) continue; + dims.push_back(tensor::DimOp::create(rewriter, op.getLoc(), input, i)); + } + + auto from_extents = + tensor::FromElementsOp::create(rewriter, op.getLoc(), dims); + rewriter.replaceOpWithNewOp( + op, result_ty, input, from_extents); + return success(); + } +}; + +// Converts tf.XlaConvV2 to stablehlo.Conv +class ConvertXlaConvV2Op : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaConvV2Op op, + PatternRewriter &rewriter) const override { + DenseElementsAttr window_strides_attr, padding_attr, lhs_dilation_attr, + rhs_dilation_attr, feature_group_count_attr; + if (!(matchPattern(op.getWindowStrides(), + m_Constant(&window_strides_attr)) && + matchPattern(op.getPadding(), m_Constant(&padding_attr)) && + matchPattern(op.getLhsDilation(), m_Constant(&lhs_dilation_attr)) && + matchPattern(op.getRhsDilation(), m_Constant(&rhs_dilation_attr)) && + matchPattern(op.getFeatureGroupCount(), + m_Constant(&feature_group_count_attr)))) + return failure(); + + auto window_strides_named_attr = rewriter.getNamedAttr( + "window_strides", ToDenseI64ArrayAttr(window_strides_attr, &rewriter)); + + auto padding_named_attr = rewriter.getNamedAttr( + "padding", mlir::cast(hlo::convertElementsAttr( + padding_attr, rewriter.getIntegerType(64)))); + + auto lhs_dilation_named_attr = rewriter.getNamedAttr( + "lhs_dilation", ToDenseI64ArrayAttr(lhs_dilation_attr, &rewriter)); + + auto rhs_dilation_named_attr = rewriter.getNamedAttr( + "rhs_dilation", ToDenseI64ArrayAttr(rhs_dilation_attr, &rewriter)); + + int64_t feature_group_count_val = + feature_group_count_attr.getValues()[0].getInt(); + auto feature_group_count_named_attr = rewriter.getNamedAttr( + "feature_group_count", + rewriter.getI64IntegerAttr(feature_group_count_val)); + + auto batch_group_count_named_attr = + rewriter.getNamedAttr("batch_group_count", op.getBatchGroupCountAttr()); + + xla::ConvolutionDimensionNumbers dnums; + dnums.ParseFromString(op.getDimensionNumbersAttr().getValue().str()); + auto dimension_numbers_named_attr = rewriter.getNamedAttr( + "dimension_numbers", + xla::stablehlo::ConvertConvDimensionNumbers(dnums, &rewriter)); + + xla::PrecisionConfig precision_config; + precision_config.ParseFromString( + op.getPrecisionConfigAttr().getValue().str()); + auto precision_config_named_attr = rewriter.getNamedAttr( + "precision_config", + xla::stablehlo::ConvertPrecisionConfig(&precision_config, &rewriter)); + + Value lhs = op.getLhs(); + Value rhs = op.getRhs(); + auto lhs_type = mlir::cast(lhs.getType()); + auto rhs_type = mlir::cast(rhs.getType()); + auto lhs_element_type = lhs_type.getElementType(); + auto rhs_element_type = rhs_type.getElementType(); + if (lhs_element_type != rhs_element_type) { + assert(lhs_element_type.getIntOrFloatBitWidth() >= + rhs_element_type.getIntOrFloatBitWidth()); + auto new_rhs_type = + RankedTensorType::get(rhs_type.getShape(), lhs_element_type); + rhs = stablehlo::ConvertOp::create(rewriter, op.getLoc(), new_rhs_type, + rhs); + } + + SmallVector operands{lhs, rhs}; + NamedAttribute attrs[] = { + window_strides_named_attr, padding_named_attr, + lhs_dilation_named_attr, rhs_dilation_named_attr, + feature_group_count_named_attr, batch_group_count_named_attr, + dimension_numbers_named_attr, precision_config_named_attr}; + rewriter.replaceOpWithNewOp( + op, op.getType(), operands, llvm::ArrayRef(attrs)); + return success(); + } +}; + +// Converts tf.XlaSelectAndScatter to stablehlo.SelectAndScatter +class ConvertXlaSelectAndScatterOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaSelectAndScatterOp op, + PatternRewriter &rewriter) const override { + ElementsAttr window_dimensions, window_strides, padding; + if (!(matchPattern(op.getWindowDimensions(), + m_Constant(&window_dimensions)) && + matchPattern(op.getWindowStrides(), m_Constant(&window_strides)) && + matchPattern(op.getPadding(), m_Constant(&padding)))) + return failure(); + + Location loc = op.getLoc(); + + SmallVector result_types{op.getResult().getType()}; + // Create the stablehlo.SelectAndScatter op. + auto select_and_scatter_op = stablehlo::SelectAndScatterOp::create( + rewriter, loc, result_types, op.getOperand(), op.getSource(), + op.getInitValue(), ToDenseI64ArrayAttr(window_dimensions, &rewriter), + ToDenseI64ArrayAttr(window_strides, &rewriter), + mlir::cast( + hlo::convertElementsAttr(padding, rewriter.getIntegerType(64)))); + + auto insert_call_to = [&](const mlir::SymbolRefAttr &func, Region *region) { + auto func_op = cast(SymbolTable::lookupSymbolIn( + op->getParentOfType(), func)); + auto func_ty = func_op.getFunctionType(); + BuildBodyWithCall(rewriter, loc, func, func_ty, region); + }; + + // Insert a call to the select function in the select region of the mhlo op. + insert_call_to(op.getSelect(), &select_and_scatter_op.getSelect()); + // Insert a call to the scatter function in the scatter region of the mhlo + // op. + insert_call_to(op.getScatter(), &select_and_scatter_op.getScatter()); + + rewriter.replaceOp(op, select_and_scatter_op.getResult()); + + return success(); + } +}; + +// Convert tf.XlaSort to stablehlo.Sort +class ConvertXlaSortOp : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaSortOp op, + PatternRewriter &rewriter) const override { + // Create the sort op. + Type element_type = getElementTypeOrSelf(op.getInput().getType()); + auto sort_op = stablehlo::createSortOp( + &rewriter, op.getLoc(), {op.getInput()}, {element_type}, + /*dimension=*/-1, /*isStable=*/false, + /*direction=*/stablehlo::ComparisonDirection::LT); + rewriter.replaceOp(op, sort_op.getResult(0)); + return success(); + } +}; + +inline std::optional TensorFlowRngAlgToXla( + tensorflow::Algorithm alg) { + if (alg == tensorflow::RNG_ALG_PHILOX) { + return xla::RandomAlgorithm::RNG_PHILOX; + } else if (alg == tensorflow::RNG_ALG_THREEFRY) { + return xla::RandomAlgorithm::RNG_THREE_FRY; + } else if (alg == tensorflow::RNG_ALG_AUTO_SELECT) { + return xla::RandomAlgorithm::RNG_DEFAULT; + } + return std::nullopt; +} + +// Converts tf.XlaRngBitGenerator op to stablehlo.RngBitGenerator op. +class ConvertXlaRngBitGeneratorOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaRngBitGeneratorOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + DenseElementsAttr algorithm; + if (!(matchPattern(op.getAlgorithm(), m_Constant(&algorithm))) || + algorithm.getType().getRank()) { + return op.emitOpError() << "algorithm must be a constant scalar"; + } + auto alg = static_cast( + algorithm.getValues()[0].getInt()); + auto xla_alg = TensorFlowRngAlgToXla(alg); + if (!xla_alg) { + return op.emitOpError() << "unknown algorithm"; + } + + auto algorithm_attr = mlir::stablehlo::RngAlgorithmAttr::get( + rewriter.getContext(), + *mlir::stablehlo::symbolizeRngAlgorithm(xla_alg.value())); + auto rng_bit_generator_op = stablehlo::RngBitGeneratorOp::create( + rewriter, loc, op.getResultTypes(), algorithm_attr, + op.getInitialState()); + + rewriter.replaceOp(op, rng_bit_generator_op.getResults()); + + return success(); + } +}; + +// Converts tf.XlaVariadicReduceV2 to stablehlo.Reduce +class ConvertXlaVariadicReduceV2Op + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaVariadicReduceV2Op op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + + mlir::SymbolRefAttr func = op.getReducer(); + auto func_op = cast(SymbolTable::lookupSymbolIn( + op->getParentOfType(), func)); + auto func_ty = func_op.getFunctionType(); + SmallVector elementTypes{llvm::map_range( + func_ty.getResults(), + [](Type ty) { return mlir::cast(ty).getElementType(); })}; + + // Create the stablehlo.reduce op. + auto reduce_op = stablehlo::ReduceOp::create( + rewriter, loc, op.getInputs(), op.getInitValues(), + ToDenseI64ArrayAttr(GetI64ElementsAttr(op.getDimensionsToReduce()), + &rewriter), + elementTypes); + + // Insert a call to the reducer in the region of the mhlo op. + BuildBodyWithCall(rewriter, loc, func, func_ty, &reduce_op.getBody()); + + rewriter.replaceOp(op, reduce_op.getResults()); + + return success(); + } +}; + +// Convert tf.XlaVariadicSort to stablehlo.Sort +class ConvertXlaVariadicSortOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaVariadicSortOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + ElementsAttr dimension; + matchPattern(op.getDimension(), m_Constant(&dimension)); + // Create the stablehlo.sort op. + auto sort_op = stablehlo::SortOp::create( + rewriter, loc, op.getInputs(), + dimension.getValues()[0].getInt(), op.getIsStable()); + mlir::SymbolRefAttr func = op.getComparator(); + auto func_op = cast(SymbolTable::lookupSymbolIn( + op->getParentOfType(), func)); + auto func_ty = func_op.getFunctionType(); + // Insert a call to the reducer in the region of the mhlo op. + BuildBodyWithCall(rewriter, loc, func, func_ty, &sort_op.getComparator()); + + rewriter.replaceOp(op, sort_op.getResults()); + return success(); + } +}; + +// Convert tf.XlaReducePrecision to stablehlo.ReducePrecision +class ConvertXlaReducePrecisionOp + : public OpRewritePattern { + public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaReducePrecisionOp op, + PatternRewriter &rewriter) const override { + IntegerType int32_type = rewriter.getIntegerType(32); + APInt exponent_bits = op.getExponentBitsAttr().getValue(); + // Truncating to 32-bits is safe, since pasing any number above the dtype + // size (which is at most 64, for float64) is equivalent to passing the + // dtype size. + IntegerAttr new_exponent_attr = + IntegerAttr::get(int32_type, exponent_bits.truncSSat(32)); + APInt mantissa_bits = op.getMantissaBitsAttr().getValue(); + IntegerAttr new_mantissa_attr = + IntegerAttr::get(int32_type, mantissa_bits.truncSSat(32)); + rewriter.replaceOpWithNewOp( + op, op.getType(), op.getOperand(), new_exponent_attr, + new_mantissa_attr); + return success(); + } +}; + +class LowerYieldOp : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + TF::YieldOp op, TF::YieldOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, adaptor.getOperands()); + return success(); + } +}; + +// Returns a new tensor type from the given type with element type updated to +// the given type. +TensorType UpdateElementTypeTo(Type ty, Type element_ty) { + auto ranked_ty = mlir::dyn_cast(ty); + if (!ranked_ty) { + return UnrankedTensorType::get(element_ty); + } + return RankedTensorType::get(ranked_ty.getShape(), element_ty, + ranked_ty.getEncoding()); +} + +template +class LowerControlFlowOp : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + SrcOpT op, typename SrcOpT::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + DstOpT stablehlo_op; + Location loc = op.getLoc(); + + // To handle quant type conversions, use the converted operands' element + // types and original source op's shapes and encoding to get converted op's + // result types. This is only done for the While op for now. + llvm::SmallVector element_types; + int64_t num_results = op.getNumResults(); + if constexpr (std::is_same::value) { + element_types.reserve(num_results); + for (Value value : adaptor.getOperands()) { + element_types.push_back(getElementTypeOrSelf(value.getType())); + } + } + + if constexpr (std::is_same::value) { + // Explicitly handle the Case op because it has variadic regions and takes + // the number of regions as an input along with the operands. + stablehlo_op = + DstOpT::create(rewriter, loc, op.getResultTypes(), + adaptor.getBranchIndex(), op.getBranches().size()); + } else if constexpr (std::is_same::value) { + llvm::SmallVector while_result_types; + while_result_types.reserve(num_results); + for (int64_t idx = 0; idx < num_results; ++idx) { + auto ty = UpdateElementTypeTo(op.getType(idx), element_types[idx]); + while_result_types.push_back(ty); + } + + stablehlo_op = DstOpT::create( + rewriter, loc, TypeRange(while_result_types), adaptor.getOperands()); + } else { + stablehlo_op = DstOpT::create(rewriter, loc, op.getResultTypes(), + adaptor.getOperands()); + } + + int64_t num_regions = op.getNumRegions(); + for (int64_t idx = 0; idx < num_regions; ++idx) { + Region ®ion = stablehlo_op.getBodyRegion(idx); + rewriter.inlineRegionBefore(op.getBodyRegion(idx), region, region.end()); + + // Update region's entry blocks argument types to handle quantized element + // types. + if constexpr (std::is_same::value) { + TypeConverter::SignatureConversion signature(num_results); + Block &block = region.front(); + for (const auto &[block_idx, original_ty] : + llvm::enumerate(block.getArgumentTypes())) { + TensorType updated_ty = + UpdateElementTypeTo(original_ty, element_types[block_idx]); + signature.addInputs(block_idx, {updated_ty}); + } + rewriter.applySignatureConversion(®ion.front(), signature); + } + } + + // Replace all uses of `op` results with the newly created op. + rewriter.replaceOp(op, stablehlo_op); + return success(); + } +}; +} // end namespace + +#include "tensorflow/compiler/mlir/tf2xla/transforms/generated_legalize_tf.inc" + +// LINT.IfChange +void PopulateLegalizeTfPatterns(MLIRContext *context, + RewritePatternSet *patterns) { + populateWithGenerated(*patterns); + // clang-format off + patterns->add< + ConvertAllOp, + ConvertAnyOp, + ConvertArgMaxOp, + ConvertArgMinOp, + ConvertBatchMatMulV2Op, + ConvertBiasAddOp, + ConvertBroadcastToOp, + ConvertBF16FloorDivOp, + ConvertClipByValueOp, + ConvertConstOp, + ConvertConv2DOp, + ConvertConv3DOp, + ConvertDepthConv2DOp, + ConvertConv2DBackpropFilterOp, + ConvertConv3DBackpropFilterOp, + ConvertConv2DBackpropInputOp, + ConvertConv3DBackpropInputOp, + ConvertCumprodOp, + ConvertCumsumOp, + ConvertDiagPartOp, + ConvertDynamicExpandDimsOp, + ConvertDynamicSqueezeOp, + ConvertEinsumOp, + ConvertRFFTOp, + ConvertIRFFTOp, + ConvertFusedBatchNormGradOp, + ConvertFusedBatchNormGradV2Op, + ConvertFusedBatchNormGradV3Op, + ConvertFusedBatchNormV2Op, + ConvertFusedBatchNormV3Op, + ConvertInfeedDequeueTupleOp, + ConvertIdentityNOp, + ConvertInplaceUpdateOp, + ConvertLinSpaceOp, + ConvertMaxOp, + ConvertMinOp, + ConvertAvgPool2DOp, + ConvertAvgPool3DOp, + ConvertAvgPool2DGradOp, + ConvertAvgPool3DGradOp, + ConvertMaxPool2DOp, + ConvertMaxPool3DOp, + ConvertMaxPool2DGradOp, + ConvertMaxPool3DGradOp, + ConvertMeanOp, + ConvertOneHotOp, + ConvertOutfeedEnqueueTupleOp, + ConvertProdOp, + ConvertDynamicRangeOp, + ConvertMatrixDiagPartV3Op, + ConvertRangeOp, + ConvertSelectOp, + ConvertShapeOp, + ConvertSplitOp, + ConvertSplitVOp, + ConvertStridedSliceOp, + ConvertStridedSliceGradOp, + ConvertSumOp, + ConvertTensorScatterAddOp, + ConvertTensorScatterSubOp, + ConvertTensorScatterMinOp, + ConvertTensorScatterMaxOp, + ConvertTensorScatterUpdateOp, + ConvertTileOp, + ConvertTopKV2Op, + ConvertUnpackOp, + ConvertUnsortedSegmentMaxOp, + ConvertUnsortedSegmentMinOp, + ConvertUnsortedSegmentProdOp, + ConvertUnsortedSegmentSumOp, + ConvertRandomShuffleOp, + ConvertXlaShardingOp, + ConvertXlaDynamicUpdateSliceOp, + ConvertXlaConvV2Op, + ConvertXlaReducePrecisionOp, + ConvertXlaReduceScatterOp, + ConvertXlaReduceWindowOp, + ConvertXlaRngBitGeneratorOp, + ConvertXlaSelectAndScatterOp, + ConvertXlaSortOp, + ConvertXlaVariadicReduceV2Op, + ConvertXlaVariadicSortOp, + ConvertRollOp, + ConvertLeakyReluOp, + ConvertLeakyReluGradOp, + ConvertSplitOpDynamic, + ConvertSliceOpDynamic, + ConvertTileOpDynamic, + ConvertUnpackOpDynamic, + ConvertSigmoidGradOpDynamic, + ConvertConv2DDynamic, + ConvertPadOpDynamic, + ConvertGatherNdOpDynamic, + LowerControlFlowOp, + LowerControlFlowOp, + LowerControlFlowOp, + LowerYieldOp>(context); + // clang-format on +} +// LINT.ThenChange(:MlirAlwaysOps) + +} // namespace hlo +} // end namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_collective.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_collective.cc new file mode 100644 index 0000000000000000000000000000000000000000..ec0a4c291ddb7b4ad636b16d4c961af5d327fe75 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_collective.cc @@ -0,0 +1,413 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements logic for lowering TensorFlow dialect's collective +// ops (TF/XLA) to the HLO dialect. + +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/utils.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/mlir_hlo/utils/convert_op_folder.h" +#include "xla/mlir_hlo/utils/hlo_utils.h" +#include "xla/xla_data.pb.h" + +namespace mlir { +namespace mhlo { + +namespace { + +constexpr absl::string_view kGroupSizeAttrName = + "tf2xla.collective_info.group_size"; +constexpr absl::string_view kGroupKeyAttrName = + "tf2xla.collective_info.group_key"; + +#define GEN_PASS_DEF_LEGALIZETFCOLLECTIVE +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +class LegalizeTFCollective + : public impl::LegalizeTFCollectiveBase { + public: + void runOnOperation() override; +}; + +LogicalResult SetOnceModuleAttribute(StringRef attr_name, + IntegerAttr attr_value, Operation* op, + ModuleOp& module) { + const auto ex_attr_value = module->getAttrOfType(attr_name); + if (ex_attr_value == nullptr) { + module->setAttr(attr_name, attr_value); + return success(); + } + if (ex_attr_value == attr_value) { + return success(); + } + return op->emitOpError() << "module already contains an attribute " + << attr_name << "=" << ex_attr_value.getInt() + << ", overwritting to a new value " + << attr_value.getInt() << " is not allowed."; +} + +LogicalResult SetCollectiveInfo(IntegerAttr group_size, IntegerAttr group_key, + Operation* op) { + ModuleOp module = op->getParentOfType(); + // The StringRef cast is necessary before cxx14. + if (failed(SetOnceModuleAttribute( + StringRef(kGroupSizeAttrName.data(), kGroupSizeAttrName.size()), + group_size, op, module))) { + return failure(); + } + if (failed(SetOnceModuleAttribute( + StringRef(kGroupKeyAttrName.data(), kGroupKeyAttrName.size()), + group_key, op, module))) { + return failure(); + } + return success(); +} + +LogicalResult SetCollectiveInfo(OpBuilder& builder, + DenseIntElementsAttr replica_groups, + Operation* op) { + // Use special group_key 0 to represent "all available devices". This + // shall resolve to a DeviceAssignment that includes all devices intended for + // replica_groups. + IntegerAttr group_size = builder.getI32IntegerAttr(replica_groups.size()); + IntegerAttr group_key = builder.getI32IntegerAttr(0); + return SetCollectiveInfo(group_size, group_key, op); +} + +LogicalResult ConvertReplicaGroups(OpBuilder& builder, + Value group_assignment_value, + DenseIntElementsAttr& replica_groups, + Operation* op) { + DenseIntElementsAttr group_assignment; + if (!matchPattern(group_assignment_value, m_Constant(&group_assignment))) { + return op->emitOpError() << "expects constant group_assignment"; + } + replica_groups = mlir::cast( + hlo::convertElementsAttr(group_assignment, builder.getIntegerType(64))); + if (replica_groups.getType().getRank() != 2) { + return op->emitOpError() << "group_assignment should have rank 2, got " + << replica_groups.getType().getRank(); + } + return success(); +} + +ChannelHandleAttr ConvertChannel(OpBuilder& builder, int64_t channel_id, + StringRef mode) { + if (mode == "CrossReplica") { + return ChannelHandleAttr(); + } + return ChannelHandleAttr::get(builder.getContext(), + /*handle=*/channel_id, + /*type=*/xla::ChannelHandle::DEVICE_TO_DEVICE); +} + +LogicalResult ConvertAllReduce(OpBuilder& builder, int64_t channel_id, + TensorType result_type, + DenseIntElementsAttr replica_groups, + StringRef mode, Value input, StringRef merge_op, + StringRef final_op, Operation* op) { + builder.setInsertionPoint(op); + ChannelHandleAttr channel_handle = ConvertChannel(builder, channel_id, mode); + Location loc = op->getLoc(); + Type element_type = getElementTypeOrSelf(input.getType()); + auto all_reduce = + AllReduceOp::create(builder, loc, result_type, input, replica_groups, + channel_handle, nullptr); + + if (all_reduce.getNumResults() != 1) { + return op->emitOpError() + << "AllReduceOp must have one result: " << *all_reduce; + } + if (merge_op == "Add") { + BuildReduceBody(element_type, &all_reduce.getComputation(), + &builder); + } else if (merge_op == "Mul") { + BuildReduceBody(element_type, &all_reduce.getComputation(), + &builder); + } else if (merge_op == "Min") { + BuildReduceBody(element_type, &all_reduce.getComputation(), + &builder); + } else if (merge_op == "Max") { + BuildReduceBody(element_type, &all_reduce.getComputation(), + &builder); + } else { + return op->emitOpError() << "invalid merge_op " << merge_op + << ", want one of [Add, Mul, Min, Max]"; + } + + Operation* result = all_reduce; + // For "Div" final op, divide the merge result by group size. + if (final_op == "Div") { + int64_t replica_group_size = replica_groups.getType().getDimSize(1); + if (replica_group_size == 0) { + op->emitOpError() + << "Div final_op requires a non-empty replica_groups argument."; + } + auto divisor = + GetScalarConstOfType(element_type, loc, replica_group_size, &builder); + auto broadcast_dims = builder.getDenseI64ArrayAttr({}); + result = chlo::BroadcastDivOp::create(builder, loc, all_reduce.getResult(0), + divisor.getResult(), broadcast_dims); + } else if (final_op != "Id") { + return op->emitOpError() + << "invalid final_op " << final_op << ", want one of [Id, Div]"; + } + op->replaceAllUsesWith(result); + + op->erase(); + return success(); +} + +template +class CollectiveRewritePattern : public OpRewritePattern { + public: + // Does not take any ownership. Caller must ensure channel_id is valid during + // life-cylce of this object. + CollectiveRewritePattern(MLIRContext* context, int64_t* channel_id) + : OpRewritePattern(context), channel_id_(*channel_id) {} + + protected: + int64_t& channel_id_; // A unique channel_id shared by all rewrite patterns + // in this pass. Not thread-safe. +}; + +// Converts XlaAllReduce. Not thread-safe. +class ConvertXlaAllReduce + : public CollectiveRewritePattern { + public: + using CollectiveRewritePattern::CollectiveRewritePattern; + + LogicalResult matchAndRewrite(TF::XlaAllReduceOp all_reduce, + PatternRewriter& rewriter) const override { + DenseIntElementsAttr replica_groups; + if (failed(ConvertReplicaGroups(rewriter, all_reduce.getGroupAssignment(), + replica_groups, all_reduce))) { + return failure(); + } + + // TODO(b/226201111): Stop emitting CollectiveInfo when it is no longer + // needed. + if (failed(SetCollectiveInfo(rewriter, replica_groups, all_reduce))) { + return failure(); + } + + StringRef reduce_op = all_reduce.getReduceOp(); + + StringRef merge_op, final_op; + if (reduce_op == "Add") { + merge_op = "Add"; + final_op = "Id"; + } else if (reduce_op == "Mul") { + merge_op = "Mul"; + final_op = "Id"; + } else if (reduce_op == "Min") { + merge_op = "Min"; + final_op = "Id"; + } else if (reduce_op == "Max") { + merge_op = "Max"; + final_op = "Id"; + } else if (reduce_op == "Mean") { + merge_op = "Add"; + final_op = "Div"; + } else { + return all_reduce->emitOpError() + << "invalid reduce_op " << reduce_op + << ", want one of [Add, Mul, Min, Max, Mean]"; + } + + int64_t channel_id = channel_id_++; + return ConvertAllReduce(rewriter, channel_id, all_reduce.getType(), + replica_groups, all_reduce.getMode(), + all_reduce.getInput(), merge_op, final_op, + all_reduce); + } +}; + +// Converts CollectiveReduceV2, with or without a preceding +// CollectiveAssignGroupV2. Not thread-safe. +class ConvertCollectiveReduceV2 + : public CollectiveRewritePattern { + public: + using CollectiveRewritePattern::CollectiveRewritePattern; + + LogicalResult matchAndRewrite(TF::CollectiveReduceV2Op all_reduce, + PatternRewriter& rewriter) const override { + TF::CollectiveAssignGroupV2Op assign_group = + all_reduce.getGroupSize() + .getDefiningOp(); + + if (assign_group) { + // Found a group assignment. Use replica_groups to represent group + // assignment. + + if (assign_group != all_reduce.getGroupKey() + .getDefiningOp()) { + return all_reduce->emitOpError() + << "group_size and group_key are not from the " + "same CollectiveAssignGroupV2Op"; + } + + DenseIntElementsAttr replica_groups; + if (failed(ConvertReplicaGroups(rewriter, + assign_group.getGroupAssignment(), + replica_groups, all_reduce))) { + return failure(); + } + + // TODO(b/226201111): Stop emitting CollectiveInfo when it is no longer + // needed. + if (failed(SetCollectiveInfo(rewriter, replica_groups, all_reduce))) { + return failure(); + } + + int64_t channel_id = channel_id_++; + // FIXME(b/226139061): Mode should be set to CrossReplicaAndPartition + // in order to use XLA:GPU for more than one workers. + // The mode is set to use CrossReplica to keep the + // behavior on the primary user of this optimized path, because + // CrossReplicaAndPartition triggers a conflict with the channel_id + // allocation in the communication lowering, and the user uses both set of + // ops are used. + return ConvertAllReduce(rewriter, channel_id, all_reduce.getType(), + replica_groups, /* mode=*/"CrossReplica", + all_reduce.getInput(), all_reduce.getMergeOp(), + all_reduce.getFinalOp(), all_reduce); + } + + // No group assignment, use separate channels per group_key. + DenseIntElementsAttr group_size_attr; + if (!matchPattern(all_reduce.getGroupSize(), + m_Constant(&group_size_attr))) { + return all_reduce.emitOpError() + << "group_size must be a compile time constant"; + } + if (!group_size_attr.isSplat() || group_size_attr.size() != 1) { + return all_reduce.emitOpError() << "group_size must be a scalar"; + } + const auto group_size = group_size_attr.getSplatValue(); + + // Create a full group assignment. Empty group assignment errors when + // final_op = "Div" + llvm::SmallVector indices(group_size.getInt()); + std::iota(indices.begin(), indices.end(), 0); + + auto replica_groups = mlir::DenseIntElementsAttr::get( + mlir::RankedTensorType::get({1, group_size.getInt()}, + rewriter.getI64Type()), + indices); + + { + // TODO(b/226201111): Stop emitting CollectiveInfo when it is no longer + // needed. + DenseIntElementsAttr group_key_attr; + if (!matchPattern(all_reduce.getGroupKey(), + m_Constant(&group_key_attr))) { + return all_reduce.emitOpError() + << "group_key must be a compile time constant"; + } + if (failed(SetCollectiveInfo( + /* group_size=*/group_size, + /* group_key=*/group_key_attr.getSplatValue(), + all_reduce))) { + return failure(); + } + } + + // CrossReplicaAndPartition: + // Even though TF2XLA will setup the device assignment to include + // devices in this group as replicas before launching this module, + // "CrossReplica" mode (no channel) produces a deadlock when + // not using XLA SPMD expansion. + int64_t channel_id = channel_id_++; + return ConvertAllReduce( + rewriter, channel_id, all_reduce.getType(), replica_groups, + /* mode= */ "CrossReplicaAndPartition", all_reduce.getInput(), + all_reduce.getMergeOp(), all_reduce.getFinalOp(), all_reduce); + } +}; + +class ConvertCollectiveAssignGroupV2 + : public CollectiveRewritePattern { + public: + using CollectiveRewritePattern::CollectiveRewritePattern; + + LogicalResult matchAndRewrite(TF::CollectiveAssignGroupV2Op assign_group, + PatternRewriter& rewriter) const override { + DenseIntElementsAttr replica_groups; + if (failed(ConvertReplicaGroups(rewriter, assign_group.getGroupAssignment(), + replica_groups, assign_group))) { + return failure(); + } + IntegerAttr group_size = rewriter.getI32IntegerAttr(replica_groups.size()); + IntegerAttr group_key = rewriter.getI32IntegerAttr(0); + + auto const_group_size = + TF::ConstOp::create(rewriter, assign_group->getLoc(), + assign_group.getResult(0).getType(), group_size); + auto const_group_key = + TF::ConstOp::create(rewriter, assign_group->getLoc(), + assign_group.getResult(1).getType(), group_key); + rewriter.replaceAllUsesWith(assign_group.getResult(0), const_group_size); + rewriter.replaceAllUsesWith(assign_group.getResult(1), const_group_key); + rewriter.eraseOp(assign_group); + return success(); + } +}; + +void LegalizeTFCollective::runOnOperation() { + // FIXME(b/226139061): Figure out a way to share the channel_id with + // send/recv Ops. For now, start with a different range to avoid collision. + int64_t channel_id = 10000; + auto module = getOperation(); + MLIRContext* context = module->getContext(); + + RewritePatternSet patterns(context); + patterns.insert(context, &channel_id); + patterns.insert(context, &channel_id); + patterns.insert(context, &channel_id); + + if (failed(applyPatternsGreedily(module, std::move(patterns)))) { + signalPassFailure(); + } +} +} // namespace + +std::unique_ptr> CreateLegalizeTFCollectivePass() { + return std::make_unique(); +} + +} // namespace mhlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_communication.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_communication.cc new file mode 100644 index 0000000000000000000000000000000000000000..12842b11c58de162302cb89c6e7f94528ee7a7a7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_communication.cc @@ -0,0 +1,994 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements logic for lowering TensorFlow dialect's communication +// ops (TF/XLA) to the HLO dialect. + +#include +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/hlo/builder/sharding_builder.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "xla/primitive_util.h" +#include "xla/side_effect_util.h" +#include "xla/xla_data.pb.h" + +namespace mlir { + +using func::FuncOp; + +namespace mhlo { + +namespace { +constexpr char kShardingAttr[] = "mhlo.sharding"; +constexpr char kFrontendAttributesAttr[] = "mhlo.frontend_attributes"; +// TPU core that sends to and receives from host. +constexpr int64_t kShardingTpuCore = 0; + +#define GEN_PASS_DEF_LEGALIZETFCOMMUNICATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/transforms/tf_xla_passes.h.inc" + +// A pass that legalizes TF/XLA communication ops, propagate their respective +// tokens (for ordering), and rewrite their respective functions and control +// flow ops when necessary. +// Note, this currently does not handle nested modules/functions or region based +// ops other than certain control flow ops (`mhlo.if`, `mhlo.while`). +class LegalizeTFCommunication + : public impl::LegalizeTFCommunicationPassBase { + void runOnOperation() override; +}; + +// A generator to serve out unique channel ids. +class ChannelIdGenerator { + public: + ChannelIdGenerator() = default; + ChannelIdGenerator(const ChannelIdGenerator&) = delete; + ChannelIdGenerator& operator=(const ChannelIdGenerator&) = delete; + ChannelIdGenerator(ChannelIdGenerator&&) = delete; + ChannelIdGenerator& operator=(ChannelIdGenerator&&) = delete; + int64_t operator++(int) { return next(); } + int64_t next() { return channel_id_.fetch_add(1, std::memory_order_relaxed); } + + private: + // All usage code expects positive int64_t values so we can't use uint64_t + // and will just have to limit ourselves to half the number space. + std::atomic channel_id_ = 1; +}; + +int64_t GetNextChannelId() { + static ChannelIdGenerator* channel_id = new ChannelIdGenerator(); + return channel_id->next(); +} + +// Checks if an op is a TF/XLA communication op. +bool IsCommunicationOp(Operation* op) { + return isa(op); +} + +// Checks if an op is a supported HLO control flow op. +bool IsControlFlowOp(Operation* op) { return isa(op); } + +// Collects control flow op ancestors of a given op, up until FuncOp. If any +// ancestor is not a control flow op or a FuncOp, or of a single block region, +// an error will be returned. +LogicalResult GetControlFlowAncestors( + Operation* op, llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks) { + Block* block = op->getBlock(); + Operation* parent = block->getParentOp(); + while (block && parent && !isa(parent)) { + if (!IsControlFlowOp(parent)) + return op->emitOpError() + << "expects ancestor(s) to be of ['" << IfOp::getOperationName() + << "', '" << func::FuncOp::getOperationName() << "']"; + + if (!llvm::hasSingleElement(block->getParent()->getBlocks())) + return op->emitOpError() << "expects single block region ancestor(s)"; + + control_flow_ops.insert(parent); + control_flow_blocks.insert(block); + + parent = block->getParentOp(); + block = parent->getBlock(); + } + return success(); +} + +// Finds communication ops in a function. `control_flow_ops` and +// `control_flow_blocks` will be populated with control flow op ancestors for +// every communication op. +LogicalResult FindCommunicationOps( + func::FuncOp func, llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks, + bool& has_communication_ops) { + auto result = func.walk([&](Operation* op) { + if (!IsCommunicationOp(op)) return WalkResult::advance(); + has_communication_ops = true; + if (failed( + GetControlFlowAncestors(op, control_flow_ops, control_flow_blocks))) + return WalkResult::interrupt(); + return WalkResult::advance(); + }); + return failure(result.wasInterrupted()); +} + +// Helper struct holding a function to be rewritten, it's control flow ops that +// lead to a communication op or function call with a communication op +// (transitively), and an optional clone of itself. If `clone` is set, function +// calls to `original` will be replaced with `clone`. +struct FuncToRewrite { + func::FuncOp original; + llvm::SmallPtrSet control_flow_ops; + llvm::SmallPtrSet control_flow_blocks; + func::FuncOp clone; +}; + +// Finds all functions that need to be rewritten with communication ops and +// and associated tokens. +LogicalResult GetFunctionsToRewrite( + ModuleOp module, + llvm::SmallDenseMap& funcs_to_rewrite) { + // Find functions containing communication ops. + SmallVector funcs_to_visit; + for (func::FuncOp func : module.getOps()) { + FuncToRewrite func_to_rewrite{/*original=*/func, /*control_flow_ops=*/{}, + /*control_flow_blocks=*/{}, + /*clone=*/nullptr}; + bool has_communication_ops = false; + if (failed(FindCommunicationOps(func, func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks, + has_communication_ops))) + return failure(); + + if (!has_communication_ops) continue; + funcs_to_rewrite.insert({func.getName(), func_to_rewrite}); + funcs_to_visit.push_back(func); + } + + // Find functions that call functions with communication ops, transitively. + while (!funcs_to_visit.empty()) { + SmallVector new_funcs_to_visit; + for (func::FuncOp& func : funcs_to_visit) { + auto uses = func.getSymbolUses(module); + if (!uses) continue; + for (auto& use : *uses) { + // Only `mlir::func::CallOp` is supported as this requires knowing how + // to rewrite arguments and results to a function. + if (!isa(use.getUser())) continue; + auto caller_parent_func = + use.getUser()->getParentOfType(); + if (!caller_parent_func) continue; + + FuncToRewrite func_to_rewrite{/*original=*/caller_parent_func, + /*control_flow_ops=*/{}, + /*control_flow_blocks=*/{}, + /*clone=*/nullptr}; + if (failed(GetControlFlowAncestors( + use.getUser(), func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks))) + return failure(); + + auto it = funcs_to_rewrite.insert( + {caller_parent_func.getName(), func_to_rewrite}); + if (it.second) { + new_funcs_to_visit.push_back(caller_parent_func); + } else { + it.first->getSecond().control_flow_ops.insert( + func_to_rewrite.control_flow_ops.begin(), + func_to_rewrite.control_flow_ops.end()); + it.first->getSecond().control_flow_blocks.insert( + func_to_rewrite.control_flow_blocks.begin(), + func_to_rewrite.control_flow_blocks.end()); + } + } + } + + funcs_to_visit.swap(new_funcs_to_visit); + } + + // Clone public functions that need to be rewritten. Function calls to this + // function will be replaced with the cloned function. + SymbolTable symbol_table(module); + for (auto& func : funcs_to_rewrite) { + if (func.getSecond().original.isPublic() && + !func.getSecond().original.symbolKnownUseEmpty(module)) { + auto clone = func.getSecond().original.clone(); + clone.setPrivate(); + symbol_table.insert(clone); + func.getSecond().clone = clone; + } + } + + return success(); +} + +// Assigns either MAXIMAL or MANUAL sharding. The MAXIMAL sharding sends/recvs +// one message from core `kShardingTpuCore` with the full tensor. MANUAL +// sharding sends/recvs one message for each core with the core's shard. +void SetOpSharding(Operation* op, bool manual_sharding) { + xla::OpSharding sharding = + manual_sharding ? ::xla::sharding_builder::Manual() + : ::xla::sharding_builder::SingleDevice(kShardingTpuCore); + op->setAttr(kShardingAttr, + StringAttr::get(op->getContext(), sharding.SerializeAsString())); +} + +// Assigns frontend attributes holding information about data type and +// TensorFlow rendezvous channel name. The TensorFlow rendezvous channel name is +// handled differently as individual names are used per data send and receive. +void SetFrontendAttributes(Operation* op, int32_t index, StringRef key, + Type type, bool device_to_host, + StringRef host_handler_name) { + MLIRContext* context = op->getContext(); + + std::string formatted_key = + device_to_host ? llvm::formatv("{0}_dtoh_{1}", key, index).str() + : llvm::formatv("{0}_htod_{1}", key, index).str(); + + auto rendezvous_name = StringAttr::get(context, formatted_key); + auto rendezvous_name_attr = NamedAttribute( + StringAttr::get(context, xla::kXlaHostTransferRendezvousNameAttr), + rendezvous_name); + + auto host_handler_name_value = + StringAttr::get(context, host_handler_name.str()); + auto host_handler_name_attr = NamedAttribute( + StringAttr::get(context, xla::kXlaHostTransferHandlerNameAttr), + host_handler_name_value); + + auto frontend_attributes = DictionaryAttr::get( + context, + ArrayRef{rendezvous_name_attr, host_handler_name_attr}); + op->setAttr(kFrontendAttributesAttr, frontend_attributes); +} + +// Creates a `mhlo.send` op for sending value `operand`. +Value CreateSendOp(OpBuilder& builder, Location loc, Value operand, + StringRef key, size_t index, Value token, + StringRef host_handler_name, bool manual_sharding) { + // type 2 == DEVICE_TO_HOST + auto channel_handle = ChannelHandleAttr::get(builder.getContext(), + /*handle=*/GetNextChannelId(), + /*type=*/2); + auto empty_source_target_pairs = builder.getI64TensorAttr({}); + auto send = SendOp::create(builder, loc, token.getType(), operand, token, + channel_handle, + /*is_host_transfer=*/builder.getBoolAttr(true), + /*source_target_pairs=*/empty_source_target_pairs); + SetFrontendAttributes(send, index, key, operand.getType(), + /*device_to_host=*/true, host_handler_name); + + SetOpSharding(send, manual_sharding); + + return send.getResult(); +} + +// Creates a `mhlo.recv` op for receiving a value. +Value CreateRecvOp(OpBuilder& builder, Location loc, Value result, + StringRef key, size_t index, Value token, + StringRef host_handler_name, bool manual_sharding) { + // type 3 == HOST_TO_DEVICE + auto channel_handle = ChannelHandleAttr::get(builder.getContext(), + /*handle=*/GetNextChannelId(), + /*type=*/3); + auto result_type = result.getType(); + SmallVector recv_result_type = {result_type, token.getType()}; + auto recv = + RecvOp::create(builder, loc, recv_result_type, token, channel_handle, + /*is_host_transfer=*/builder.getBoolAttr(true), + /*source_target_pairs=*/builder.getI64TensorAttr({})); + + SetFrontendAttributes(recv, index, key, result_type, + /*device_to_host=*/false, host_handler_name); + + SetOpSharding(recv, manual_sharding); + + result.replaceAllUsesWith(recv.getResult(0)); + + return recv.getResult(1); +} + +// Creates a new token if necessary, acting as a sink to previous tokens. If +// there is only one token in `tokens`, the only token is returned. If `tokens` +// is empty, `original_token` is returned instead. +Value CreateSinkToken(OpBuilder& builder, Location loc, ArrayRef tokens, + Value original_token) { + if (tokens.empty()) { + return original_token; + } else if (llvm::hasSingleElement(tokens)) { + return tokens[0]; + } else { + return AfterAllOp::create(builder, loc, original_token.getType(), tokens) + .getResult(); + } +} + +// Replaces `tf._XlaHostComputeMlir` with individual `mhlo.send` and `mhlo.recv` +// ops per operand and result. Unique Channel IDs are assigned per transfer. +// Sink tokens are created across all `mhlo.send` ops first and then by +// all `mhlo.recv` ops. +Value RewriteHostComputeOp(OpBuilder& builder, + TF::_XlaHostComputeMlirOp host_compute, + Value token) { + builder.setInsertionPoint(host_compute); + Location loc = host_compute.getLoc(); + bool manual_sharding = host_compute.getManualSharding(); + + SmallVector send_tokens; + for (auto operand : llvm::enumerate(host_compute.getInputs())) { + auto send_token = CreateSendOp( + builder, loc, operand.value(), host_compute.getSendKey(), + operand.index(), token, xla::kXlaHostTransferTfRendezvousHandlerName, + manual_sharding); + send_tokens.push_back(send_token); + } + token = CreateSinkToken(builder, loc, send_tokens, token); + + SmallVector recv_tokens; + for (auto result : llvm::enumerate(host_compute.getOutputs())) { + auto recv_token = CreateRecvOp( + builder, loc, result.value(), host_compute.getRecvKey(), result.index(), + token, xla::kXlaHostTransferTfRendezvousHandlerName, manual_sharding); + recv_tokens.push_back(recv_token); + } + token = CreateSinkToken(builder, loc, recv_tokens, token); + + host_compute.erase(); + return token; +} + +// Replaces `tf.XlaSendToHost` with a `mhlo.send`. +Value RewriteSendToHostOp(OpBuilder& builder, TF::XlaSendToHostOp send_to_host, + Value token) { + builder.setInsertionPoint(send_to_host); + token = CreateSendOp(builder, send_to_host.getLoc(), send_to_host.getInput(), + send_to_host.getKey(), + /*index=*/0, token, + xla::kXlaHostTransferTfRendezvousHandlerName, + /*manual_sharding=*/false); + + send_to_host.erase(); + return token; +} + +// Replaces `tf.XlaRecvFromHost` with a `mhlo.recv`. +Value RewriteRecvFromHostOp(OpBuilder& builder, + TF::XlaRecvFromHostOp recv_from_host, Value token) { + builder.setInsertionPoint(recv_from_host); + token = CreateRecvOp(builder, recv_from_host.getLoc(), + recv_from_host.getOutput(), recv_from_host.getKey(), + /*index=*/0, token, + xla::kXlaHostTransferTfRendezvousHandlerName, + /*manual_sharding=*/false); + + recv_from_host.erase(); + return token; +} + +// Replaces a `mlir::func::CallOp` with one that has an extra `!mhlo.token` +// operand and `!mhlo.token` result. If `new_symbol` is set, the new call will +// be updated to call the `new_symbol` instead. +Value RewriteCallOp(OpBuilder& builder, func::CallOp call, + const std::optional& new_symbol, Value token) { + builder.setInsertionPoint(call); + auto new_operands = llvm::to_vector(call.getArgOperands()); + new_operands.push_back(token); + auto new_result_types = llvm::to_vector(call.getResultTypes()); + new_result_types.push_back(token.getType()); + auto new_call = func::CallOp::create( + builder, call.getLoc(), new_result_types, + new_symbol ? *new_symbol : call.getCallee(), new_operands); + + for (auto results : llvm::zip(call.getResults(), new_call.getResults())) + std::get<0>(results).replaceAllUsesWith(std::get<1>(results)); + call.erase(); + return new_call.getResults().back(); +} + +// Helper struct holding state of which op to visit to next. If `op` is in a +// control flow op region, `region_idx` will be set with the respective region +// index. `token` will be current token from the last communication op/control +// flow op transitive communication ops. +struct OpVisitorState { + std::optional region_idx; + Value token; + Operation* op; +}; + +// Creates a tuple from a sequence of values. +Value CreateTuple(OpBuilder& builder, Location loc, ArrayRef operands) { + return TupleOp::create(builder, loc, operands).getResult(); +} + +// Extends `values` with the value `token` attached. If `flatten_tuple` is +// false, `values` will have a single element, say `value`. If `value` is not a +// tuple, a new tuple is formed with `token`. If `values` is a tuple, it is +// extended instead. New tuple values created are cached. +SmallVector GetValueWithToken( + OpBuilder& builder, ArrayRef values, Value token, + llvm::SmallDenseMap& rewritten_values, bool flatten_tuple) { + if (flatten_tuple) { + auto operands = llvm::to_vector(values); + operands.push_back(token); + return operands; + } + + auto value = values[0]; + // If value with token already exists, reuse it. + auto it = rewritten_values.find(value); + if (it != rewritten_values.end()) return {it->getSecond()}; + + auto create_tuple = [&](ArrayRef operands) { + auto new_result = CreateTuple(builder, value.getLoc(), operands); + rewritten_values.insert({value, new_result}); + return new_result; + }; + + auto tuple_type = mlir::dyn_cast(value.getType()); + // `value` is not a tuple, create a new tuple. + if (!tuple_type) return {create_tuple({value, token})}; + + // Extend tuple if `value` is a tuple. + // If `value` is an op result and the owner is a `mhlo.tuple`, simply unpack + // the tuple. + if (auto tuple_op = value.getDefiningOp()) { + auto tuple_operands = llvm::to_vector(tuple_op.getOperands()); + tuple_operands.push_back(token); + return {create_tuple(tuple_operands)}; + } + + // `value` is not created via a `mhlo.tuple` directly, unpack individual + // elements directly with `mhlo.get_tuple_element`. + SmallVector tuple_operands; + for (auto idx : llvm::seq(0, tuple_type.getTypes().size())) + tuple_operands.push_back( + GetTupleElementOp::create(builder, value.getLoc(), value, idx) + .getResult()); + + tuple_operands.push_back(token); + return {create_tuple(tuple_operands)}; +} + +// Extends the 'types' to include a `mhlo.token` type. If `flatten_tuple` is +// false, `types` will have a single element, say `type`. If `type` is not a +// tuple type, a new tuple type with `type` and `mhlo.token` type is created +// instead. +SmallVector GetTypeWithToken(OpBuilder& builder, ArrayRef types, + bool flatten_tuple) { + SmallVector new_result_types; + auto token_type = TokenType::get(builder.getContext()); + + if (flatten_tuple) { + auto result_types = llvm::to_vector(types); + result_types.push_back(token_type); + return result_types; + } + + auto type = types[0]; + if (auto tuple_type = mlir::dyn_cast(type)) { + auto result_types = llvm::to_vector(tuple_type.getTypes()); + result_types.push_back(token_type); + return {builder.getTupleType(result_types)}; + } + + return {builder.getTupleType({type, token_type})}; +} + +// Creates a slice of a tuple `value` with `mhlo.get_tuple_element` from index 0 +// to `end`, exclusive. +Value CreateSubTuple(OpBuilder& builder, Value value, size_t end) { + SmallVector tuple_operands; + for (auto idx : llvm::seq(0, end)) + tuple_operands.push_back( + GetTupleElementOp::create(builder, value.getLoc(), value, idx) + .getResult()); + + return CreateTuple(builder, value.getLoc(), tuple_operands); +} + +// Replaces uses of `values` with `replacements`. If `flatten_tuple` is false, +// `values` will have a single element, say `value`. If `value` is not a tuple +// type, an explicit `mhlo.get_tuple_element` is created to unpack the tuple and +// return the first element. Otherwise, `mhlo.get_tuple_element` users are +// simply updated with `replacement`, and all other users are updated with a +// slice of `replacement`. +void ReplaceWithTupleResult(OpBuilder& builder, ValueRange values, + ValueRange replacements, bool flatten_tuple) { + if (flatten_tuple) { + for (size_t result_index = 0; result_index < values.size(); result_index++) + values[result_index].replaceAllUsesWith(replacements[result_index]); + return; + } + + auto value = values[0]; + auto replacement = replacements[0]; + auto tuple_type = mlir::dyn_cast(value.getType()); + if (!tuple_type) { + if (!value.use_empty()) { + auto new_element = GetTupleElementOp::create( + builder, replacement.getLoc(), replacement, 0); + value.replaceAllUsesWith(new_element.getResult()); + } + return; + } + + Value sub_tuple; + for (auto& use : llvm::make_early_inc_range(value.getUses())) { + if (isa(use.getOwner())) { + use.set(replacement); + continue; + } + + if (!sub_tuple) + sub_tuple = CreateSubTuple(builder, replacement, tuple_type.size()); + + use.set(sub_tuple); + } +} + +// Replaces control flow op block arguments with new block arguments +// of types `types`. The last element of the new block argument (token) is +// returned. +Value UpdateControlFlowBlockArgWithToken(OpBuilder& builder, Block& block, + ArrayRef types) { + builder.setInsertionPointToStart(&block); + + auto old_args_size = block.getNumArguments(); + + block.addArguments( + types, SmallVector(types.size(), block.getParent()->getLoc())); + + ValueRange old_args = block.getArguments().take_front(old_args_size); + ValueRange new_args = block.getArguments().drop_front(old_args_size); + assert(!new_args.empty()); + + ReplaceWithTupleResult(builder, old_args, new_args, /*flatten_tuple=*/true); + auto new_arg = new_args[new_args.size() - 1]; + + block.eraseArguments(0, old_args_size); + + return new_arg; +} + +// Updates control flow op terminator with an extra element `token`. +void RewriteControlFlowTerminator(OpBuilder& builder, Operation* terminator, + Value token, bool flatten_tuple) { + assert(flatten_tuple || terminator->getNumOperands() == 1); + assert(flatten_tuple || terminator->getBlock()->getNumArguments() == 1); + // `mhlo.while` cond terminator does not need to be rewritten as it always + // returns a tensor predicate value. + if (auto while_parent = dyn_cast_or_null(terminator->getParentOp())) + if (terminator->getParentRegion() == &while_parent.getCond()) return; + + builder.setInsertionPoint(terminator); + llvm::SmallDenseMap rewritten_operands; + auto new_results = + GetValueWithToken(builder, llvm::to_vector(terminator->getOperands()), + token, rewritten_operands, flatten_tuple); + terminator->setOperands(new_results); +} + +// Rewrites a `mhlo.if` op to receive and forward a `mhlo.token`. As If op does +// not have any operands other than the predicate, hence we implicitly capture +// the parent token. Also we use the same implicit token for use in the If op's +// regions. +void RewriteRegionIfOp(OpBuilder& builder, IfOp region_if, + SmallVectorImpl& ops_to_visit, + Value token) { + llvm::SmallDenseMap rewritten_operands; + + auto new_result_types = + GetTypeWithToken(builder, llvm::to_vector(region_if.getResultTypes()), + /*flatten_tuple=*/true); + + // Create new `mhlo.if` op with extra token operands and result. + auto new_if = IfOp::create(builder, region_if.getLoc(), new_result_types, + region_if.getPred()); + + // Move all regions from the old `mhlo.if` op to its replacement. + new_if.getTrueBranch().takeBody(region_if.getTrueBranch()); + new_if.getFalseBranch().takeBody(region_if.getFalseBranch()); + + // Forward result from old `mhlo.if` with replacement. + SmallVector old_if_results = region_if.getResults(); + SmallVector new_if_results = new_if.getResults(); + + ReplaceWithTupleResult(builder, old_if_results, new_if_results, + /*flatten_tuple=*/true); + + // auto new_token = new_if_results[new_if_results.size() - 1]; + + region_if.erase(); + + // Next op to visit. The replacement is visited but at its first region. + // The new region use the same implicit token used by the If op. + ops_to_visit.push_back({/*region_idx=*/0, token, new_if}); +} + +// Rewrites a `mhlo.if`/`mhlo.while` region to receive and forward a +// `mhlo.token`. The block argument is updated to have an extra `mhlo.token` +// element. If the region block is to be rewritten, the next op to visit is set +// to the first op in the block. Otherwise the terminator is updated to forward +// `token`. +void RewriteControlFlowOpRegion( + OpBuilder& builder, Operation* region_op, unsigned region_idx, + ArrayRef block_arg_types, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, Value token) { + ops_to_visit.push_back({region_idx + 1, token, region_op}); + + Region& region = region_op->getRegion(region_idx); + assert(llvm::hasSingleElement(region)); + + auto block_token = UpdateControlFlowBlockArgWithToken(builder, region.front(), + block_arg_types); + + if (control_flow_blocks.contains(®ion.front())) { + ops_to_visit.push_back( + {/*region_idx=*/std::nullopt, block_token, ®ion.front().front()}); + return; + } + + RewriteControlFlowTerminator(builder, region.front().getTerminator(), + block_token, /*flatten_tuple=*/true); +} + +// For mlir::IfOp or mlir::CaseOp, replace the use of their region's block +// argument (of type token) with 'implicit_operand'. +void ReplaceBlockArgumentsWithImplicitOperands(mlir::Operation* op, + unsigned region_idx, + Value implicit_operand) { + assert((mlir::dyn_cast(*op) || + mlir::dyn_cast(*op)) && + "Unexpected mlir op in " + "HloFunctionImporter::ReplaceBlockArgumentsWithImplicitOperands!"); + + auto& region = op->getRegion(region_idx); + region.getArgument(0).replaceAllUsesWith(implicit_operand); + region.front().eraseArguments(0, region.getNumArguments()); +} + +// Rewrites an `mhlo.if` op or its region. If `region_idx` is not set, the op +// operands and results are rewritten. If `region_idx` is set, region +// `region_idx` is rewritten to take in and return an additional token. Returns +// true if the op or its region was rewritten. +bool ProcessRegionIfOp(OpBuilder& builder, IfOp region_if, + std::optional region_idx, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, + Value token) { + builder.setInsertionPoint(region_if); + + if (!region_idx) { + RewriteRegionIfOp(builder, region_if, ops_to_visit, token); + return true; + } + + if (*region_idx < region_if.getNumRegions()) { + // For the region-blocks of If op, we create a dummy token argument. Later + // we replace that block-argument's uses with the same (implicitly captured) + // token 'token', used for If op, and erase the argument. + // Note that 'RewriteControlFlowOpRegion' sets the token, used for the first + // operation of region_idx'th region, to the dummy block-argument. As we + // erase that argument, we also need to make sure that the token used for + // the next operation is set to 'token'. + RewriteControlFlowOpRegion(builder, region_if, *region_idx, + {token.getType()}, ops_to_visit, + control_flow_blocks, token); + + ReplaceBlockArgumentsWithImplicitOperands(region_if.getOperation(), + *region_idx, token); + + auto next_visitor_state = ops_to_visit.back(); + next_visitor_state.token = token; + ops_to_visit.pop_back(); + ops_to_visit.push_back(next_visitor_state); + return true; + } + + return false; +} + +// Rewrites a `mhlo.while` op to receive and forward a `mhlo.token`. Operands to +// the op for all of its regions are extended to have an extra operand `token`. +void RewriteRegionWhileOp(OpBuilder& builder, WhileOp region_while, + SmallVectorImpl& ops_to_visit, + Value token) { + llvm::SmallDenseMap rewritten_operands; + + // Rewrite region operand to have an extra operand `token`. + auto new_val_operands = + GetValueWithToken(builder, llvm::to_vector(region_while.getOperands()), + token, rewritten_operands, + /*flatten_tuple=*/true); + + auto new_result_types = + GetTypeWithToken(builder, llvm::to_vector(region_while.getResultTypes()), + /*flatten_tuple*/ true); + + // Create new `mhlo.while` op with extra token operand and result. + auto new_while = WhileOp::create(builder, region_while.getLoc(), + new_result_types, new_val_operands); + + // Move all regions from the old `mhlo.while` op to its replacement. + new_while.getCond().takeBody(region_while.getCond()); + new_while.getBody().takeBody(region_while.getBody()); + + // Forward result from old `mhlo.while` with replacement. + SmallVector old_while_results = region_while.getResults(); + SmallVector new_while_results = new_while.getResults(); + + ReplaceWithTupleResult(builder, old_while_results, new_while_results, + /*flatten_tuple*/ true); + + auto new_token = new_while_results[new_while_results.size() - 1]; + + region_while.erase(); + + // Next op to visit. The replacement is visited but at its first region. The + // token result of the new region if is propagated. + ops_to_visit.push_back({/*region_idx=*/0, new_token, new_while}); +} + +// Rewrites an `mhlo.while` op or its region. If `region_idx` is not set, the op +// operands and results are rewritten. If `region_idx` is set, region +// `region_idx` is rewritten to take in and return an additional token. Returns +// true if the op or its region was rewritten. +bool ProcessRegionWhileOp( + OpBuilder& builder, WhileOp region_while, + std::optional region_idx, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, Value token) { + builder.setInsertionPoint(region_while); + + if (!region_idx) { + RewriteRegionWhileOp(builder, region_while, ops_to_visit, token); + return true; + } + + if (*region_idx < region_while.getNumRegions()) { + SmallVector operand_types; + for (auto operand : region_while.getOperand()) + operand_types.push_back(operand.getType()); + RewriteControlFlowOpRegion(builder, region_while, *region_idx, + operand_types, ops_to_visit, control_flow_blocks, + token); + return true; + } + + return false; +} + +// Updates function type based on current function body block arguments and +// terminator operand types. +void UpdateFunctionType(OpBuilder& builder, func::FuncOp func, + Block& func_body) { + auto new_argument_types = llvm::to_vector(func_body.getArgumentTypes()); + auto new_result_types = + llvm::to_vector(func_body.getTerminator()->getOperandTypes()); + func.setType(FunctionType::get(builder.getContext(), new_argument_types, + new_result_types)); +} + +// Replaces a function terminator `return` with another `return` that has an +// extra `mhlo.token` operand. +void RewriteFunctionTerminator(OpBuilder& builder, + mlir::func::ReturnOp terminator, Value token) { + auto new_results = llvm::to_vector(terminator.getOperands()); + new_results.push_back(token); + builder.setInsertionPoint(terminator); + mlir::func::ReturnOp::create(builder, terminator.getLoc(), new_results); + terminator.erase(); +} + +// Rewrites a function body and communication ops inside. Region control flow +// are updated when necessary, to propagate tokens. The function may either be +// rewritten to create a token or take in and return a token, depending on its +// visibility and if there are any callers. +LogicalResult RewriteFunction( + OpBuilder& builder, ModuleOp module, FuncOp func, + const llvm::SmallDenseMap& funcs, + const llvm::SmallPtrSetImpl& control_flow_ops, + const llvm::SmallPtrSetImpl& control_flow_blocks, bool is_clone) { + MLIRContext* context = module.getContext(); + if (!llvm::hasSingleElement(func.getBody())) + return func.emitError() + << "'" << FuncOp::getOperationName() + << "' ops with more than one block are not supported"; + + bool rewrite_block = + is_clone || (!func.isPublic() && !func.symbolKnownUseEmpty(module)); + Block& func_body = func.front(); + + builder.setInsertionPointToStart(&func_body); + auto token_type = TokenType::get(context); + // If a function is public, it's signature should not be modified, and instead + // a token will be created. Otherwise a token block argument is inserted. + Value init_token = + rewrite_block ? func_body.addArgument(token_type, func.getLoc()) + : CreateTokenOp::create(builder, func.getLoc(), token_type) + .getResult(); + + // Stack to keep track of region based control flow op nesting and current + // op to visit. + SmallVector ops_to_visit{ + {/*region_idx=*/std::nullopt, init_token, &func_body.front()}}; + + while (!ops_to_visit.empty()) { + OpVisitorState op_to_visit = ops_to_visit.pop_back_val(); + Operation* curr_op = op_to_visit.op; + + Value token = op_to_visit.token; + // Ops may be removed, so the next op is kept track of beforehand. + Operation* next_op = curr_op->getNextNode(); + + if (auto host_compute = dyn_cast(curr_op)) { + token = RewriteHostComputeOp(builder, host_compute, token); + } else if (auto send_to_host = dyn_cast(curr_op)) { + token = RewriteSendToHostOp(builder, send_to_host, token); + } else if (auto recv_from_host = dyn_cast(curr_op)) { + token = RewriteRecvFromHostOp(builder, recv_from_host, token); + } else if (auto call = dyn_cast(curr_op)) { + // Only `mlir::func::CallOp` is supported as this requires knowing how to + // rewrite arguments and results to a function. + auto it = funcs.find(call.getCallee()); + if (it != funcs.end()) { + func::FuncOp clone = it->getSecond().clone; + std::optional symbol_name = + clone ? std::optional(clone.getName()) : std::nullopt; + // If the function being called is to be cloned, update the call to also + // point to the cloned function. + token = RewriteCallOp(builder, call, symbol_name, token); + } + } else if (auto region_if = dyn_cast(curr_op)) { + if (op_to_visit.region_idx || control_flow_ops.contains(region_if)) { + auto exist_unprocessed_region = + ProcessRegionIfOp(builder, region_if, op_to_visit.region_idx, + ops_to_visit, control_flow_blocks, token); + + // Once all the IfOp regions are processed (i.e. + // 'exist_unprocessed_region' == false), select returned token-value + // from IfOp as the token to be used for the following op. + if (!exist_unprocessed_region) { + token = curr_op->getResult(curr_op->getNumResults() - 1); + } else { + continue; + } + } + } else if (auto region_while = dyn_cast(curr_op)) { + if (op_to_visit.region_idx || control_flow_ops.contains(region_while)) + if (ProcessRegionWhileOp(builder, region_while, op_to_visit.region_idx, + ops_to_visit, control_flow_blocks, token)) + continue; + } else if (auto region_terminator = dyn_cast(curr_op)) { + bool flatten_tuple = isa( + region_terminator->getParentOp()); + RewriteControlFlowTerminator(builder, region_terminator, token, + flatten_tuple); + // There is no next op after the control flow op terminator, simply let + // stack have one less element. + continue; + } else if (auto func_terminator = dyn_cast(curr_op)) { + if (rewrite_block) + RewriteFunctionTerminator(builder, func_terminator, token); + + // There is no next op after the function terminator, simply let stack + // have one less element/be empty. + continue; + } + + // Visit next op. + ops_to_visit.push_back({/*region_idx=*/std::nullopt, token, next_op}); + } + + if (rewrite_block) UpdateFunctionType(builder, func, func_body); + + return success(); +} + +// Checks if a function call is pointing to a function with communication ops. +bool IsFunctionCallWithCommunication( + Operation* op, + const llvm::SmallDenseMap& funcs_to_rewrite) { + if (auto call = dyn_cast(op)) + return funcs_to_rewrite.count(call.getCallee()); + + return false; +} + +// Collects all control flow op ancestors of communication ops or function calls +// with communication ops (transitively). +void GetCommunicationControlFlowOps( + func::FuncOp func, + const llvm::SmallDenseMap& funcs_to_rewrite, + llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks) { + func.walk([&](Operation* op) { + if (IsCommunicationOp(op) || + IsFunctionCallWithCommunication(op, funcs_to_rewrite)) + if (failed(GetControlFlowAncestors(op, control_flow_ops, + control_flow_blocks))) + llvm_unreachable( + "checking original function for control flow ancestors should have " + "errored first"); + }); +} + +void LegalizeTFCommunication::runOnOperation() { + auto module = getOperation(); + llvm::SmallDenseMap funcs_to_rewrite; + if (failed(GetFunctionsToRewrite(module, funcs_to_rewrite))) + return signalPassFailure(); + + OpBuilder builder(&getContext()); + for (const auto& func_and_name : funcs_to_rewrite) { + const auto& func_to_rewrite = func_and_name.getSecond(); + func::FuncOp func = func_to_rewrite.original; + if (failed(RewriteFunction(builder, module, func, funcs_to_rewrite, + func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks, + /*is_clone=*/false))) + return signalPassFailure(); + + func::FuncOp clone = func_and_name.getSecond().clone; + if (!clone) continue; + llvm::SmallPtrSet clone_control_flow_ops; + llvm::SmallPtrSet clone_control_flow_blocks; + GetCommunicationControlFlowOps(clone, funcs_to_rewrite, + clone_control_flow_ops, + clone_control_flow_blocks); + if (failed(RewriteFunction(builder, module, clone, funcs_to_rewrite, + clone_control_flow_ops, + clone_control_flow_blocks, + /*is_clone=*/true))) + llvm_unreachable( + "rewriting of original function should have errored first"); + } +} + +} // namespace + +std::unique_ptr> CreateLegalizeTFCommunicationPass() { + return std::make_unique(); +} + +} // namespace mhlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_patterns.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_patterns.td new file mode 100644 index 0000000000000000000000000000000000000000..85818da4527c34efbcd39cf1a3d743d92237a495 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_patterns.td @@ -0,0 +1,827 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is the legalization pattern definition file for TF to XLA. + +include "mlir/IR/OpBase.td" +include "mlir/Dialect/Shape/IR/ShapeOps.td" +include "mlir/Dialect/Func/IR/FuncOps.td" +include "mlir/Dialect/Tensor/IR/TensorOps.td" +include "stablehlo/dialect/ChloOps.td" +include "stablehlo/dialect/StablehloOps.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td" +include "mhlo/IR/hlo_ops.td" // for hlo_utils.td + +def SignedIntTensor : TensorOf<[I1, I8, I16, I32, I64]>; +def UnsignedIntTensor : TensorOf<[UI8, UI16, UI32, UI64]>; + +// IEEE compliant floating point tensors. +def IEEEFloatTensor : TensorOf<[F16, F32, F64]>; + +//===----------------------------------------------------------------------===// +// BatchNorm op patterns. +//===----------------------------------------------------------------------===// + +def FalseBoolAttr : AttrConstraint($_self).getValue()">>; +def TrueBoolAttr : AttrConstraint($_self).getValue()">>; + +def CastValueToI64: NativeCodeCall< + "CastValueToI64($0.getLoc(), $1, &$_builder)">; + +def CastValueToElementType: NativeCodeCall< + "stablehlo::ConvertOp::create($_builder, $0.getLoc(), $1, " + "getElementTypeOrSelf($2.getType()))">; + +// Here, $0 is an ElementsAttr with exactly one element of type integer. $1 is +// the corresponding value of ranked tensor type whose axis is referred in $0. +def GetHLOAxisFromTFAxis : NativeCodeCall< + "GetHLOAxisFromTFAxis(" + "$0, llvm::cast($1.getType()).getRank(), &$_builder)">; + +// Same as the above but with $1 of type operand_range from variadic TensorFlow +// input. +def GetHLOAxisFromTFAxisVariadic : NativeCodeCall< + "GetHLOAxisFromTFAxis(" + "$0, llvm::cast((*$1.begin()).getType()).getRank(), " + "&$_builder)">; + +def CastElementsToI64Elements : NativeCodeCall<[{ + llvm::cast(hlo::convertElementsAttr( + llvm::cast($0), $_builder.getIntegerType(64))) + }]>; + +def CastElementsToI64Array : NativeCodeCall<[{ + ToDenseI64ArrayAttr( + llvm::cast(hlo::convertElementsAttr( + llvm::cast($0), $_builder.getIntegerType(64))), &$_builder) + }]>; + +def EmptyDotAlgorithmAttr : NativeCodeCall<"mlir::stablehlo::DotAlgorithmAttr{}">; + +def ConstDefaultResultAccuracyAttr : + ConstantAttr; + +//===----------------------------------------------------------------------===// +// ApproximateEqual op pattern. +//===----------------------------------------------------------------------===// + +class StableHLO_ComparisonDirectionValue : + ConstantAttr; + +class CHLO_ComparisonDirectionValue : + ConstantAttr; + +// TODO(b/228291745): Assert that $x and $y have the same shape. +def : Pat<(TF_ApproximateEqualOp:$result $x, $y, $tolerance), + (CHLO_BroadcastCompareOp + (StableHLO_AbsOp:$abs (StableHLO_SubtractOp $x, $y)), + (CastValueToElementType $result, (StableHLO_ConstantOp $tolerance), $abs), + (NullDenseI64ArrayAttr), + CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE))>; + +//===----------------------------------------------------------------------===// +// Assert op pattern. +//===----------------------------------------------------------------------===// + +// HLO and XLA doesn't support Assertions. +def LowerAssert : Pattern<(TF_AssertOp $condition, $data, $summarize), []>; + +//===----------------------------------------------------------------------===// +// Binary op patterns. +//===----------------------------------------------------------------------===// + +// Check that two values can be broadcasted together +def AreBroadcastCompatible : Constraint, + "types must be broadcastable">; + +class DirectBinaryPat + : Pat<(FromOp AnyTensor:$l, AnyTensor:$r), + (ToOp $l, $r, (BinBroadcastDimensions $l, $r))>; + +foreach fromToBinPair = [[TF_AddV2Op, CHLO_BroadcastAddOp], + [TF_Atan2Op, CHLO_BroadcastAtan2Op], + [TF_ComplexOp, CHLO_BroadcastComplexOp], + [TF_DivOp, CHLO_BroadcastDivOp], + [TF_LeftShiftOp, CHLO_BroadcastShiftLeftOp], + [TF_MaximumOp, CHLO_BroadcastMaxOp], + [TF_MinimumOp, CHLO_BroadcastMinOp], + [TF_ModOp, CHLO_BroadcastRemOp], + [TF_MulOp, CHLO_BroadcastMulOp], + [TF_NextAfterOp, CHLO_BroadcastNextAfterOp], + [TF_PolygammaOp, CHLO_BroadcastPolygammaOp], + [TF_PowOp, CHLO_BroadcastPowOp], + [TF_RealDivOp, CHLO_BroadcastDivOp], + [TF_SubOp, CHLO_BroadcastSubOp], + [TF_ZetaOp, CHLO_BroadcastZetaOp]] in + def : DirectBinaryPat; + +def LowerRightShiftSigned : + Pat<(TF_RightShiftOp AnyTensor:$l, AnyTensor:$r), + (CHLO_BroadcastShiftRightArithmeticOp $l, $r, + (BinBroadcastDimensions $l, $r)), + [(SignedIntTensor $r)]>; + +def LowerRightShiftUnsigned : + Pat<(TF_RightShiftOp AnyTensor:$l, AnyTensor:$r), + (CHLO_BroadcastShiftRightLogicalOp $l, $r, + (BinBroadcastDimensions $l, $r)), + [(UnsignedIntTensor $r)]>; + +// Performs a substitution of FloorDiv, pseudo code below: +// +// return floor(div(x, y)) +def : Pat<(TF_FloorDivOp AnyTensor:$l, AnyTensor:$r), + (StableHLO_FloorOp + (CHLO_BroadcastDivOp $l, $r, (BinBroadcastDimensions $l, $r))), + [(IEEEFloatTensor $l)]>; + +// Performs a substitution of FloorDiv for integer tensors, which required +// additional correction for a negative numerator / denominator. Equivalent +// pseudocode is shown below: +// +// T z = x / y +// return (z * y != x && (x < 0) != (y < 0)) ? z - 1 : z +// +// BroadcastToDimensions is used to compute the broadcast attr to higher +// dimensions. This computes the broadcast of 'l' to broadcast('l', 'r') +// without returning the broadcast of 'r' to broadcast('l', 'r'). +def : Pat<(TF_FloorDivOp AnyTensor:$l, AnyTensor:$r), + (StableHLO_SelectOp + (CHLO_BroadcastAndOp + (CHLO_BroadcastCompareOp + (CHLO_BroadcastMulOp:$mul + (CHLO_BroadcastDivOp:$div $l, $r, + (BinBroadcastDimensions $l, $r)), + $r, (BinBroadcastDimensions $div, $r)), + $l, (BinBroadcastDimensions $mul, $l), CHLO_ComparisonDirectionValue<"NE">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (CHLO_BroadcastCompareOp + (CHLO_BroadcastCompareOp:$l_cmp $l, + (StableHLO_ConstantOp:$l_zeros (GetScalarOfType<0> $l)), + (NullDenseI64ArrayAttr), CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (CHLO_BroadcastCompareOp:$r_cmp $r, + (StableHLO_ConstantOp:$r_zeros (GetScalarOfType<0> $r)), + (NullDenseI64ArrayAttr), CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (BinBroadcastDimensions $l_cmp, $r_cmp), CHLO_ComparisonDirectionValue<"NE">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (NullDenseI64ArrayAttr)), + (CHLO_BroadcastSubOp $div, + (StableHLO_ConstantOp:$ones (GetScalarOfType<1> $div)), + (NullDenseI64ArrayAttr)), $div), + [(SignedIntTensor $l)]>; + +// FloorDiv of unsigned is just div. +def : Pat<(TF_FloorDivOp AnyTensor:$l, AnyTensor:$r), + (CHLO_BroadcastDivOp $l, $r, (BinBroadcastDimensions $l, $r)), + [(UnsignedIntTensor $l)]>; + +// Performs a substitution of FloorMod designed to correct for possibly negative +// values. Pseudocode shown below: +// +// T trunc_mod = std::fmod(x, y); +// return trunc_mod != 0 && (y < 0 != trunc_mod < 0) ? trunc_mod + y +// : trunc_mod +def : Pat<(TF_FloorModOp AnyTensor:$l, AnyTensor:$r), + (StableHLO_SelectOp + (CHLO_BroadcastAndOp + (CHLO_BroadcastCompareOp + (CHLO_BroadcastRemOp:$rem $l, $r, (BinBroadcastDimensions $l, $r)), + (StableHLO_ConstantOp:$l_zeros (GetScalarOfType<0> $l)), + (NullDenseI64ArrayAttr), CHLO_ComparisonDirectionValue<"NE">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (CHLO_BroadcastCompareOp + (CHLO_BroadcastCompareOp:$r_cmp $r, + (StableHLO_ConstantOp:$r_zeros (GetScalarOfType<0> $r)), + (NullDenseI64ArrayAttr), CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (CHLO_BroadcastCompareOp:$rem_cmp $rem, $r_zeros, + (BinBroadcastDimensions $rem, $r_zeros), CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (BinBroadcastDimensions $r_cmp, $rem_cmp), CHLO_ComparisonDirectionValue<"NE">, + (CHLO_DEFAULT_COMPARISON_TYPE)), + (NullDenseI64ArrayAttr)), + (CHLO_BroadcastAddOp $r, + $rem, (BinBroadcastDimensions $r, $rem)), $rem), + [(TensorOf<[I8, I16, I32, I64, F16, F32, F64]> $l)]>; + +// FloorMod of unsigned is just mod. +def : Pat<(TF_FloorModOp AnyTensor:$l, AnyTensor:$r), + (CHLO_BroadcastRemOp $l, $r, (BinBroadcastDimensions $l, $r)), + [(UnsignedIntTensor $l)]>; + +def Get2DTransposePerm: NativeCodeCall< + "Get2DTransposePerm($0, &$_builder)">; + +def : Pat<(TF_RiscAddOp $l, $r), (StableHLO_AddOp $l, $r)>; + +def : Pat<(TF_RiscDotOp $a, $b, $transpose_a, $transpose_b), + (StableHLO_DotOp + (TF_TransposeOp $a, (TF_ConstOp (Get2DTransposePerm $transpose_a))), + (TF_TransposeOp $b, (TF_ConstOp (Get2DTransposePerm $transpose_b))), + /*precision_config=*/(NullArrayAttr))>; + +//===----------------------------------------------------------------------===// +// Logical & bitwise binary op patterns. +//===----------------------------------------------------------------------===// + +class DirectLogicalBinaryPat + : Pat<(FromOp AnyTensor:$l, AnyTensor:$r), + (ToOp $l, $r, (BinBroadcastDimensions $l, $r)), + [(AnyTypeOf<[SignedIntTensor, UnsignedIntTensor]> $l)]>; + +foreach fromToBinPair = [[TF_LogicalAndOp, CHLO_BroadcastAndOp], + [TF_LogicalOrOp, CHLO_BroadcastOrOp], + [TF_BitwiseAndOp, CHLO_BroadcastAndOp], + [TF_BitwiseOrOp, CHLO_BroadcastOrOp], + [TF_BitwiseXorOp, CHLO_BroadcastXorOp]] in + def : DirectLogicalBinaryPat; + +//===----------------------------------------------------------------------===// +// Compare op patterns. +//===----------------------------------------------------------------------===// + +class DirectComparePat + : Pat<(FromOp AnyTensor:$l, AnyTensor:$r), + (CHLO_BroadcastCompareOp + $l, $r, (BinBroadcastDimensions $l, $r), direction, + (CHLO_DEFAULT_COMPARISON_TYPE))>; + +def : DirectComparePat>; +def : DirectComparePat>; +def : DirectComparePat>; +def : DirectComparePat>; + +class EqualityPat + : Pat<(FromOp AnyTensor:$l, AnyTensor:$r, + TrueBoolAttr:$incompatible_shape_error), + (CHLO_BroadcastCompareOp + $l, $r, (BinBroadcastDimensions $l, $r), direction, + (CHLO_DEFAULT_COMPARISON_TYPE)), + [(HLO_Tensor $l)]>; + +def : EqualityPat>; +def : EqualityPat>; + +//===----------------------------------------------------------------------===// +// Concat op patterns. +//===----------------------------------------------------------------------===// + +def OneElementAttrPred + : CPred<"llvm::cast($_self).getShapedType().getNumElements() == 1">; + +def OneElementAttr + : ElementsAttrBase, + "Scalar ElementsAttr">; + +def HasRankedFirstOperand + : Constraint((*$0.begin()).getType())">>; + +def IsShapedTensor + : Constraint($0.getType())">>; + +// This pattern converts TensorFlow axis format to HLO axis format which +// doesn't wrap around like TensorFlow and is always positive. For this +// conversion, use the first input to get inputs rank. Other inputs need not be +// ranked. +// Defining op for `axis` is TensorFlow constant op in the pattern as during +// the conversion, original Concat op operands still refers to the old ops even +// if HLO constant op is introduced as an replacement for the TensorFlow +// Constant op. +def : Pat<(TF_ConcatV2Op $inputs, (ConstantLikeMatcher OneElementAttr:$axis)), + (StableHLO_ConcatenateOp $inputs, + (GetHLOAxisFromTFAxisVariadic $axis, $inputs)), + [(HasRankedFirstOperand $inputs)]>; + +//===----------------------------------------------------------------------===// +// CollectivePermute op patterns. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_CollectivePermuteOp $input, (ConstantLikeMatcher ElementsAttr:$source_target_pairs)), + (StableHLO_CollectivePermuteOp $input, + (CastElementsToI64Elements $source_target_pairs), + (StableHLO_NullChannelHandleAttr))>; + +//===----------------------------------------------------------------------===// +// CrossReplicaSum op patterns. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_CrossReplicaSumOp $input, (ConstantLikeMatcher ElementsAttr:$group_assignment)), + (StableHLO_CrossReplicaSumOp $input, + (CastElementsToI64Elements $group_assignment))>; + +//===----------------------------------------------------------------------===// +// All2All op patterns. +//===----------------------------------------------------------------------===// + +def ValueToVariadic: NativeCodeCall<"SmallVector{$0}">; +def : Pat<(TF_AllToAllOp AnyRankedTensor:$input, (ConstantLikeMatcher ElementsAttr:$group_assignment), I64Attr:$concat_dimension, $split_dimension, $split_count), + (StableHLO_AllToAllOp (ValueToVariadic $input), $split_dimension, $concat_dimension, $split_count, (CastElementsToI64Elements $group_assignment), (StableHLO_NullChannelHandleAttr))>; + +//===----------------------------------------------------------------------===// +// FFT op patterns. +//===----------------------------------------------------------------------===// + +class StableHLO_FftTypeValue : + ConstantAttr; + +def GetInnerDimFromValue : NativeCodeCall< + "GetInnerDimFromValue(llvm::cast($0.getType()), &$_builder)">; + +def CheckInnerDimStatic + : Constraint($0.getType()), &$_builder)">>; + +def : Pat<(TF_FFTOp:$res $input), + (StableHLO_FftOp $input, StableHLO_FftTypeValue<"FFT">, (GetInnerDimFromValue $res)), + [(CheckInnerDimStatic $input)]>; + +def : Pat<(TF_IFFTOp:$res $input), + (StableHLO_FftOp $input, StableHLO_FftTypeValue<"IFFT">, (GetInnerDimFromValue $res)), + [(CheckInnerDimStatic $input)]>; + +//===----------------------------------------------------------------------===// +// GatherV2 op patterns. +//===----------------------------------------------------------------------===// + +// Here, $params and $indices needs to be ranked so that $axis and $batch_dims +// attributes can be converted from TensorFlow axis format supporting negative +// indexing to the HLO format. +def LegalizeGatherV2 : + Pat<(TF_GatherV2Op AnyRankedTensor:$params, AnyRankedTensor:$indices, + (ConstantLikeMatcher ElementsAttr:$axis), $batch_dims), + (StableHLO_TorchIndexSelectOp $params, $indices, + (GetHLOAxisFromTFAxis $axis, $params), + (GetHLOAxisFromTFAxis $batch_dims, $indices))>; + +//===----------------------------------------------------------------------===// +// Pad op patterns. +//===----------------------------------------------------------------------===// + +class SliceDenseIntElementsAttrColumn2D : NativeCodeCall< + "SliceDenseIntElementsAttrColumn2D(llvm::cast($0), " # column # " )">; + +class SliceDenseIntElementsAttr : NativeCodeCall< + "SliceDenseIntElementsAttr(llvm::cast($0), " # index # ", " # axis # ")">; + +// Interior padding attribute based on the TF padding. +def GetInteriorPadding : NativeCodeCall< + "GetInteriorPadding(llvm::cast($0))">; + +def : Pat<(TF_PadV2Op $input, (ConstantLikeMatcher ElementsAttr:$padding), $c), + (StableHLO_PadOp $input, $c, + (SliceDenseIntElementsAttrColumn2D<"0"> $padding), + (SliceDenseIntElementsAttrColumn2D<"1"> $padding), + (GetInteriorPadding $padding))>; + +//===----------------------------------------------------------------------===// +// Identity op patterns. +//===----------------------------------------------------------------------===// + +foreach src = [TF_IdentityOp, TF_StopGradientOp, TF__EagerConstOp] in + def : Pat<(src $op), (replaceWithValue $op)>; + +// TODO(b/32223192): Support CheckNumerics in HLO. +foreach src = [TF_PreventGradientOp, TF_CheckNumericsOp] in + def : Pat<(src $op, $msg), (replaceWithValue $op)>; + +//===----------------------------------------------------------------------===// +// MatMul op patterns. +//===----------------------------------------------------------------------===// + +def StableHLO_GetPrecisionConfig: NativeCodeCall< + "GetPrecisionConfig(&$_builder)">; + +def : Pat<(TF_MatMulOp $a, $b, $transpose_a, $transpose_b, $grad_a, $grad_b), + (StableHLO_DotOp + (TF_TransposeOp $a, (TF_ConstOp (Get2DTransposePerm $transpose_a))), + (TF_TransposeOp $b, (TF_ConstOp (Get2DTransposePerm $transpose_b))), + /*precision_config=*/(StableHLO_GetPrecisionConfig))>; + +//===----------------------------------------------------------------------===// +// Lower `tf.ZerosLike` +//===----------------------------------------------------------------------===// + +def : Pat<(TF_ZerosLikeOp AnyTensor:$arg), + (StableHLO_ConstantLike<"0"> $arg)>; + +//===----------------------------------------------------------------------===// +// Lower `tf.OnesLike` +//===----------------------------------------------------------------------===// + +def : Pat<(TF_OnesLikeOp AnyTensor:$arg), + (StableHLO_ConstantLike<"1"> $arg)>; + +//===----------------------------------------------------------------------===// +// Elu op patterns. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_EluOp AnyTensor:$features), + (StableHLO_SelectOp + (StableHLO_CompareOp + $features, + (StableHLO_ConstantLike<"0">:$zero $features), + StableHLO_ComparisonDirectionValue<"GT">, (STABLEHLO_DEFAULT_COMPARISON_TYPE)), + $features, + (StableHLO_Expm1Op $features, ConstDefaultResultAccuracyAttr))>; + +def : Pat<(TF_EluGradOp AnyStaticShapeTensor:$gradients, AnyRankedTensor:$features), + (StableHLO_SelectOp + (CHLO_BroadcastCompareOp + $features, + (StableHLO_ConstantOp:$zero (GetScalarOfType<0> $features)), + (BinBroadcastDimensions $zero, $features), + CHLO_ComparisonDirectionValue<"GT">, (CHLO_DEFAULT_COMPARISON_TYPE)), + $gradients, + (StableHLO_MulOp + $gradients, + (CHLO_BroadcastAddOp + $features, + (StableHLO_ConstantOp:$one (GetScalarOfType<1> $features)), + (BinBroadcastDimensions $one, $features))))>; + +//===----------------------------------------------------------------------===// +// Relu op patterns. +//===----------------------------------------------------------------------===// + +// TODO(hinsu): Make these patterns to TF to TF lowering. Relu6 lowering will +// require HLO canonicalization of min and max on a tensor to ClampOp. + +// TODO(hinsu): Lower quantized types after supporting them in GetScalarOfType. +def : Pat<(TF_ReluOp AnyTensor:$input), + (CHLO_BroadcastMaxOp + (StableHLO_ConstantOp:$zero (GetScalarOfType<0> $input)), $input, + (BinBroadcastDimensions $zero, $input)), + [(TF_IntOrFpTensor $input)]>; + +// TODO(hinsu): Lower quantized types after supporting them in GetScalarOfType. +def : Pat<(TF_Relu6Op AnyRankedTensor:$input), + (StableHLO_ClampOp (StableHLO_ConstantOp (GetScalarOfType<0> $input)), $input, + (StableHLO_ConstantOp (GetScalarOfType<6> $input))), + [(TF_IntOrFpTensor $input)]>; + +// ReluGrad(gradients, features) = gradients * (features > 0) +// The condition that $gradients and $features need to have the same shape is +// implicitly enforced: $zero is created to have the same shape as $features, +// StableHLO_SelectOp enforces that $gradients and $zero have the same shape. +def : Pat<(TF_ReluGradOp AnyTensor:$gradients, AnyTensor:$features), + (StableHLO_SelectOp + (StableHLO_CompareOp $features, (StableHLO_ConstantLike<"0">:$zero $features), + StableHLO_ComparisonDirectionValue<"GT">, (STABLEHLO_DEFAULT_COMPARISON_TYPE)), + $gradients, $zero)>; + +//===----------------------------------------------------------------------===// +// Softsign op patterns. +//===----------------------------------------------------------------------===// + +/// Converts a TF::SoftsignOp to HLO. +/// Softsign(features) = features / (1 + abs(features)) +def : Pat<(TF_SoftsignOp AnyTensor:$input), + (StableHLO_DivOp + $input, + (StableHLO_AddOp (StableHLO_ConstantLike<"1"> $input), (StableHLO_AbsOp $input)) + ) + >; + +/// Converts a TF::SoftsignGradOp to HLO. +/// SoftsignGrad(gradient, features) = gradient / ((1 + abs(features)) ^ 2) +def : Pattern< + (TF_SoftsignGradOp AnyRankedTensor:$gradients, AnyRankedTensor:$features), + [(CHLO_BroadcastAddOp:$add + (StableHLO_ConstantOp:$one (GetScalarOfType<1> $features)), (StableHLO_AbsOp $features), + (BinBroadcastDimensions $one, $features) + ), + (CHLO_BroadcastDivOp + $gradients, + (StableHLO_MulOp $add, $add), + (BinBroadcastDimensions $gradients, $add) + ) + ]>; + +//===----------------------------------------------------------------------===// +// Slice op patterns. +//===----------------------------------------------------------------------===// + +def UnpackStartingIndices: NativeCodeCall< + "UnpackTensorAlongZeroDim($0.getLoc(), $1, &$_builder).getOutput()">; + +def CanBeTranslatedToDynamicSlice : Constraint($2))">>; + +def TFSliceSizes2HLOSliceSizes : NativeCodeCall< + "TFSliceSizes2HLOSliceSizes($0, $1, llvm::cast($2)," + "&$_builder)">; + +def : Pat<(TF_SliceOp:$op HLO_Tensor:$input, HLO_Tensor:$starting_indices, + (ConstantLikeMatcher AnyAttr:$slice_sizes)), + (StableHLO_DynamicSliceOp $input, + (UnpackStartingIndices $op, $starting_indices), + (TFSliceSizes2HLOSliceSizes $input, $starting_indices, $slice_sizes)), + [(CanBeTranslatedToDynamicSlice $input, $starting_indices, + $slice_sizes)]>; + +//===----------------------------------------------------------------------===// +// Select op patterns. +//===----------------------------------------------------------------------===// + + def : Pat<(TF_SelectV2Op HLO_Tensor:$pred, HLO_Tensor:$on_true, + HLO_Tensor:$on_false), + (CHLO_BroadcastSelectOp $pred, $on_true, $on_false)>; + +//===----------------------------------------------------------------------===// +// PartitionedCall and LegacyCall op patterns. +//===----------------------------------------------------------------------===// + +def ArgTypesMatchCallee : Constraint< + // $0 is a resultset (possibly empty), and $_op isn't assigned. So retrieve + // the op using the builder. + CPred<"ArgTypesMatchCallee(&*$_builder.getInsertionPoint(), $1, $2)">>; + +foreach callOp = [TF_PartitionedCallOp, TF_StatefulPartitionedCallOp] in { + def : Pat<(callOp:$op $args, $args_attrs, $res_attrs, FlatSymbolRefAttr:$f, + $config, $config_proto, $executor_type), + (CallOp $f, $args, $args_attrs, $res_attrs, + ConstantAttr), + [(ArgTypesMatchCallee $op, $args, $f)]>; +} + +// The extra attr on this op is _disable_call_shape_inference, which we ignore +// in the bridge. +def : Pat<(TF_LegacyCallOp:$op $args, $args_attrs, $res_attrs, + FlatSymbolRefAttr:$f, $attr), + (CallOp $f, $args, $args_attrs, $res_attrs, + ConstantAttr), + [(ArgTypesMatchCallee $op, $args, $f)]>; + +//===----------------------------------------------------------------------===// +// Reverse op patterns. +//===----------------------------------------------------------------------===// + +// Handles axis conversion for TF reverse. +def ConvertAxisAttr : NativeCodeCall<"ConvertAxisAttr($0, llvm::cast($1), &$_builder)">; + +def : Pat<(TF_ReverseV2Op AnyRankedTensor:$values, (ConstantLikeMatcher ElementsAttr:$axis)), + (StableHLO_ReverseOp $values, (ConvertAxisAttr $values, $axis))>; + +//===----------------------------------------------------------------------===// +// Unary op patterns. +//===----------------------------------------------------------------------===// + +foreach Mapping = [ + [TF_AbsOp, StableHLO_AbsOp], + [TF_CeilOp, StableHLO_CeilOp], + [TF_ComplexAbsOp, StableHLO_AbsOp], + [TF_ErfOp, CHLO_ErfOp], + [TF_FloorOp, StableHLO_FloorOp], + [TF_ImagOp, StableHLO_ImagOp], + [TF_InvertOp, StableHLO_NotOp], + [TF_IsFiniteOp, StableHLO_IsFiniteOp], + [TF_LogicalNotOp, StableHLO_NotOp], + [TF_NegOp, StableHLO_NegOp], + [TF_RealOp, StableHLO_RealOp], + ] in { + def : Pat<(Mapping[0] HLO_Tensor:$input), + (Mapping[1] $input)>; +} + + +foreach Mapping = [ + [TF_CosOp, StableHLO_CosineOp], + [TF_ExpOp, StableHLO_ExpOp], + [TF_Expm1Op, StableHLO_Expm1Op], + [TF_LogOp, StableHLO_LogOp], + [TF_Log1pOp, StableHLO_Log1pOp], + [TF_RsqrtOp, StableHLO_RsqrtOp], + [TF_SigmoidOp, StableHLO_LogisticOp], + [TF_SinOp, StableHLO_SineOp], + [TF_SqrtOp, StableHLO_SqrtOp], + [TF_TanhOp, StableHLO_TanhOp], + [TF_TanOp, StableHLO_TanOp] + ] in { + def : Pat<(Mapping[0] HLO_Tensor:$input), + (Mapping[1] $input, ConstDefaultResultAccuracyAttr)>; +} + +foreach Mapping = [ + [TF_AcosOp, CHLO_AcosOp], + [TF_AcoshOp, CHLO_AcoshOp], + [TF_AsinOp, CHLO_AsinOp], + [TF_AsinhOp, CHLO_AsinhOp], + [TF_AtanOp, CHLO_AtanOp], + [TF_AtanhOp, CHLO_AtanhOp], + [TF_CoshOp, CHLO_CoshOp], + [TF_ConjOp, CHLO_ConjOp], + [TF_DigammaOp, CHLO_DigammaOp], + [TF_ErfcOp, CHLO_ErfcOp], + [TF_IsInfOp, CHLO_IsInfOp], + [TF_LgammaOp, CHLO_LgammaOp], + [TF_SinhOp, CHLO_SinhOp], + ] in { + def : Pat<(Mapping[0] HLO_AnyTensor:$input), + (Mapping[1] $input)>; +} + +def : Pat<(TF_AngleOp $x), (StableHLO_Atan2Op (StableHLO_ImagOp $x), (StableHLO_RealOp $x))>; + +// TODO(bixia): Lower with Truncate=True for floating point value conversions. +def : Pat<(TF_CastOp $arg, ConstBoolAttrFalse), (StableHLO_ConvertOp $arg)>; + +def : Pat<(TF_TransposeOp:$res $arg, (ConstantLikeMatcher ElementsAttr:$permutation)), + (StableHLO_TransposeOp $arg, (CastElementsToI64Array $permutation))>; + + +// Lowering these ops with static shape to stablehlo.reshape +foreach TfOp = [TF_ExpandDimsOp, TF_ReshapeOp, TF_SqueezeOp, ] in { + def : Pat<(TfOp:$res HLO_Tensor:$arg, $ignored), + (StableHLO_ReshapeOp $arg), [(AnyStaticShapeTensor $res)], [], + (addBenefit 2)>; +} + +// Returns NaN if x is NaN, 0 if x is 0, -1 if x < 0 and 1 if x > 0. +def : Pat<(TF_SignOp $x), (StableHLO_SignOp $x)>; + +def BothElementTypesSameWidthIntOrFloat : Constraint, + "element types must be integers or floats">; + +// TODO(mgester): Due to restrictions of xla::BitcastConvertType we currently +// only lower if both input and output types are int or float and have same width + +def : Pat<(TF_BitcastOp:$res HLO_Tensor:$arg), + (StableHLO_BitcastConvertOp $arg), + [(BothElementTypesSameWidthIntOrFloat $res, $arg)]>; + +// TODO(jpienaar): Lower constant like to constant to broadcast if dynamic +// and going to StableHLO. + +//===----------------------------------------------------------------------===// +// Random ops. +//===----------------------------------------------------------------------===// +// TODO(b/148269299): handle random number generator seeds/states correctly. + +class StableHLO_RngDistributionValue : + ConstantAttr; + +def : Pat<(TF_RandomUniformOp:$old $shape, $seed, $seed2), + (StableHLO_RngOp + (StableHLO_ConstantOp + (NativeCodeCall<"$_builder.getFloatAttr(old.getDtype(), 0.0)">)), + (StableHLO_ConstantOp + (NativeCodeCall<"$_builder.getFloatAttr(old.getDtype(), 1.0)">)), + (CastValueToI64 $old, $shape), + StableHLO_RngDistributionValue<"UNIFORM">), + [(IsShapedTensor $shape)]>; + +def : Pat<(TF_RandomStandardNormalOp:$old $shape, $seed, $seed2), + (StableHLO_RngOp + (StableHLO_ConstantOp + (NativeCodeCall<"$_builder.getFloatAttr(old.getDtype(), 0.0)">)), + (StableHLO_ConstantOp + (NativeCodeCall<"$_builder.getFloatAttr(old.getDtype(), 1.0)">)), + (CastValueToI64 $old, $shape), + StableHLO_RngDistributionValue<"NORMAL">), + [(IsShapedTensor $shape)]>; + +//===----------------------------------------------------------------------===// +// Sigmoid grad op. +//===----------------------------------------------------------------------===// + +// Only handle static shape here, dynamic shape is handled by +// ConvertSigmoidGradOpDynamic +def HasStaticShape : Constraint< + CPred<"::llvm::dyn_cast($0.getType()).hasStaticShape()">>; + +def : Pat<(TF_SigmoidGradOp AnyRankedTensor:$l, AnyRankedTensor:$r), + (StableHLO_MulOp + (StableHLO_MulOp $r, $l), + (StableHLO_SubtractOp (StableHLO_ConstantOp (ConstantSplat<"1"> $l)), $l)), + [(HasStaticShape $l)]>; + +//===----------------------------------------------------------------------===// +// Softplus op. +//===----------------------------------------------------------------------===// + +def EpsilonValue : NativeCodeCall<"GetEpsilonValue($0.getType())">; + +def : Pattern<(TF_SoftplusOp AnyTensor:$features), + [ + (StableHLO_ExpOp:$features_exp $features, ConstDefaultResultAccuracyAttr), + (CHLO_BroadcastAddOp:$threshold + (StableHLO_LogOp (StableHLO_ConstantOp (EpsilonValue $features)), ConstDefaultResultAccuracyAttr), + (StableHLO_ConstantOp (GetScalarOfType<2> $features)), + (NullDenseI64ArrayAttr) + ), + (StableHLO_SelectOp:$output + (CHLO_BroadcastCompareOp + $features, + (StableHLO_NegOp $threshold), + (NullDenseI64ArrayAttr), + CHLO_ComparisonDirectionValue<"GT">, + (CHLO_DEFAULT_COMPARISON_TYPE) + ), + $features, + (StableHLO_SelectOp + (CHLO_BroadcastCompareOp + $features, + $threshold, + (NullDenseI64ArrayAttr), + CHLO_ComparisonDirectionValue<"LT">, + (CHLO_DEFAULT_COMPARISON_TYPE) + ), + $features_exp, + (StableHLO_Log1pOp $features_exp, ConstDefaultResultAccuracyAttr) + ) + ), + (replaceWithValue $output) + ]>; + +//===----------------------------------------------------------------------===// +// XlaReplicaId op. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_XlaReplicaIdOp), + (TF_CastOp (StableHLO_ReplicaIdOp), /*truncate=*/ConstBoolAttrFalse)>; + +//===----------------------------------------------------------------------===// +// XlaGather op. +//===----------------------------------------------------------------------===// + +def ToGatherDimNumsAttr : NativeCodeCall<"GetGatherDimNumsAttr($0, &$_builder)">; + +def HasValidGatherDims : Constraint>; + +def : Pat<(TF_XlaGatherOp $operand, $start_indices, (ConstantLikeMatcher ElementsAttr:$slice_sizes), + $dimension_numbers, $indices_are_sorted), + (StableHLO_GatherOp $operand, $start_indices, + (ToGatherDimNumsAttr $dimension_numbers), + (CastElementsToI64Array $slice_sizes), + $indices_are_sorted), + [(HasValidGatherDims $dimension_numbers)]>; + +//===----------------------------------------------------------------------===// +// XlaDotOp op. +//===----------------------------------------------------------------------===// + +def ToDotDimNumsAttr : NativeCodeCall<"GetDotDimNumsAttr($0, &$_builder)">; + +def ToPrecisionConfigsAttr : NativeCodeCall<"GetPrecisionConfigAttr($0, &$_builder)">; + +def HasValidDotDims : Constraint>; + +def HasValidPrecisionConfig : Constraint>; + +def : Pat<(TF_XlaDotOp $lhs, $rhs, $dimension_numbers, $precision_config), + (StableHLO_DotGeneralOp $lhs, $rhs, + (ToDotDimNumsAttr $dimension_numbers), + (ToPrecisionConfigsAttr $precision_config), + (EmptyDotAlgorithmAttr)), + [(HasValidDotDims $dimension_numbers), (HasValidPrecisionConfig $precision_config)]>; + +//===----------------------------------------------------------------------===// +// XlaDotV2Op op. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_XlaDotV2Op $lhs, $rhs, $dimension_numbers, $precision_config), + (StableHLO_DotGeneralOp $lhs, $rhs, + (ToDotDimNumsAttr $dimension_numbers), + (ToPrecisionConfigsAttr $precision_config), + (EmptyDotAlgorithmAttr)), + [(HasValidDotDims $dimension_numbers), (HasValidPrecisionConfig $precision_config)]>; + +//===----------------------------------------------------------------------===// +// XlaDynamicSlice op. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_XlaDynamicSliceOp:$op HLO_Tensor:$input, HLO_Tensor:$starting_indices, + (ConstantLikeMatcher AnyAttr:$slice_sizes)), + (StableHLO_DynamicSliceOp $input, + (UnpackStartingIndices $op, $starting_indices), + (TFSliceSizes2HLOSliceSizes $input, $starting_indices, $slice_sizes))>; + +//===----------------------------------------------------------------------===// +// XlaEisumOp op. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_XlaEinsumOp $lhs, $rhs, $equation), + (StableHLO_EinsumOp $lhs, $rhs, $equation)>; + +//===----------------------------------------------------------------------===// +// XlaOptimizationBarrierOp op. +//===----------------------------------------------------------------------===// + +def : Pat<(TF_XlaOptimizationBarrierOp $args), + (StableHLO_OptimizationBarrierOp $args)>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla.cc new file mode 100644 index 0000000000000000000000000000000000000000..d69d8e799c785885498bde46c49e8f2d69fa3aae --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla.cc @@ -0,0 +1,240 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "stablehlo/dialect/Base.h" // from @stablehlo +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tpu_embedding_ops_registry.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h" +#include "tensorflow/compiler/tf2xla/xla_compilation_device.h" +#include "tensorflow/compiler/tf2xla/xla_context.h" +#include "tensorflow/compiler/tf2xla/xla_expression.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/tsl/platform/env.h" +#include "xla/tsl/platform/status.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_factory.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/process_function_library_runtime.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_properties.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace mlir { +namespace mhlo { + +namespace { + +// Returns true if the given type is a ranked tensor type with static or bounded +// dimensions. +bool IsBounded(Type ty) { + auto ranked_ty = mlir::dyn_cast(ty); + if (!ranked_ty) return false; + + if (ranked_ty.hasStaticShape()) return true; + + auto bounds = hlo::encodingToBounds(ranked_ty.getEncoding()); + if (bounds.empty()) return false; + + for (int i = 0; i < ranked_ty.getRank(); ++i) { + if (ranked_ty.isDynamicDim(i) && bounds[i] == ShapedType::kDynamic) { + return false; + } + } + return true; +} + +bool HasSymbolRefAttr(Operation* op) { + for (const auto& attr : op->getAttrs()) { + Attribute attr_value = attr.getValue(); + if (mlir::isa(attr_value)) { + return true; + } else if (auto array_attr = mlir::dyn_cast(attr_value)) { + if (!array_attr.empty() && + mlir::isa(*array_attr.begin())) { + return true; + } + } + } + return false; +} + +class Tf2XlaRewritePattern : public ConversionPattern { + public: + explicit Tf2XlaRewritePattern(MLIRContext* ctx, TypeConverter& converter, + const std::string& device_type, + bool prefer_tf2xla) + : ConversionPattern(converter, MatchAnyOpTypeTag(), /*benefit=*/1, ctx), + device_type_(device_type), + prefer_tf2xla_(prefer_tf2xla) {} + + LogicalResult matchAndRewrite( + Operation* op, ArrayRef operands, + ConversionPatternRewriter& rewriter) const override { + // This pattern is a conversion pattern because we want to specify a type + // converter. However, this pattern still uses the original op's operands + // while creating the ops so make sure there aren't any type changes between + // the original op operands and the operands during the conversion. + for (auto&& [old_val, new_val] : llvm::zip(op->getOperands(), operands)) { + if (old_val.getType() != new_val.getType()) return failure(); + } + + auto abstractOp = op->getRegisteredInfo(); + if (!abstractOp) return failure(); + + if (!(hlo::IsOpAllowedTf2xlaFallback(abstractOp->getTypeID()) || + (prefer_tf2xla_ && + hlo::IsOpAllowedTf2xlaPreferred(abstractOp->getTypeID())))) { + return failure(); + } + + return hlo::Tf2XlaRewriter::RewriteOp(op, rewriter, device_type_); + } + + private: + std::string device_type_; + bool prefer_tf2xla_; + bool use_tf2xla_hlo_importer_; +}; + +bool ShouldRefineTypeTo(Type original_ty, Type updated_ty) { + auto updated = mlir::dyn_cast(updated_ty); + auto original = mlir::dyn_cast(original_ty); + + // Both types must be shaped types. + if (!original || !updated) return false; + + // Element types must match. + if (original.getElementType() != updated.getElementType()) return false; + + // If the updated type doesn't have a rank, then it can't be a more refined + // type. + if (!updated.hasRank()) return false; + + // If the original type doesn't have a rank, then refine as the updated type + // has a rank. + if (!original.hasRank()) return true; + + // Both types must have the same rank. + if (original.getRank() != updated.getRank()) return false; + + // Refine if the updated type is bounded. + return IsBounded(updated); +} + +// Propagates more refined type by cloning op using the new operands. This +// allows all rewrite patterns that requires refined types to work without +// requiring a rewrite to the conversion pattern. Declarative rewrite pattern +// (DRR) doesn't even support conversion patterns with TableGen. +class TypePropagator : public ConversionPattern { + public: + explicit TypePropagator(MLIRContext* ctx) + : ConversionPattern(MatchAnyOpTypeTag(), /*benefit=*/1, ctx) {} + + LogicalResult matchAndRewrite( + Operation* op, ArrayRef operands, + ConversionPatternRewriter& rewriter) const override { + // This could be generalized to other ops as needs arise. We could even + // remove this restriction altogether except for the terminators that + // require function signature change and shouldn't be + if (op->getName().getDialectNamespace() != + TF::TensorFlowDialect::getDialectNamespace()) + return failure(); + + // Refining types may have implications to the attached regions or symbol + // references so do not update such ops. + if (!op->getRegions().empty() || HasSymbolRefAttr(op)) return failure(); + + IRMapping mapper; + bool has_type_change = false; + for (auto [original, updated] : llvm::zip(op->getOperands(), operands)) { + Type original_ty = original.getType(); + Type updated_ty = updated.getType(); + if (original_ty != updated_ty) has_type_change = true; + + if (!ShouldRefineTypeTo(original_ty, updated_ty)) return failure(); + mapper.map(original, updated); + } + if (!has_type_change) return failure(); + + Operation* cloned_op = rewriter.clone(*op, mapper); + rewriter.replaceOp(op, cloned_op->getResults()); + return success(); + } +}; + +} // end namespace + +Tf2XlaTypeConverter::Tf2XlaTypeConverter() { + // Currently, we don't do any type conversions. Any TensorFlow op with a type + // that is not supported in MHLO will fail conversion. Quantized types are + // going to handled separately so we don't need to handle those. + addConversion([](Type ty) { return ty; }); + + // This materialization is helpful in cases where we have more refined types + // after conversion to mhlo compared to the original type in TF. For example, + // a TF op with result type tensor<*xf32> will have a bounded type after + // fallback legalization. + auto cast_value = [&](OpBuilder& builder, Type result_type, ValueRange inputs, + Location loc) -> Value { + return mlir::tensor::CastOp::create(builder, loc, result_type, + inputs.front()); + }; + addSourceMaterialization(cast_value); +} + +void PopulateLegalizeTfWithTf2XlaPatterns(llvm::StringRef device_type, + RewritePatternSet& patterns, + MLIRContext* ctx, + Tf2XlaTypeConverter& converter, + bool prefer_tf2xla) { + patterns.add(ctx); + patterns.add(ctx, converter, device_type.str(), + prefer_tf2xla); +} + +} // end namespace mhlo +} // end namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..4800c502342915c41738e588d94a8848ab6ea393 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla_passes.h @@ -0,0 +1,63 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZE_TF_WITH_TF2XLA_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZE_TF_WITH_TF2XLA_PASSES_H_ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project + +namespace mlir { + +namespace func { +class FuncOp; +} +class ModuleOp; +class Operation; +template +class OperationPass; +class Pass; + +namespace mhlo { + +/// Converter to be used along with the fallback Tf2Xla patterns below. +class Tf2XlaTypeConverter : public TypeConverter { + public: + Tf2XlaTypeConverter(); +}; + +/// Adds the TF to XLA via TF2XLA rewrite patterns to the pattern list. +/// `prefer_tf2xla` means an op will be included iff it is not in +/// `MlirLegalizedUnderPreferTf2XlaSet`. `!prefer_tf2xla` mean an op will be +/// included if there is no native MLIR legalization for the op. +void PopulateLegalizeTfWithTf2XlaPatterns(llvm::StringRef device_type, + RewritePatternSet& patterns, + MLIRContext* ctx, + Tf2XlaTypeConverter& converter, + bool prefer_tf2xla = false); + + +} // namespace mhlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_LEGALIZE_TF_WITH_TF2XLA_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/passes.h new file mode 100644 index 0000000000000000000000000000000000000000..abdd5ce69e0f8cbac3887bd3cacaabc1e3333113 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/passes.h @@ -0,0 +1,124 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_PASSES_H_ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project + +namespace mlir { + +namespace func { +class FuncOp; +} +class ModuleOp; +class Operation; +template +class OperationPass; +class Pass; + +namespace hlo { + +// Verifies that the TF/XLA ops have all been lowered to MHLO. +std::unique_ptr> CreateVerifyTFXLALegalizationPass( + bool legalize_chlo = true); + +/// Adds the TF to TF lowerings and TF to XLA rewrite patterns to the pattern +/// list. +void PopulateLegalizeTfPatterns(MLIRContext* context, + RewritePatternSet* patterns); + +} // namespace hlo + +namespace mhlo { + +/// Lowers from TF dialect to HLO dialect. When allow_partial_conversion is +/// false, emits an error if there is any operation that can't be legalized. +/// When `tf2xla_fallback_device_type` is not `None`, also uses legalization +/// patterns from TF2XLA fallback for provided device type (see +/// legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is not +/// used. +/// Note: This is a module pass because when legalizing with TF2XLA fallback, +/// functions are imported into the module. Importing functions into a +/// module is not thread safe. +std::unique_ptr> createLegalizeTFPass( + bool legalize_chlo = true, + std::optional tf2xla_fallback_device_type = std::nullopt, + bool prefer_tf2xla = false); + +// Populates TF to MHLO legalization for some of the quantization ops. +// +// TODO(hinsu): Remove this once we combine quantized and non quantized op +// legalization in the ODML conversion pipeline. +void PopulateLegalizeTfQuantizationPatterns(MLIRContext* context, + RewritePatternSet* patterns); + +/// Converts the provided Operation as well as all nested operations into HLO +/// dialect using the conversion patterns registered by the HLO dialect. When +/// allow_partial_conversion is false, emits an error if there is any operation +/// that can't be legalized. +/// When `tf2xla_fallback_device_type` is not `None`, also uses legalization +/// patterns from TF2XLA fallback for provided device type (see +/// legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is not +/// used. +LogicalResult legalizeTF( + Operation* op, bool allow_partial_conversion = false, + bool legalize_chlo = true, + std::optional tf2xla_fallback_device_type = std::nullopt, + bool prefer_tf2xla = false); + +// Legalizes TF/XLA communication ops (TF dialect) to HLO dialect communication +// ops. +std::unique_ptr> CreateLegalizeTFCommunicationPass(); + +// Legalizes TF/XLA collective ops (TF dialect) to HLO dialect collective +// ops. +std::unique_ptr> CreateLegalizeTFCollectivePass(); + +// Transforms TFXLA Device specific ops into device independent ops. +std::unique_ptr> +CreateTFXLADeviceSpecificTransformsPass( + std::optional tf2xla_fallback_device_type = std::nullopt); + +// Adjusts XLA layout for Infeed ops. +std::unique_ptr> +CreateInfeedsOpsXlaAdjustLayoutPass(); + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_INFEEDSOPSXLAADJUSTLAYOUT +#define GEN_PASS_DECL_LEGALIZETF +#define GEN_PASS_DECL_LEGALIZETFCOLLECTIVE +#define GEN_PASS_DECL_LEGALIZETFMODULEPASS +#define GEN_PASS_DECL_LEGALIZETFTYPESPASS +#define GEN_PASS_DECL_TFXLADEVICESPECIFICTRANSFORMS +#define GEN_PASS_DECL_VERIFYTFXLALEGALIZATION +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +#define GEN_PASS_REGISTRATION +#define GEN_PASS_DECL_LEGALIZETFCOMMUNICATIONPASS +#include "tensorflow/compiler/mlir/tf2xla/transforms/tf_xla_passes.h.inc" +} // namespace mhlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..9c9744f146748a1446702b85ba09d0b51795a94c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.cc @@ -0,0 +1,192 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h" + +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +// This pass is used in preparation for Graph export. +// The GraphDef exporter expects each op to be in its own island. +// This pass puts the IR in that form. +// +// We do this as an IR->IR transform to keep the Graph exporter as simple as +// possible. + +namespace mlir { +namespace TF { + +namespace { + +#define GEN_PASS_DEF_SPLITINTOISLANDPEROPPASS +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_passes.h.inc" + +class SplitIntoIslandPerOpPass + : public impl::SplitIntoIslandPerOpPassBase { + public: + void runOnOperation() override; +}; + +void SplitIntoIslandPerOpPass::runOnOperation() { + func::FuncOp func = getOperation(); + + if (func.isExternal()) { + // Just ignore the op if this is an external func with no body. + return; + } + + tf_executor::GraphOp graph_op; + + if (llvm::hasSingleElement(func.front().without_terminator())) { + graph_op = dyn_cast(func.front().front()); + } + + if (!graph_op) { + func.emitError("expected function to contain only a graph_op"); + signalPassFailure(); + return; + } + + if (!(llvm::hasSingleElement(graph_op.GetBody().without_terminator()) && + llvm::isa(graph_op.GetBody().front()))) { + graph_op.emitError( + "expected graph op to contain only a single island_op and a single " + "fetch_op"); + signalPassFailure(); + return; + } + + tf_executor::IslandOp island_op = + dyn_cast(graph_op.GetBody().front()); + + // We don't need to honor *any* control deps that are already placed on + // islands. Drop them now in this pass - a following pass will use side + // effect analysis to completely explain and apply correct control deps. + island_op.getControl().dropAllUses(); + + // Break up all islands by simply creating a new island wrapping each + // individual sub op. Do not create any control dependencies between the + // newly created islands. + SplitIsland(island_op, tf_executor::ControlType::get(&getContext())); + + // None of the originally given control deps are necessary. + tf_executor::FetchOp fetch_op = graph_op.GetFetch(); + int num_control_fetches = + fetch_op.getNumOperands() - graph_op.getNumResults(); + if (num_control_fetches > 0) { + fetch_op.getFetchesMutable().erase(graph_op.getNumResults(), + num_control_fetches); + } +} + +} // namespace + +// Populates an empty IslandOp and with a NoOp or Identity/IdentityN depending +// on if there are any data results. +void PopulateEmptyIsland(tf_executor::IslandOp island) { + OpBuilder builder(&island.GetBody(), island.GetBody().begin()); + tf_executor::YieldOp yield = island.GetYield(); + if (yield.getNumOperands() == 0) { + TF::NoOp::create(builder, island.getLoc(), TypeRange{}, ValueRange{}); + } else if (yield.getNumOperands() == 1) { + Value operand = yield.getOperand(0); + auto identity = TF::IdentityOp::create(builder, island.getLoc(), + operand.getType(), operand); + yield.setOperand(0, identity.getOutput()); + } else { + auto identity_n = TF::IdentityNOp::create( + builder, island.getLoc(), yield.getOperandTypes(), yield.getOperands()); + for (const auto& it : llvm::enumerate(identity_n.getResults())) + yield.setOperand(it.index(), it.value()); + } +} + +// Helper that creates an island that `sub_op` will be moved to. +tf_executor::IslandOp CreateIsland(TypeRange result_types, + const tf_executor::ControlType& control_type, + const Location& loc, Operation& sub_op, + tf_executor::IslandOp original_island) { + OpBuilder builder(original_island); + auto island = tf_executor::IslandOp::create(builder, loc, result_types, + control_type, mlir::ValueRange{}); + island.getBody().push_back(new Block); + Block* block = &island.getBody().back(); + OpBuilder island_builder(original_island); + island_builder.setInsertionPointToEnd(block); + sub_op.replaceAllUsesWith(island.getOutputs()); + sub_op.moveBefore(block, block->begin()); + tf_executor::YieldOp::create(island_builder, loc, sub_op.getResults()); + return island; +} + +// Converts a single island into multiple islands (one for each op). +void SplitIsland(mlir::tf_executor::IslandOp island_op, + mlir::tf_executor::ControlType control_type) { + auto island_body = island_op.GetBody().without_terminator(); + // Populate islands that are empty (only yield). + if (island_body.empty()) { + PopulateEmptyIsland(island_op); + return; + } + + // Skip islands that are already only a single op. + if (island_op.WrapsSingleOp()) return; + + // For each operation in the island, construct a new island to wrap the op, + // yield all the results, and replace all the usages with the results of the + // new island. + for (auto& sub_op : llvm::make_early_inc_range(island_body)) { + CreateIsland(sub_op.getResultTypes(), control_type, sub_op.getLoc(), sub_op, + island_op); + } + + // Ensure that consumers of the outputs of the original island now depend + // directly on the new island wrapping the original output op instead of + // depending on the original island's output since we're about to erase the + // original island op. + for (auto item : + llvm::zip(island_op.getOutputs(), island_op.GetYield().getFetches())) + std::get<0>(item).replaceAllUsesWith(std::get<1>(item)); + + auto graph_op = island_op->getParentOfType(); + + // Dropping all uses of an island op's control dep using + // `island_op.getControl().dropAllUses();` of a control dep that's only used + // in a graph's fetch, immediately leads to a segfault. Turns out we need to + // drop its uses manually so that we don't leave dangling controls. + for (const auto& fetch : llvm::enumerate(graph_op.GetFetch().getFetches())) { + if (fetch.value() == island_op.getControl()) { + graph_op.GetFetch().getFetchesMutable().erase(fetch.index(), 1); + break; + } + } + island_op.erase(); +} + +std::unique_ptr> CreateSplitIntoIslandPerOpPass() { + return std::make_unique(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..b8a6ea9696acff895a5e079290ea1db82303bc82 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h @@ -0,0 +1,31 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_SPLIT_INTO_ISLAND_PER_OP_PASS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_SPLIT_INTO_ISLAND_PER_OP_PASS_H_ + +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" + +namespace mlir { +namespace TF { + +// Converts a single island into multiple islands (one for each op). +void SplitIsland(mlir::tf_executor::IslandOp island_op, + mlir::tf_executor::ControlType control_type); + +} // namespace TF +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_SPLIT_INTO_ISLAND_PER_OP_PASS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..bc65a5af39e05fcc07e4e6924c761d5832c443ba --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass_test.cc @@ -0,0 +1,150 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tf2xla/transforms/split_into_island_per_op_pass.h" + +#include +#include +#include + +#include +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/TypeRange.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" + +namespace mlir { +namespace TF { + +using mlir::DialectRegistry; +using mlir::MLIRContext; +class SplitIntoIslandPerOpPass : public ::testing::Test { + public: + SplitIntoIslandPerOpPass() : op_builder_(&context_) { + mlir::RegisterCommonToolingDialects(registry_); + context_.appendDialectRegistry(registry_); + context_.loadAllAvailableDialects(); + context_.allowUnregisteredDialects(true); + } + + mlir::tf_executor::IslandOp GenerateIslandOp() { + mlir::OperationState op_state(mlir::UnknownLoc::get(&context_), + StringRef("test_island")); + llvm::SmallVector island_result_types; + island_result_types.push_back(op_builder_.getF64Type()); + + mlir::Operation* yield_op = mlir::tf_executor::YieldOp::create( + op_builder_, op_state.location, mlir::ValueRange{}); + mlir::tf_executor::IslandOp island_op = mlir::tf_executor::IslandOp::create( + op_builder_, op_state.location, island_result_types, mlir::ValueRange{}, + mlir::ArrayRef{}); + island_op.getBody().push_back(new mlir::Block); + island_op.getBody().back().push_back(yield_op); + return island_op; + } + + mlir::Operation* GenerateOp(StringRef name, bool add_return_type = false) { + mlir::OperationState state(mlir::UnknownLoc::get(&context_), name); + if (add_return_type) { + state.addTypes(ArrayRef{op_builder_.getF64Type()}); + } + return mlir::Operation::create(state); + } + + std::vector GetOpNames(mlir::Operation* op) { + std::vector op_names; + for (auto& op : op->getBlock()->getOperations()) { + op_names.push_back(op.getName().getStringRef().str()); + } + return op_names; + } + + DialectRegistry registry_; + MLIRContext context_; + OpBuilder op_builder_; +}; + +TEST_F(SplitIntoIslandPerOpPass, EmptyIslandPoulatesNoOp) { + mlir::tf_executor::ControlType control_type; + mlir::tf_executor::IslandOp islandOp = GenerateIslandOp(); + + SplitIsland(islandOp, control_type); + + std::set actual_op_names; + for (auto& op : islandOp.getBody().getOps()) { + actual_op_names.insert(op.getName().getStringRef().str()); + } + std::set expected_op_names = {"tf_executor.yield", "tf.NoOp"}; + ASSERT_EQ(actual_op_names, expected_op_names); + islandOp.erase(); +} + +TEST_F(SplitIntoIslandPerOpPass, IslandOpSingleOpLeftUnchanged) { + mlir::tf_executor::ControlType control_type; + mlir::tf_executor::IslandOp islandOp = GenerateIslandOp(); + mlir::Operation* inner_op = GenerateOp("inner_op"); + islandOp.getBody().front().push_front(inner_op); + + SplitIsland(islandOp, control_type); + + std::set actual_op_names; + for (auto& op : islandOp.getBody().getOps()) { + actual_op_names.insert(op.getName().getStringRef().str()); + } + std::set expected_op_names = {"inner_op", "tf_executor.yield"}; + ASSERT_EQ(actual_op_names, expected_op_names); + islandOp.erase(); +} + +TEST_F(SplitIntoIslandPerOpPass, IslandOpTwoOpsSplitsIntoTwoIslands) { + auto control_type = mlir::tf_executor::ControlType::get(&context_); + mlir::tf_executor::IslandOp islandOp = GenerateIslandOp(); + mlir::Operation* inner_op_1 = GenerateOp("inner_op_1", true); + mlir::Operation* inner_op_2 = GenerateOp("inner_op_2", true); + islandOp.getBody().front().push_front(inner_op_1); + islandOp.getBody().back().push_front(inner_op_2); + // Code relies on a parent with a fetch op containing the island op. + mlir::tf_executor::GraphOp parent_graph_op = + mlir::tf_executor::GraphOp::create( + op_builder_, mlir::UnknownLoc::get(&context_), + mlir::TypeRange{op_builder_.getF64Type()}); + parent_graph_op.getRegion().push_back(new mlir::Block); + parent_graph_op.push_back(islandOp); + mlir::tf_executor::FetchOp fetch_op = + mlir::tf_executor::FetchOp::create(op_builder_, parent_graph_op.getLoc()); + parent_graph_op.GetBody().push_back(fetch_op); + + SplitIsland(islandOp, control_type); + + std::vector actual_op_names; + for (auto& op : parent_graph_op.getBody().getOps()) { + actual_op_names.push_back(op.getName().getStringRef().str()); + } + std::vector expected_op_names = { + "tf_executor.island", "tf_executor.island", "tf_executor.fetch"}; + ASSERT_EQ(actual_op_names, expected_op_names); + parent_graph_op.erase(); +} + +} // namespace TF +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..64b979dacc8e5604a4ec5568e363ff0a843c7116 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.cc @@ -0,0 +1,54 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/tsl/platform/statusor.h" + +namespace mlir { +namespace hlo { +namespace test { + +using ::mlir::DialectRegistry; +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::tsl::StatusOr; + +absl::StatusOr> GetMlirModuleFromString( + absl::string_view module_string, MLIRContext* context) { + DialectRegistry mlir_registry; + RegisterCommonToolingDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + + OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(module_string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +} // namespace test +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..dd8975c9273adafd5379c709850328cecd896b18 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h @@ -0,0 +1,39 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TEST_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TEST_UTILS_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "xla/tsl/platform/statusor.h" + +namespace mlir { +namespace hlo { +namespace test { + +// Given a raw string, return a ModuleOp that can be used with the given +// MLIRContext. +absl::StatusOr> GetMlirModuleFromString( + absl::string_view module_string, MLIRContext* mlir_context); + +} // namespace test +} // namespace hlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TEST_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.cc new file mode 100644 index 0000000000000000000000000000000000000000..5e3387bf677d514d71ff1c1083e59b6c17b4dc5c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.cc @@ -0,0 +1,526 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/types/span.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/Base.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tpu_embedding_ops_registry.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/tf2xla/xla_compilation_device.h" +#include "tensorflow/compiler/tf2xla/xla_context.h" +#include "tensorflow/compiler/tf2xla/xla_expression.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/hlo/builder/xla_builder.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/hlo/ir/hlo_instruction.h" +#include "xla/hlo/ir/hlo_opcode.h" +#include "xla/hlo/translate/hlo_to_mhlo/hlo_function_importer.h" +#include "xla/hlo/translate/hlo_to_mhlo/hlo_to_mlir_hlo.h" +#include "xla/hlo/translate/mhlo_to_hlo/type_to_shape.h" +#include "xla/service/hlo.pb.h" +#include "xla/tsl/platform/env.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/status.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla.pb.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/common_runtime/process_function_library_runtime.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/public/session_options.h" + +namespace mlir { +namespace hlo { +namespace { + +using ::mlir::ModuleOp; +using ::tensorflow::Tensor; +using ::tsl::StatusOr; +using ::xla::XlaComputation; + +// The OpOrArgLocNameMapper adds invalid characters to the name of the op when +// concatenating locations. This version removes those characters to make the +// name valid for NodeDef. +class OpOrArgLocNameMapperWithoutInvalidCharacters + : public tensorflow::OpOrArgLocNameMapper { + public: + OpOrArgLocNameMapperWithoutInvalidCharacters() = default; + ~OpOrArgLocNameMapperWithoutInvalidCharacters() override = default; + + protected: + std::string GetName(tensorflow::OpOrVal op_or_val) override { + std::string name = OpOrArgLocNameMapper::GetName(op_or_val); + return absl::StrReplaceAll(name, {{";", "."}}); + } +}; + +static std::unique_ptr CreateDeviceMgr( + const std::string& device_type) { + // Register compilation kernels for all registered XLA backends. + tensorflow::XlaOpRegistry::RegisterCompilationKernels(); + + auto device = std::make_unique( + tensorflow::SessionOptions(), tensorflow::DeviceType(device_type)); + return std::make_unique(std::move(device)); +} + +bool RootInstructionIsTuple(const xla::HloModule& hlo_module) { + xla::HloInstruction* root_instruction = + hlo_module.entry_computation()->root_instruction(); + + return root_instruction->opcode() == xla::HloOpcode::kTuple; +} + +}; // namespace + +LogicalResult Tf2XlaRewriter::RewriteOp(Operation* op, + PatternRewriter& rewriter, + const std::string& device_type) { + Tf2XlaRewriter tf2xla_rewriter(op, rewriter, device_type); + return tf2xla_rewriter.LegalizeOp(); +} + +Tf2XlaRewriter::Tf2XlaRewriter(Operation* op, PatternRewriter& rewriter, + const std::string& device_type) + : op_(op), + device_type_(device_type), + rewriter_(rewriter), + name_mapper_( + std::make_unique()), + context_(nullptr), + xla_builder_(op_->getName().getStringRef().str()) {} + +Tf2XlaRewriter::~Tf2XlaRewriter() { + if (context_) context_->Unref(); +} + +absl::StatusOr Tf2XlaRewriter::ImportXlaComputation( + XlaComputation& computation) { + xla::DebugOptions debug_options; + TF_ASSIGN_OR_RETURN(auto hlo_module_config, + xla::HloModule::CreateModuleConfigFromProto( + computation.proto(), debug_options)); + TF_ASSIGN_OR_RETURN( + std::unique_ptr hlo_module, + xla::HloModule::CreateFromProto(computation.proto(), hlo_module_config)); + + if (!RootInstructionIsTuple(*hlo_module)) { + return absl::InvalidArgumentError("Imported XLA Root is not a tuple op"); + } + + if (op_->getNumOperands() != + hlo_module->entry_computation()->num_parameters()) { + return absl::InvalidArgumentError( + "Entry computation does not have equal number of parameters to op " + "operands"); + } + + ModuleOp mlir_module = op_->getParentOfType(); + mlir::OpBuilder builder(op_); + mlir::SymbolTable symbol_table(mlir_module); + + llvm::SmallVector arguments; + for (int i = 0; i < op_->getNumOperands(); i++) { + arguments.push_back(op_->getOperand(i)); + } + + // Ideally we could use the Function Importer but it increases compilation + // time when we have a model with thousands of tf2xla op fallbacks. At time + // of writing, this caused compilation time to be greater than 2x slower. + // So we have to directly import these instructions. + TF_ASSIGN_OR_RETURN( + mlir::Value root_value, + xla::HloFunctionImporter::ImportInstructions( + *hlo_module->entry_computation(), arguments, symbol_table, &builder)); + + stablehlo::TupleOp root_tuple = + mlir::dyn_cast_or_null(root_value.getDefiningOp()); + if (!root_tuple) { + return absl::InvalidArgumentError( + "Imported XLA Root Value is not a tuple op"); + } + + return root_tuple; +} + +LogicalResult Tf2XlaRewriter::PrepareParams() { + // XlaCompiler within the context is only used by the functional ops to + // compile functions. We are not handling those at the moment so + // XlaCompiler is not required. + context_ = new tensorflow::XlaContext(/*compiler=*/nullptr, &xla_builder_, + /*graph=*/nullptr); + context_->Ref(); + + device_mgr_ = CreateDeviceMgr(device_type_); + if (!device_mgr_) return failure(); + + // Type of params_.device is DeviceBase* so store it as Device* to access + // derived class method. + device_ = device_mgr_->ListDevices().front(); + params_.device = device_; + params_.resource_manager = device_->resource_manager(); + + // Resources are cleared at the time of device manager destruction so pass + // no-op cleanup function. + auto cleanup = [](const std::string& name) {}; + // Use step_id zero as we only have a single context concurrently and + // concurrently running each of the MLIR functions create a new device. + step_container_ = std::make_unique( + /*step_id=*/0, cleanup); + absl::Status status = step_container_->Create( + device_->resource_manager(), + tensorflow::XlaContext::kXlaContextResourceName, context_); + if (!status.ok()) { + return emitRemark(op_->getLoc()) + << "failed to create XlaContext resource: " << status.ToString(); + } + params_.step_container = step_container_.get(); + + absl::StatusOr version_or = tensorflow::GetTfGraphProducerVersion( + op_->getParentOfType()); + if (!version_or.ok()) { + return emitError(op_->getLoc()) << version_or.status().ToString(); + } + + flib_def_ = std::make_unique( + tensorflow::OpRegistry::Global(), tensorflow::FunctionDefLibrary()); + pflr_ = std::make_unique( + device_mgr_.get(), tensorflow::Env::Default(), /*config=*/nullptr, + version_or.value(), flib_def_.get(), tensorflow::OptimizerOptions()); + params_.function_library = pflr_->GetFLR(device_->name()); + return success(); +} + +// Returns true if the given type is a ranked tensor type with static or +// bounded dimensions. +bool IsBounded(Type ty) { + auto ranked_ty = mlir::dyn_cast(ty); + if (!ranked_ty) return false; + + if (ranked_ty.hasStaticShape()) return true; + + ArrayRef bounds = hlo::encodingToBounds(ranked_ty.getEncoding()); + if (bounds.empty()) return false; + + for (int i = 0; i < ranked_ty.getRank(); ++i) { + if (ranked_ty.isDynamicDim(i) && bounds[i] == ShapedType::kDynamic) { + return false; + } + } + return true; +} + +bool HasSymbolRefAttr(Operation* op) { + for (const auto& attr : op->getAttrs()) { + Attribute attr_value = attr.getValue(); + if (mlir::isa(attr_value)) { + return true; + } else if (auto array_attr = mlir::dyn_cast(attr_value)) { + if (!array_attr.empty() && + mlir::isa(*array_attr.begin())) { + return true; + } + } + } + return false; +} + +LogicalResult Tf2XlaRewriter::PrepareKernelInputs( + const llvm::SmallDenseSet& required_consts, + std::vector& expressions, + std::vector& tensors, + std::vector& inputs) { + // Prepare the list of Tensor inputs for the kernel. + for (auto it : llvm::enumerate(op_->getOperands())) { + Value operand = it.value(); + size_t idx = it.index(); + + tensorflow::XlaExpression expr = GetExprForOperand(operand, op_, idx); + tensorflow::XlaExpression::Kind kind = expr.kind(); + if (kind == tensorflow::XlaExpression::Kind::kInvalid) return failure(); + expressions.push_back(expr); + + if (!tensorflow::DataTypeCanUseMemcpy(expr.dtype())) { + return op_->emitRemark() + << "skipping legalization due to unsupported type " + << operand.getType(); + } + + auto shape_or = expr.GetShape(); + if (!shape_or.ok()) { + return op_->emitRemark() + << "failed to get shape for expression. " << expr.HumanString(); + } + + tensors.emplace_back( + device_->GetAllocator(tensorflow::AllocatorAttributes()), expr.dtype(), + shape_or.value()); + + tensorflow::Tensor& tensor = tensors.back(); + tensorflow::XlaExpression::AssignExpressionToTensor(expr, &tensor); + inputs.emplace_back(&tensor); + } + + return success(); +} + +LogicalResult Tf2XlaRewriter::LegalizeOp() { + for (Type ty : op_->getOperandTypes()) { + auto ranked_ty = mlir::dyn_cast(ty); + // Only bounded operands are supported in the XLA builders. + if (!IsBounded(ranked_ty)) { + return op_->emitRemark() + << "lowering requires bounded tensor operands " << ranked_ty; + } + } + + if (HasSymbolRefAttr(op_)) { + return op_->emitRemark() << "ops with symbol references are not supported"; + } + + auto nodedef_or = tensorflow::ConvertTFDialectOpToNodeDef( + op_, name_mapper_->GetUniqueName(op_), + /*ignore_unregistered_attrs=*/true); + if (!nodedef_or.ok()) { + return op_->emitRemark() << "failed to convert op to NodeDef: " + << nodedef_or.status().ToString(); + } + + if (failed(PrepareParams())) return failure(); + + std::shared_ptr props; + absl::Status status = tensorflow::NodeProperties::CreateFromNodeDef( + *nodedef_or.value(), + params_.function_library->GetFunctionLibraryDefinition(), &props); + if (!status.ok()) { + return op_->emitRemark() + << "failed to create NodeProperties: " << status.ToString(); + } + tensorflow::OpKernel* op_kernel_raw; + status = params_.function_library->CreateKernel(props, &op_kernel_raw); + if (!status.ok()) { + return op_->emitRemark() + << "failed to create tf2xla kernel: " << status.ToString(); + } + // Transfer ownership of the kernel to a local smart pointer. + auto op_kernel = absl::WrapUnique(op_kernel_raw); + + std::vector required_constants; + status = tensorflow::XlaOpRegistry::CompileTimeConstantInputs( + *op_kernel, &required_constants); + if (!status.ok()) { + return op_->emitRemark() + << "failed to compute required constants: " << status.ToString(); + } + + llvm::SmallDenseSet required_consts; + required_consts.insert(required_constants.begin(), required_constants.end()); + + // TensorValue in inputs are backed by tensors which in turn depend on + // expressions. So, pre-allocate them to the required size. Subtle note: + // Since these are assigned to params_, these have to live past the kernel + // compilation. + std::vector expressions; + std::vector tensors; + std::vector inputs; + expressions.reserve(op_->getNumOperands()); + tensors.reserve(op_->getNumOperands()); + inputs.reserve(op_->getNumOperands()); + + if (failed( + PrepareKernelInputs(required_consts, expressions, tensors, inputs))) + return failure(); + + params_.inputs = inputs; + params_.op_kernel = op_kernel.get(); + llvm::SmallVector output_attr( + op_->getNumResults()); + params_.output_attr_array = output_attr.data(); + + tensorflow::OpKernelContext op_context(¶ms_, op_->getNumResults()); + device_->Compute(params_.op_kernel, &op_context); + + status = op_context.status(); + if (!status.ok()) { + return op_->emitRemark() + << "compilation to HLO failed: " << status.ToString(); + } + + if (failed(VerifyOpResults(op_context))) return failure(); + + absl::StatusOr tuple_result_or_status = + CompileWithHloImporter(op_context); + if (!tuple_result_or_status.ok()) { + return op_->emitRemark() << tuple_result_or_status.status().ToString(); + } + stablehlo::TupleOp tuple_result = tuple_result_or_status.value(); + + llvm::SmallVector output_values; + if (failed(GetKernelOutputs(op_context, tuple_result, output_values))) { + return failure(); + } + + rewriter_.replaceOp(op_, output_values); + return success(); +} + +absl::StatusOr Tf2XlaRewriter::CompileWithHloImporter( + tensorflow::OpKernelContext& op_context) { + // XLA can only return a single value. Wrap all output op return values + // in a Tuple op that gets unpacked later. + std::vector output_values; + for (int i = 0, e = op_->getNumResults(); i < e; i++) { + tensorflow::Tensor* output = op_context.mutable_output(i); + const tensorflow::XlaExpression* expr = + tensorflow::XlaExpression::CastExpressionFromTensor(*output); + output_values.push_back(expr->AsXlaOp(&xla_builder_)); + } + + absl::Span return_values(output_values); + xla::XlaOp root_value = xla::Tuple(&xla_builder_, return_values); + + TF_ASSIGN_OR_RETURN(XlaComputation computation, + xla_builder_.Build(root_value, + /*remove_dynamic_dimensions=*/false)); + + return ImportXlaComputation(computation); +} + +mlir::LogicalResult Tf2XlaRewriter::VerifyOpResults( + tensorflow::OpKernelContext& op_context) { + for (int i = 0, e = op_->getNumResults(); i < e; i++) { + tensorflow::Tensor* output = op_context.mutable_output(i); + const tensorflow::XlaExpression* expr = + tensorflow::XlaExpression::CastExpressionFromTensor(*output); + + if (expr->kind() != tensorflow::XlaExpression::Kind::kXlaOp && + expr->kind() != tensorflow::XlaExpression::Kind::kConstant) { + return op_->emitRemark(absl::StrCat( + "expects XlaExpression of kind kXlaOp or kConstant in compiled " + "output index ", + i)); + } + } + return success(); +} + +// XLA computations can only return a single value, but TF ops can return +// multiple values. We get around this by returning a tuple as an XLA op. We +// then unpack it here to return the multiple values instead. +mlir::LogicalResult Tf2XlaRewriter::UnpackTupleResults( + stablehlo::TupleOp tuple_result, llvm::SmallVector& outputs) { + if (tuple_result->getNumOperands() != op_->getNumResults()) { + return op_->emitRemark() << "Translated TF2XLA tuple has different " + "number of results than original op"; + } + + for (int i = 0; i < tuple_result->getNumOperands(); i++) { + outputs.push_back(tuple_result->getOperand(i)); + } + + tuple_result.getOperation()->erase(); + return success(); +} + +mlir::LogicalResult Tf2XlaRewriter::GetKernelOutputs( + tensorflow::OpKernelContext& op_context, stablehlo::TupleOp tuple_results, + llvm::SmallVector& outputs) { + outputs.reserve(op_->getNumResults()); + + return UnpackTupleResults(tuple_results, outputs); +} + +tensorflow::XlaExpression Tf2XlaRewriter::GetExprForOperand( + Value operand, Operation* op, int64_t operand_index) { + ElementsAttr const_attr; + auto defining_op = operand.getDefiningOp(); + + ::xla::XlaOp xla_op = xla::Parameter(&xla_builder_, operand_index, + xla::TypeToShape(operand.getType()), + std::to_string(operand_index)); + + if (defining_op && matchPattern(defining_op, m_Constant(&const_attr))) { + tensorflow::Tensor tensor; + auto status = tensorflow::ConvertToTensor(const_attr, &tensor); + if (!status.ok()) { + op->emitRemark() << "skipping legalization due to failed const conversion" + << status.ToString(); + return tensorflow::XlaExpression::Invalid(); + } + + return tensorflow::XlaExpression::Constant(tensor); + } + + tensorflow::DataType dtype; + auto status = tensorflow::ConvertToDataType(operand.getType(), &dtype); + if (!status.ok()) { + op->emitRemark() << "skipping legalization due to " << status.ToString(); + return tensorflow::XlaExpression::Invalid(); + } + return tensorflow::XlaExpression::XlaOp(xla_op, dtype); +} + +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h new file mode 100644 index 0000000000000000000000000000000000000000..9c28adb72f1f3185b7995392c83d629586f9f265 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h @@ -0,0 +1,127 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TF2XLA_REWRITER_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TF2XLA_REWRITER_H_ + +#include +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/op_or_arg_name_mapper.h" +#include "tensorflow/compiler/tf2xla/xla_context.h" +#include "tensorflow/compiler/tf2xla/xla_expression.h" +#include "xla/hlo/builder/xla_builder.h" +#include "xla/hlo/builder/xla_computation.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/framework/op_kernel.h" + +namespace mlir { +namespace hlo { +class Tf2XlaRewriterTestPeer; + +class Tf2XlaRewriter { + public: + static mlir::LogicalResult RewriteOp(mlir::Operation* op, + mlir::PatternRewriter& rewriter, + const std::string& device_type); + + private: + friend class Tf2XlaRewriterTestPeer; + + Tf2XlaRewriter(mlir::Operation* op, mlir::PatternRewriter& rewriter, + const std::string& device_type); + + ~Tf2XlaRewriter(); + + // Compiles the given Operation with XlaBuilder and imports the generated HLO + // via the HLO -> MHLO importer. + absl::StatusOr CompileWithHloImporter( + tensorflow::OpKernelContext& op_context); + + // Import the given XlaComputation into the parent module. Returns the given + // generated function. + absl::StatusOr ImportXlaComputation( + xla::XlaComputation& computation); + + // Prepares OpKernelContext params common to all the ops. + // Emits an error on failure. + mlir::LogicalResult PrepareParams(); + + // Given the required_consts, it will fill the 3 output vectors with + // their respective data. + // Expressions: Output XLA expressions as required by the compiled kernel. + // Tensors: Vector of tensors that back the TensorValue inputs + // Inputs: Vector of inputs that are backed by tensors. + mlir::LogicalResult PrepareKernelInputs( + const llvm::SmallDenseSet& required_consts, + std::vector& expressions, + std::vector& tensors, + std::vector& inputs); + + mlir::LogicalResult VerifyOpResults(tensorflow::OpKernelContext& op_context); + mlir::LogicalResult GetKernelOutputs(tensorflow::OpKernelContext& op_context, + stablehlo::TupleOp tuple_results, + llvm::SmallVector& outputs); + + // Given a translated function with a single return value, unpack the tuple + // results. + mlir::LogicalResult UnpackTupleResults(stablehlo::TupleOp tuple_result, + llvm::SmallVector& outputs); + + // Tries to legalize the specified TensorFlow op, if supported. + // + // Emits an error and returns failure if an error is encountered during + // conversion. Note that success return value doesn't mean successful + // legalization. + mlir::LogicalResult LegalizeOp(); + + // Converts the given operand to expression of kind kConstant or kXlaOp. + // Emits a remark and returns expression of kind kInvalid on failure. + tensorflow::XlaExpression GetExprForOperand(mlir::Value operand, + mlir::Operation* op, + int64_t operand_index); + + mlir::Operation* op_; + std::string device_type_; + + mlir::PatternRewriter& rewriter_; + std::unique_ptr name_mapper_; + + tensorflow::XlaContext* context_; // Ref-counted. + + std::unique_ptr device_mgr_; + tensorflow::Device* device_; // Owned by device_mgr_; + std::unique_ptr step_container_; + std::unique_ptr flib_def_; + std::unique_ptr pflr_; + tensorflow::OpKernelContext::Params params_; + + xla::XlaBuilder xla_builder_; +}; + +} // namespace hlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_TF2XLA_REWRITER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..6d59a951e6b84d01eb11446373ab66550b293a29 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter_test.cc @@ -0,0 +1,366 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tf2xla/transforms/tf2xla_rewriter.h" + +#include +#include +#include + +#include +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/hlo/builder/xla_builder.h" +#include "xla/hlo/builder/xla_computation.h" +#include "xla/shape_util.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/status.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla_data.pb.h" + +namespace mlir { +namespace hlo { + +using ::mlir::LogicalResult; +using ::mlir::ModuleOp; +using ::mlir::OpBuilder; +using ::mlir::Operation; +using ::mlir::func::FuncOp; +using ::tsl::Status; +using ::tsl::StatusOr; +using ::xla::ReplicaGroup; +using ::xla::ShapeUtil; +using ::xla::XlaBuilder; +using ::xla::XlaComputation; +using ::xla::XlaOp; + +static constexpr char kMlirModuleStr[] = R"( +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1442 : i32}} { + func.func @main(%arg0: tensor<3xi64> {tf._user_specified_name = "resource", tf.aliasing_output = 3 : i64}) -> () attributes {tf.entry_function = {control_outputs = "stateful_normal/RngReadAndSkip,stateful_uniform/RngReadAndSkip,stateful_uniform_full_int/RngReadAndSkip", inputs = "stateful_normal_rngreadandskip_resource", outputs = "identity_RetVal,identity_1_RetVal,identity_2_RetVal"}} { + %0:3 = "tf.Unpack"(%arg0) {axis = 0 : i64} : (tensor<3xi64>) -> (tensor, tensor, tensor) + return + } +})"; + +XlaComputation GetTestXlaComputation() { + XlaBuilder xla_builder("test"); + auto param = + Parameter(&xla_builder, 0, ShapeUtil::MakeScalarShape(xla::F32), "a"); + + XlaOp add = xla::Add(param, xla::ConstantR0(&xla_builder, 2.0)); + + std::vector tuple_values; + tuple_values.push_back(add); + + xla::Tuple(&xla_builder, tuple_values); + return xla_builder.Build().value(); +} + +class EmptyPatternRewriter : public mlir::PatternRewriter { + public: + explicit EmptyPatternRewriter(const OpBuilder& other_builder) + : mlir::PatternRewriter(other_builder) {} + ~EmptyPatternRewriter() override = default; +}; + +class Tf2XlaRewriterTestPeer { + public: + explicit Tf2XlaRewriterTestPeer() = delete; + explicit Tf2XlaRewriterTestPeer(mlir::Operation* op) + : op_builder_(op), + empty_rewriter_(op_builder_), + tf2xla_rewriter_(op, empty_rewriter_, + /*device_type=*/"XLA_CPU_JIT") {} + + absl::StatusOr ImportXlaComputationIntoModule( + XlaComputation& computation) { + return tf2xla_rewriter_.ImportXlaComputation(computation); + } + + private: + OpBuilder op_builder_; + EmptyPatternRewriter empty_rewriter_; + Tf2XlaRewriter tf2xla_rewriter_; +}; + +// This should only have unit tests. End to end tests should be done with +// FILECHECK and MLIR tests. +class Tf2XlaRewriterTest : public ::testing::Test { + public: + void SetUp() override { + tensorflow::XlaOpRegistry::RegisterCompilationKernels(); + } + + Status CreateMlirModule(std::string module_string = kMlirModuleStr) { + TF_ASSIGN_OR_RETURN( + module_, hlo::test::GetMlirModuleFromString(module_string, &context_)); + + context_.loadAllAvailableDialects(); + return absl::OkStatus(); + } + + Status LegalizeSingleOp(Operation& op) { + SourceMgrDiagnosticHandler sourceMgrHandler(source_manager_, &context_); + + OpBuilder op_builder(&op); + EmptyPatternRewriter pattern_rewriter(op_builder); + + LogicalResult result = + Tf2XlaRewriter::RewriteOp(&op, pattern_rewriter, + /*device_type=*/"XLA_CPU_JIT"); + if (!result.succeeded()) { + return absl::InternalError("Failed to rewrite op"); + } + + return absl::OkStatus(); + } + + Status LegalizeModule(std::string module_string = kMlirModuleStr) { + TF_EXPECT_OK(CreateMlirModule(module_string)); + FuncOp main = module_->lookupSymbol("main"); + if (!main) { + return absl::InvalidArgumentError("Could not find a main function"); + } + + WalkResult walk_result = main.walk([&](Operation* op) { + if (op->getDialect()->getNamespace() != + TF::TensorFlowDialect::getDialectNamespace()) { + return WalkResult::advance(); + } + + if (!LegalizeSingleOp(*op).ok()) { + return WalkResult::interrupt(); + } + + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + return absl::InternalError("Could not legalize all ops"); + } + + return absl::OkStatus(); + } + + mlir::func::FuncOp GetMainFunc() { + func::FuncOp main_func = module_->lookupSymbol("main"); + EXPECT_TRUE(main_func); + + return main_func; + } + + mlir::Operation& GetFirstOpFromMain() { + mlir::func::FuncOp main_func = GetMainFunc(); + return main_func.getBody().front().front(); + } + + absl::StatusOr ImportXlaComputationIntoModule( + XlaComputation& computation) { + SourceMgrDiagnosticHandler sourceMgrHandler(source_manager_, &context_); + + mlir::Operation& first_op = GetFirstOpFromMain(); + + Tf2XlaRewriterTestPeer test_peer(&first_op); + return test_peer.ImportXlaComputationIntoModule(computation); + } + + protected: + MLIRContext context_; + OwningOpRef module_; + llvm::SourceMgr source_manager_; +}; + +TEST_F(Tf2XlaRewriterTest, LegalizesOpWithTf2xlaHloImporter) { + TF_EXPECT_OK(LegalizeModule()); + + int num_tuple_ops = 0; + module_->walk( + [&num_tuple_ops](stablehlo::TupleOp tuple_op) { num_tuple_ops += 1; }); + + EXPECT_EQ(num_tuple_ops, 0); +} + +TEST_F(Tf2XlaRewriterTest, ImportsXlaComputationIntoModule) { + TF_ASSERT_OK(CreateMlirModule()); + + XlaComputation computation = GetTestXlaComputation(); + + TF_ASSERT_OK_AND_ASSIGN(stablehlo::TupleOp root_tuple, + ImportXlaComputationIntoModule(computation)); + + ModuleOp parent_module = + root_tuple.getOperation()->getParentOfType(); + EXPECT_EQ(parent_module, *module_); +} + +TEST_F(Tf2XlaRewriterTest, FailsWithoutRootTuple) { + TF_ASSERT_OK(CreateMlirModule()); + + XlaBuilder xla_builder("test_fail"); + xla::Add(xla::ConstantR0(&xla_builder, 1.0), + xla::ConstantR0(&xla_builder, 2.0)); + XlaComputation bad_computation = xla_builder.Build().value(); + + EXPECT_FALSE(ImportXlaComputationIntoModule(bad_computation).ok()); +} + +TEST_F(Tf2XlaRewriterTest, ImportsSingleComputation) { + XlaBuilder builder("test_builder"); + XlaComputation to_apply; + { + auto sub_builder = builder.CreateSubBuilder("add"); + auto arg0 = Parameter(sub_builder.get(), 0, + ShapeUtil::MakeScalarShape(xla::F32), "x"); + auto arg1 = Parameter(sub_builder.get(), 1, + ShapeUtil::MakeScalarShape(xla::F32), "y"); + Add(arg0, arg1); + TF_ASSERT_OK_AND_ASSIGN(to_apply, sub_builder->Build()); + } + auto x = Parameter(&builder, 0, ShapeUtil::MakeShape(xla::F32, {4, 16}), "x"); + ReplicaGroup group; + group.add_replica_ids(0); + group.add_replica_ids(1); + XlaOp reduce_scatter = + ReduceScatter(x, to_apply, /*scatter_dimension=*/1, /*shard_count=*/2, + /*replica_groups=*/{group}); + + std::vector tuple_values; + tuple_values.push_back(reduce_scatter); + xla::Tuple(&builder, tuple_values); + + TF_ASSERT_OK_AND_ASSIGN(XlaComputation computation, builder.Build()); + EXPECT_EQ(computation.proto().computations_size(), 2); + + TF_ASSERT_OK(CreateMlirModule()); + TF_ASSERT_OK_AND_ASSIGN(stablehlo::TupleOp root_tuple, + ImportXlaComputationIntoModule(computation)); + EXPECT_TRUE(root_tuple); + + int num_func_ops = 0; + module_->walk([&num_func_ops](func::FuncOp func_op) { num_func_ops++; }); + + // Ensure that only a single computation was imported. + EXPECT_EQ(num_func_ops, 1); +} + +TEST_F(Tf2XlaRewriterTest, InsertsConstantParameters) { + static constexpr char kModuleWithConstParam[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1442 : i32}} { + func.func @main(%arg0: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.Const"() {value = dense<1.42> : tensor<2xf32>} : () -> tensor<2xf32> + %1 = "tf.Atan2"(%arg0, %0) : (tensor<2xf32>, tensor<2xf32>) -> tensor<2xf32> + func.return %0 : tensor<2xf32> + } + })"; + + TF_ASSERT_OK(LegalizeModule(kModuleWithConstParam)); +} + +TEST_F(Tf2XlaRewriterTest, DoesntEnforceCompileTimeConstantCheck) { + static constexpr char kModuleWithNonConstParam[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1610 : i32}} { + func.func @main(%arg0: tensor<3x3x10xbf16>, %arg1: tensor<3xi32>) -> tensor<1x?x4xbf16> attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2", outputs = "_retval0"}} { + %cst = "tf.Const"() {value = dense<[1, -1, 4]> : tensor<3xi32>} : () -> tensor<3xi32> + %0 = "tf.Slice"(%arg0, %arg1, %cst) {_XlaHasReferenceVars = false, _xla_inferred_shapes = [#tf_type.shape<1x?x4>], device = "/job:localhost/replica:0/task:0/device:TPU:0"} : (tensor<3x3x10xbf16>, tensor<3xi32>, tensor<3xi32>) -> tensor<1x?x4xbf16> + return %0 : tensor<1x?x4xbf16> + } + })"; + + TF_ASSERT_OK(LegalizeModule(kModuleWithNonConstParam)); +} + +TEST_F(Tf2XlaRewriterTest, CreatesDefaultValues) { + // If a TF op has default value attributes and the mlir is missing them then + // the LegalizeOp should insert the default values when converting the dialect + // op to a node def. + // TF.RandomUniform would fail without the seeds being set if they were not + // automatically inserted with the default values. + static constexpr char kModuleWithOpWithoutValuesThatShouldBeDefaulted[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1610 : i32}} { + func.func @main() -> tensor<1x2x3x4xf32> attributes {allow_soft_placement = false, tf.entry_function = {control_outputs = "", inputs = "_arg0,_arg1,_arg2", outputs = "_retval0"}} { + %cst = "tf.Const"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32>} : () -> tensor<4xi32> + %0 = "tf.RandomUniform"(%cst) : (tensor<4xi32>) -> tensor<1x2x3x4xf32> + return %0 : tensor<1x2x3x4xf32> + } + })"; + + TF_ASSERT_OK(LegalizeModule(kModuleWithOpWithoutValuesThatShouldBeDefaulted)); +} + +TEST_F(Tf2XlaRewriterTest, OpWithLocationDoesntBreakNodeDefName) { + // A named location 'Name(Source)' causes the GetNameFromLoc method to append + // all the other locations to the name with a ';' separator. This test ensures + // that the name used for the NodeDef does not contain that invalid character. + static constexpr char kModuleWithOpWithoutValuesThatShouldBeDefaulted[] = + R"mlir( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1610 : i32}} { + func.func @main(%arg0: tensor<2xf32>) -> tensor<2xf32> { + %0 = "tf.Exp"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> loc(fused["exp"("exp"), "exp"]) + func.return %0 : tensor<2xf32> + } + })mlir"; + + TF_ASSERT_OK(LegalizeModule(kModuleWithOpWithoutValuesThatShouldBeDefaulted)); +} + +TEST_F(Tf2XlaRewriterTest, ErrorsWithInvalidNumberOfParametersToArgs) { + XlaBuilder builder("test_builder"); + XlaComputation to_apply; + { + auto sub_builder = builder.CreateSubBuilder("add"); + auto arg0 = Parameter(sub_builder.get(), 0, + ShapeUtil::MakeScalarShape(xla::F32), "x"); + auto arg1 = Parameter(sub_builder.get(), 1, + ShapeUtil::MakeScalarShape(xla::F32), "y"); + Add(arg0, arg1); + TF_ASSERT_OK_AND_ASSIGN(to_apply, sub_builder->Build()); + } + auto a = Parameter(&builder, 0, ShapeUtil::MakeScalarShape(xla::F32), "a"); + auto b = Parameter(&builder, 1, ShapeUtil::MakeScalarShape(xla::F32), "b"); + XlaOp call_op = xla::Call(&builder, to_apply, {a, b}); + + std::vector tuple_values; + tuple_values.push_back(call_op); + xla::Tuple(&builder, tuple_values); + + TF_ASSERT_OK_AND_ASSIGN(XlaComputation computation, builder.Build()); + EXPECT_EQ(computation.proto().computations_size(), 2); + + TF_ASSERT_OK(CreateMlirModule()); + absl::StatusOr status_or_tuple_op = + ImportXlaComputationIntoModule(computation); + EXPECT_FALSE(status_or_tuple_op.ok()); +} + +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf_xla_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf_xla_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..813da0ecaad682b652c13aa67b49f2c2f3fcd2ee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tf_xla_passes.td @@ -0,0 +1,47 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + +def LegalizeTFCommunicationPass : Pass<"xla-legalize-tf-communication", "ModuleOp"> { + let summary = "Legalize TF/XLA communication ops (TensorFlow dialect) to the HLO " + "dialect"; + let dependentDialects = ["mhlo::MhloDialect", "sparse_tensor::SparseTensorDialect"]; + let constructor = "::mlir::mhlo::CreateLegalizeTFCommunicationPass()"; + let description = [{ + A pass that legalizes TF/XLA communication ops, propagates their respective + tokens (for ordering), and rewrites their respective functions and control + flow ops when necessary. + + For example, given the program + + ```mlir + func @send_to_host(%arg0: tensor) { + "tf.XlaSendToHost"(%arg0) {key = "send_key"} : (tensor) -> () + return + } + ``` + + This might be legalized like this: + + ```mlir + func @send_to_host(%arg0: tensor) { + %0 = "mhlo.create_token"() : () -> !mhlo.token + %1 = "mhlo.send"(%arg0, %0) {...}} : (tensor, !mhlo.token) -> !mhlo.token + return + } + ''' + }]; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tfxla_device_specific_transforms.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tfxla_device_specific_transforms.cc new file mode 100644 index 0000000000000000000000000000000000000000..018dc1459c0df25da07a445d7d917e0ddf04fa7b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/tfxla_device_specific_transforms.cc @@ -0,0 +1,94 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/tf2xla/kernels/rng_converter_utils.h" +#include "xla/xla_data.pb.h" + +namespace mlir { +namespace mhlo { + +namespace { + +#define GEN_PASS_DEF_TFXLADEVICESPECIFICTRANSFORMS +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +class TFXLADeviceSpecificTransforms + : public impl::TFXLADeviceSpecificTransformsBase< + TFXLADeviceSpecificTransforms> { + public: + explicit TFXLADeviceSpecificTransforms(std::optional device_type) { + if (device_type.has_value()) { + device_type_ = device_type.value().str(); + } + } + void runOnOperation() override; + + private: + LogicalResult ConvertGetAlgOp(TF::StatelessRandomGetAlgOp get_alg_op); +}; + +LogicalResult TFXLADeviceSpecificTransforms::ConvertGetAlgOp( + TF::StatelessRandomGetAlgOp get_alg_op) { + if (!device_type_.hasValue()) return failure(); + + xla::RandomAlgorithm xla_rng = + tensorflow::DefaultRngAlgForDeviceType(device_type_); + tensorflow::Algorithm tensorflow_rng = + tensorflow::ToTensorflowAlgorithm(xla_rng); + + OpBuilder opbuilder(get_alg_op); + + auto tf_const = + TF::ConstOp::create(opbuilder, get_alg_op->getLoc(), + opbuilder.getI32IntegerAttr((int)tensorflow_rng)); + + get_alg_op->replaceAllUsesWith(tf_const); + get_alg_op->erase(); + return success(); +} + +void TFXLADeviceSpecificTransforms::runOnOperation() { + if (!device_type_.hasValue()) return; + auto func_op = getOperation(); + + auto walk_result = func_op->walk([&](TF::StatelessRandomGetAlgOp op) { + if (failed(ConvertGetAlgOp(op))) { + op->emitOpError( + "Could not convert and remove Device specific information"); + return WalkResult::interrupt(); + } + + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) signalPassFailure(); +} + +} // namespace + +std::unique_ptr> +CreateTFXLADeviceSpecificTransformsPass(std::optional device_type) { + return std::make_unique(device_type); +} + +} // namespace mhlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..88536233c8e3d97615f4acc82153282d3a421b34 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.cc @@ -0,0 +1,49 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/utils.h" + +#include + +#include "xla/mlir_hlo/utils/hlo_utils.h" + +namespace mlir { +namespace mhlo { + +ConstantOp GetScalarConstOfType(Type ty, Location loc, int64_t raw_value, + OpBuilder* builder) { + return ConstantOp::create(*builder, loc, hlo::getScalarOfType(ty, raw_value)); +} + +ConstantOp GetScalarNegZeroOfType(Type ty, Location loc, OpBuilder* builder) { + return ConstantOp::create(*builder, loc, hlo::getScalarNegZeroOfType(ty)); +} + +DenseIntElementsAttr GetI64ElementsAttr(ArrayAttr attr) { + RankedTensorType ty = + RankedTensorType::get(static_cast(attr.size()), + IntegerType::get(attr.getContext(), 64)); + return DenseIntElementsAttr::get(ty, attr.getValue()); +} + +DenseIntElementsAttr GetI64ElementsAttr(ArrayRef values, + Builder* builder) { + RankedTensorType ty = RankedTensorType::get( + {static_cast(values.size())}, builder->getIntegerType(64)); + return DenseIntElementsAttr::get(ty, values); +} + +} // namespace mhlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..ad7acf4be3c62f880a34ced5f07e9bb5d89630c0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/utils.h @@ -0,0 +1,62 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_UTILS_H_ + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +namespace mlir { +namespace mhlo { + +// Builds body for reduce op by using the template binary op as the +// reducer op. +template +void BuildReduceBody(Type element_type, Region* body, OpBuilder* builder) { + OpBuilder::InsertionGuard guard(*builder); + Block* block = builder->createBlock(body); + + // Block arguments are scalars of the given element type. + Type type = RankedTensorType::get(/*shape=*/{}, element_type); + Location loc = body->getLoc(); + block->addArguments({type, type}, SmallVector(2, loc)); + + auto reducer = + Op::create(*builder, loc, block->getArgument(0), block->getArgument(1)); + ReturnOp::create(*builder, loc, reducer.getResult()); +} + +ConstantOp GetScalarConstOfType(Type ty, Location loc, int64_t raw_value, + OpBuilder* builder); + +ConstantOp GetScalarNegZeroOfType(Type ty, Location loc, OpBuilder* builder); + +// Converts an ArrayAttr to a 1D 64-bit dense elements attribute. +DenseIntElementsAttr GetI64ElementsAttr(ArrayAttr attr); +DenseIntElementsAttr GetI64ElementsAttr(llvm::ArrayRef values, + Builder* builder); + +} // namespace mhlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization.cc new file mode 100644 index 0000000000000000000000000000000000000000..a1740fd1b69f5c3373f777abb6d1a596f96ee917 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization.cc @@ -0,0 +1,180 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/IR/BuiltinOps.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/Base.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/platform/errors.h" + +namespace mlir { +namespace mhlo { + +namespace { + +#define GEN_PASS_DEF_VERIFYTFXLALEGALIZATION +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +auto* mlir_failed_legalization_op_count = + tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/" + "mlir_second_phase_failed_legalization_op_count", + "Counts which op fails to legalize", "op_name"); + +auto* mlir_non_static_op_count = tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/" + "mlir_second_phase_non_static_op_count", + "Counts which ops do not have static results", "op_name"); + +auto* mlir_non_static_op_skip_count = tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/" + "mlir_second_phase_non_static_op_skip_count", + "Counts skipped ops which do not have static results", "op_name"); + +static const char* kMustBeConstantError = + "must have compile-time constant inputs and outputs.\n\n" + "XLA compilation requires that operator arguments that represent shapes or " + "dimensions be evaluated to concrete values at compile time. This error " + "means that a shape or dimension argument could not be evaluated at " + "compile time, usually because the value of the argument depends on a " + "parameter to the computation, on a variable, or on a stateful operation " + "such as a random number generator."; + +// TODO(b/282188914) remove the operations to skip once tests are fixed. +static const DenseSet* operations_to_skip = + new DenseSet{mlir::TypeID::get()}; + +class VerifyTFXLALegalization + : public impl::VerifyTFXLALegalizationBase { + public: + explicit VerifyTFXLALegalization(bool legalize_chlo) { + legalize_chlo_ = legalize_chlo; + } + + void runOnOperation() override; +}; + +static void IncrementCounterFor(tensorflow::monitoring::Counter<1>* counter, + Operation* op) { + counter->GetCell(op->getName().getStringRef().str())->IncrementBy(1); +} + +bool HasBounds(RankedTensorType type) { + auto bounds = hlo::encodingToBounds(type.getEncoding()); + return !bounds.empty(); +} + +bool HasStaticShapeOrBounded(Value val) { + auto type = val.getType(); + if (mlir::isa(type)) { + return false; + } + if (mlir::isa(type)) { + auto ranked_tensor = mlir::dyn_cast(type); + if (ranked_tensor.hasStaticShape()) { + return true; + } + return HasBounds(ranked_tensor); + } + return true; +} + +bool EmitMustBeConstantError(Operation* op) { + if (operations_to_skip->contains(op->getRegisteredInfo()->getTypeID())) { + IncrementCounterFor(mlir_non_static_op_skip_count, op); + return true; + } + emitError(op->getLoc()) << absl::StrCat( + "Node `", op->getName().getStringRef().str(), "` ", kMustBeConstantError); + return false; +} + +bool IsStaticOperation(Operation* op) { + for (auto o : op->getResults()) { + if (!HasStaticShapeOrBounded(o)) { + return EmitMustBeConstantError(op); + } + } + return true; +} + +bool IsMhloAndStatic(Operation* op) { + if (!llvm::isa(op->getDialect())) { + // Skip this op if it isn't an mhlo op. + return true; + } + return IsStaticOperation(op); +} + +bool IsDefaultConversionLegal( + Operation* op, const ConversionTarget& default_conversion_target) { + if (!default_conversion_target.isLegal(op)) { + emitError(op->getLoc()) << "Could not legalize op: " << op->getName(); + return false; + } + return true; +} + +void VerifyTFXLALegalization::runOnOperation() { + Operation* func_op = getOperation(); + ConversionTarget default_conversion_target = + hlo::GetDefaultLegalConversionTargets(getContext(), legalize_chlo_); + + bool has_invalid_ops = false; + func_op->walk([&](Operation* op) { + if (!IsMhloAndStatic(op)) { + has_invalid_ops = true; + IncrementCounterFor(mlir_non_static_op_count, op); + return WalkResult::interrupt(); + } + if (!IsDefaultConversionLegal(op, default_conversion_target)) { + has_invalid_ops = true; + IncrementCounterFor(mlir_failed_legalization_op_count, op); + } + return WalkResult::advance(); + }); + + if (has_invalid_ops) signalPassFailure(); +} + +} // namespace + +} // namespace mhlo + +namespace hlo { + +std::unique_ptr> +CreateVerifyTFXLALegalizationPass(bool legalize_chlo) { + return std::make_unique(legalize_chlo); +} +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..5b8c6837ea8a9ad3710a2c406faf20308d2b437b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/verify_tfxla_legalization_test.cc @@ -0,0 +1,180 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/test_utils.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" + +namespace tensorflow { +namespace { + +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::mlir::hlo::test::GetMlirModuleFromString; +using ::tensorflow::monitoring::testing::CellReader; + +static constexpr char kFailedLegalizationStreamz[] = + "/tensorflow/core/tf2xla/mlir_second_phase_failed_legalization_op_count"; +static constexpr char kNonStaticOpStreamz[] = + "/tensorflow/core/tf2xla/mlir_second_phase_non_static_op_count"; +static constexpr char kNonStaticOpSkipStreamz[] = + "/tensorflow/core/tf2xla/mlir_second_phase_non_static_op_skip_count"; + +class VerifyTfxlaLegalizationTest : public ::testing::Test { + protected: + void CreateModule(const char* module_string) { + TF_ASSERT_OK_AND_ASSIGN(module_, + GetMlirModuleFromString(module_string, &context_)); + + pm_ = std::make_unique(&context_); + pm_->addNestedPass( + mlir::hlo::CreateVerifyTFXLALegalizationPass(/*legalize_chlo=*/false)); + } + mlir::LogicalResult Run() { return pm_->run(module_.get()); } + + private: + MLIRContext context_; + OwningOpRef module_; + std::unique_ptr pm_; +}; + +TEST_F(VerifyTfxlaLegalizationTest, RecordsStreamzFailedVerification) { + // Using a string constant here instead of testdata to make this compatible + // with open source. + static constexpr char kMlirModuleStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.BadValue"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + CellReader error(kFailedLegalizationStreamz); + CreateModule(kMlirModuleStr); + + auto result = Run(); + + EXPECT_TRUE(result.failed()); + EXPECT_EQ(error.Delta("tf.BadValue"), 1); +} + +TEST_F(VerifyTfxlaLegalizationTest, ErrorsNonStaticInputs) { + // Using a string constant here instead of testdata to make this compatible + // with open source. + static constexpr char kNonStaticFailure[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1504 : i32}} { + func.func @main() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "i,j", outputs = "identity_RetVal"}} { + %0 = mhlo.constant dense<1.000000e+00> : tensor + %1 = mhlo.convert %0 : (tensor) -> tensor + %2 = mhlo.reshape %1 : (tensor) -> tensor<1xi64> + %3 = "mhlo.dynamic_iota"(%2) {iota_dimension = 0 : i64} : (tensor<1xi64>) -> tensor + %4 = mhlo.multiply %3, %3 : tensor + return %4 : tensor + } + })"; + CellReader legal_error(kFailedLegalizationStreamz); + CellReader static_error(kNonStaticOpStreamz); + CreateModule(kNonStaticFailure); + + auto result = Run(); + + EXPECT_TRUE(result.failed()); + EXPECT_EQ(legal_error.Delta("mhlo.dynamic_iota"), 0); + EXPECT_EQ(static_error.Delta("mhlo.dynamic_iota"), 1); +} + +TEST_F(VerifyTfxlaLegalizationTest, SkipsSpecificNonStaticInputs) { + // Using a string constant here instead of testdata to make this compatible + // with open source. + static constexpr char kNonStaticFailure[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1504 : i32}} { + func.func @main(%a : tensor<5x14x1xf32>, %b : tensor<1x14x32xf32>) -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "i,j", outputs = "identity_RetVal"}} { + %c = "mhlo.einsum"(%a, %b) {einsum_config = "bji,bjk->bik"} : (tensor<5x14x1xf32>, tensor<1x14x32xf32>) -> tensor + return %c : tensor + } + })"; + CellReader static_error(kNonStaticOpStreamz); + CellReader skipped(kNonStaticOpSkipStreamz); + CreateModule(kNonStaticFailure); + + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + EXPECT_EQ(static_error.Delta("mhlo.einsum"), 0); + EXPECT_EQ(skipped.Delta("mhlo.einsum"), 1); +} + +TEST_F(VerifyTfxlaLegalizationTest, SkipsNonStaticInputsWithBounds) { + // Using a string constant here instead of testdata to make this compatible + // with open source. + static constexpr char kNonStaticWithBoundsSuccess[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 1504 : i32}} { + func.func @main() -> tensor> attributes {tf.entry_function = {control_outputs = "", inputs = "i,j", outputs = "identity_RetVal"}} { + %0 = mhlo.constant dense<1.000000e+00> : tensor + %1 = mhlo.convert %0 : (tensor) -> tensor + %2 = mhlo.reshape %1 : (tensor) -> tensor<1xi64> + %3 = "mhlo.dynamic_iota"(%2) {iota_dimension = 0 : i64} : (tensor<1xi64>) -> tensor> + %4 = mhlo.multiply %3, %3 : tensor> + return %4 : tensor> + } + })"; + CellReader legal_error(kFailedLegalizationStreamz); + CellReader static_error(kNonStaticOpStreamz); + CreateModule(kNonStaticWithBoundsSuccess); + + auto result = Run(); + + EXPECT_TRUE(result.succeeded()); + EXPECT_EQ(legal_error.Delta("mhlo.multiply"), 0); + EXPECT_EQ(static_error.Delta("mhlo.multiply"), 0); +} + +TEST_F(VerifyTfxlaLegalizationTest, RecordsMultipleFailures) { + // Using a string constant here instead of testdata to make this compatible + // with open source. + static constexpr char kMultipleFailures[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.BadValue"() {value = dense<1000> : tensor<1xi32>} : () -> tensor<1xi32> + %1 = "tf.AlsoBad"() {value = dense<10> : tensor<1xi32>} : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + CellReader error(kFailedLegalizationStreamz); + CreateModule(kMultipleFailures); + + auto result = Run(); + + EXPECT_TRUE(result.failed()); + EXPECT_EQ(error.Delta("tf.BadValue"), 1); + EXPECT_EQ(error.Delta("tf.AlsoBad"), 1); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.cc new file mode 100644 index 0000000000000000000000000000000000000000..dadc582e2e4c1cb398ae04905076216a05cd6bfd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.cc @@ -0,0 +1,58 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" + +namespace mlir { +namespace hlo { + +ConversionTarget GetDefaultLegalConversionTargets(MLIRContext& mlir_context, + bool legalize_chlo) { + ConversionTarget target(mlir_context); + + if (legalize_chlo) { + target.addIllegalDialect(); + target.addIllegalDialect(); + } else { + target.addLegalDialect(); + } + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalOp(); + + // These ops are legalized in LegalizeTFCommunication after this and that pass + // only operates on MHLO control flow ops. + target.addLegalOp(); + + return target; +} + +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h new file mode 100644 index 0000000000000000000000000000000000000000..ae4561b0706e2710ca03a46d0f7aa4a7ce5da699 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h @@ -0,0 +1,34 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_XLA_LEGALIZE_TARGETS_H_ +#define TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_XLA_LEGALIZE_TARGETS_H_ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project + +namespace mlir { +namespace hlo { + +// Returns a ConversionTarget that includes default legalized MLIR dialects +// for conversion to XLA. +// If legalize_chlo is true, the resulting conversion target cannot have CHLO. +mlir::ConversionTarget GetDefaultLegalConversionTargets( + MLIRContext& mlir_context, bool legalize_chlo); + +} // namespace hlo +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TF2XLA_TRANSFORMS_XLA_LEGALIZE_TARGETS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a86069497a061edc401bbc463fb70abcdd1c01a6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets_test.cc @@ -0,0 +1,95 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h" + +#include +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace hlo { +namespace { + +mlir::DialectRegistry GetDefaultDialectRegistry() { + mlir::DialectRegistry registry; + + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + + return registry; +} + +class XlaLegalizeTargetsTest : public testing::Test { + public: + XlaLegalizeTargetsTest() + : context_(GetDefaultDialectRegistry()), + module_(mlir::ModuleOp::create(mlir::UnknownLoc::get(&context_))), + builder_(&module_->getBodyRegion()) { + context_.loadAllAvailableDialects(); + } + + protected: + mlir::MLIRContext context_; + mlir::OwningOpRef module_; + mlir::OpBuilder builder_; +}; + +TEST_F(XlaLegalizeTargetsTest, CreatesConversionTargets) { + auto const_int = mlir::arith::ConstantIntOp::create( + builder_, builder_.getUnknownLoc(), builder_.getI32Type(), /*value=*/10); + + ConversionTarget target = + GetDefaultLegalConversionTargets(context_, /*legalize_chlo=*/false); + EXPECT_TRUE(target.isLegal(const_int)); +} + +TEST_F(XlaLegalizeTargetsTest, AllowsCHLODialect) { + auto const_int = chlo::ConstantOp::create(builder_, builder_.getUnknownLoc(), + builder_.getI32TensorAttr({42})); + + ConversionTarget target = + GetDefaultLegalConversionTargets(context_, /*legalize_chlo=*/true); + + EXPECT_TRUE(target.isIllegal(const_int)); +} + +TEST_F(XlaLegalizeTargetsTest, DontAllowCHLODialect) { + auto const_int = chlo::ConstantOp::create(builder_, builder_.getUnknownLoc(), + builder_.getI32TensorAttr({42})); + + ConversionTarget target = + GetDefaultLegalConversionTargets(context_, /*legalize_chlo=*/false); + EXPECT_TRUE(target.isLegal(const_int)); +} + +} // namespace +} // namespace hlo +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..7320a39138a94301d2627b22e7d90ca222e9ddd7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf.cc @@ -0,0 +1,276 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "mhlo/transforms/rewriters.h" +#include "absl/log/log.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FormatVariadic.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/Quant.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" // from @llvm-project +#include "mlir/Dialect/Tensor/IR/Tensor.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/DebugStringHelper.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "stablehlo/dialect/ChloOps.h" // from @stablehlo +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo +#include "stablehlo/transforms/Passes.h" // from @stablehlo +#include "tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalization_op_config.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/legalize_tf_with_tf2xla_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_targets.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" // IWYU pragma: keep, dependent dialect +#include "xla/mlir_hlo/mhlo/transforms/passes.h" +#include "xla/mlir_hlo/mhlo/transforms/rewriters.h" +#include "xla/mlir_hlo/mhlo/utils/type_conversion.h" +#include "tensorflow/core/lib/monitoring/counter.h" + +namespace mlir { +namespace mhlo { +namespace { + +#define GEN_PASS_DEF_LEGALIZETF +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +auto *mlir_legalization_count = tensorflow::monitoring::Counter<1>::New( + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_count", + "Counts the attempts of legalization of ops", "op_name"); + +auto *mlir_failed_legalization_count = tensorflow::monitoring::Counter<2>::New( + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_pass_count", + "Counts the failure of legalization of ops", "op_name", "legality"); + +class LegalizeTF : public impl::LegalizeTFBase { + public: + explicit LegalizeTF(bool legalize_chlo, + std::optional tf2xla_fallback_device_type, + bool prefer_tf2xla) { + legalize_chlo_ = legalize_chlo; + prefer_tf2xla_ = prefer_tf2xla; + use_tf2xla_fallback_ = tf2xla_fallback_device_type.has_value(); + if (tf2xla_fallback_device_type.has_value()) { + device_type_ = tf2xla_fallback_device_type.value().str(); + } + } + /// Performs the lowering to XLA dialect. + void runOnOperation() override; +}; + +#define GEN_PASS_DEF_LEGALIZETFMODULEPASS +#include "tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.h.inc" + +// Patterns whose root op is in the set `include_ops` are moved from the set +// `from` to the returned set. This is used to partition patterns by op so they +// can be cleanly migrated from the old bridge to the MLIR bridge. +RewritePatternSet PatternsIncludeOps(RewritePatternSet &from) { + RewritePatternSet to(from.getContext()); + // Filter NativePatterns. + for (auto &pattern : from.getNativePatterns()) { + std::optional pat_op_name = pattern->getRootKind(); + // If the pattern does not have a specific operation, always include it, + // If the pattern is in include_ops then include it. + bool include = + !pat_op_name || hlo::IsTypeLegalizedWithMlir( + pat_op_name->getRegisteredInfo()->getTypeID()); + if (include) to.add(std::move(pattern)); + } + + // Don't filter PDLPatterns. + to.add(std::move(from.getPDLPatterns())); + + return to; +} + +std::string OperationLegalityString(Operation *op, + const ConversionTarget &target) { + auto op_name = op->getName(); + auto action = target.getOpAction(op_name); + if (!action.has_value()) { + return "Unknown"; + } + switch (action.value_or(ConversionTarget::LegalizationAction::Legal)) { + case ConversionTarget::LegalizationAction::Legal: + return "Legal"; + case ConversionTarget::LegalizationAction::Dynamic: + return "Dynamic"; + case ConversionTarget::LegalizationAction::Illegal: + return "Illegal"; + default: + return "Invalid"; + } +} + +void IncrementFailedLegalizationCount(Operation *op, + const ConversionTarget &target) { + auto op_name = op->getName(); + auto name_string = op_name.getStringRef().str(); + auto op_legality = OperationLegalityString(op, target); + + mlir_failed_legalization_count->GetCell(name_string, op_legality) + ->IncrementBy(1); +} + +mlir::LogicalResult ApplyPatterns(Operation *op, RewritePatternSet &patterns, + bool legalize_chlo) { + ConversionTarget target = + hlo::GetDefaultLegalConversionTargets(*op->getContext(), legalize_chlo); + + DenseSet unconverted_ops; + ConversionConfig config; + config.unlegalizedOps = &unconverted_ops; + auto result = applyPartialConversion(op, target, std::move(patterns), config); + if (failed(result)) { + IncrementFailedLegalizationCount(op, target); + } + for (const auto &unconverted_op : unconverted_ops) { + IncrementFailedLegalizationCount(unconverted_op, target); + } + return result; +} + +mlir::LogicalResult StablehloToMhlo(Operation *op) { + ConversionTarget target(*op->getContext()); + stablehlo::setupStablehloToHloConversionTarget(target); + + RewritePatternSet patterns(op->getContext()); + stablehlo::StablehloToHloTypeConverter shlo_converter; + stablehlo::populateStablehloToHloPatterns(&patterns, &shlo_converter, + patterns.getContext()); + stablehlo::registerFuncOpsForTypeConversion(target, patterns, shlo_converter); + + if (failed(applyPartialConversion(op, target, std::move(patterns)))) { + return op->emitError("TF2XLA failed to convert StableHLO to MHLO"); + } + return success(); +} + +/// When `tf2xla_fallback_device_type` is not `None`, also uses legalization +/// patterns from TF2XLA fallback for provided device type (see +/// legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is +/// not used. +LogicalResult legalizeTF(Operation *op, bool legalize_chlo, + std::optional tf2xla_fallback_device_type, + bool prefer_tf2xla) { + MLIRContext *context = op->getContext(); + RewritePatternSet legalize_lower_patterns(context); + // Note that the `OperationConverter` orders patterns lexicographically by: + // 1) Ascending legalization depth (i.e., minimum number of patterns + // necessary + // to arrive at conversion target). This requires relevant patterns to + // specify the list of ops generated by it which most of patterns + // implemented in C++ don't do so this comparison doesn't work in those + // cases. + // 2) Descending pattern benefit. + // 3) Op specific patterns over patterns with MatchAnyOpTypeTag. + // 4) Order of patterns in `RewritePatternSet`. + + // Add TF->HLO legalization patterns. + hlo::PopulateLegalizeTfPatterns(context, &legalize_lower_patterns); + + // Add TF->TF lowering patterns. + TF::PopulateTFLoweringBeforeHLOPatterns(context, &legalize_lower_patterns); + + if (tf2xla_fallback_device_type && prefer_tf2xla) { + VLOG(1) << "TF to XLA legalization patterns are partitioned by op into " + "either native MLIR legalization, or TF2XLA fallback " + "legalzation, with a preference toward TF2XLA."; + } else if (tf2xla_fallback_device_type) { + VLOG(1) << "TF to XLA legalization patterns include all native patterns " + "and TF2XLA fallback patterns."; + } else { + VLOG(1) << "TF to XLA legalization patterns are native patterns only."; + } + + // Set patterns to legalize_lower_patterns to check whether they should use + // MLIR or TF2XLA lowering patterns. + RewritePatternSet patterns = (tf2xla_fallback_device_type && prefer_tf2xla) + ? PatternsIncludeOps(legalize_lower_patterns) + : std::move(legalize_lower_patterns); + + Tf2XlaTypeConverter converter; + if (tf2xla_fallback_device_type) { + // Add TF->HLO legalization patterns via TF2XLA fallback. + PopulateLegalizeTfWithTf2XlaPatterns(tf2xla_fallback_device_type.value(), + patterns, context, converter, + prefer_tf2xla); + } + + // Populate with CHLO->HLO lowerings to account for TF ops legalized to + // CHLO first. + stablehlo::StablehloToHloTypeConverter hlo_converter; + stablehlo::populateStablehloToHloPatterns(&patterns, &hlo_converter, context); + if (legalize_chlo) { + chlo::populateChloToHighLevelMhloOpPatterns(context, &patterns); + stablehlo::populateChloToStablehloPatterns(context, &patterns); + } + // ConstantLike op is convenient to create splat constants, but is + // canonicalized to plain HLO constant if statically shaped. Add the + // canonicalization pattern to pattern list to enable multi-hop lowering. + chlo::ConstantLikeOp::getCanonicalizationPatterns(patterns, context); + + if (failed(ApplyPatterns(op, patterns, legalize_chlo))) { + return failure(); + } + + // HLO->MLIR raises to StableHLO, but users of this pass expect MHLO. + return StablehloToMhlo(op); +} + +// Performs the lowering to XLA dialect. +void LegalizeTF::runOnOperation() { + auto op = getOperation(); + VLOG(3) << "LegalizeTF(legalize_chlo=" << legalize_chlo_ + << ", prefer_tf2xla=" << prefer_tf2xla_ << ") on module:\n" + << mlir::debugString(*op); + auto op_name = op->getName().getStringRef().str(); + mlir_legalization_count->GetCell(op_name)->IncrementBy(1); + std::optional tf2xla_fallback_device_type = std::nullopt; + if (use_tf2xla_fallback_) { + tf2xla_fallback_device_type = device_type_; + } + if (failed(legalizeTF(op, legalize_chlo_, tf2xla_fallback_device_type, + prefer_tf2xla_))) { + signalPassFailure(); + } +} + +} // end namespace + +std::unique_ptr> createLegalizeTFPass( + bool legalize_chlo, std::optional tf2xla_fallback_device_type, + bool prefer_tf2xla) { + return std::make_unique( + legalize_chlo, tf2xla_fallback_device_type, prefer_tf2xla); +} + +} // end namespace mhlo +} // end namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.td new file mode 100644 index 0000000000000000000000000000000000000000..41f735a848eb227da47b5872bbcf02f6315c068e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_passes.td @@ -0,0 +1,110 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Declare passes used in xla_legalize_tf. + +include "mlir/Pass/PassBase.td" + +def LegalizeTF : Pass<"xla-legalize-tf", "ModuleOp"> { + let summary = "Legalize from TF dialect's or HLO dialect's control flow."; + + let description = [{ + Legalizes from TF dialect to HLO dialect. When allow_partial_conversion is + false, emits an error if there is any operation that can't be legalized. + When `tf2xla_fallback_device_type` is not `None`, also uses legalization + patterns from TF2XLA fallback for provided device type (see + legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is not + used. + }]; + let options = [ + Option<"legalize_chlo_", "legalize-chlo", "bool", /*default=*/"true", + "Legalizes intermediate chlo ops to hlo">, + Option<"use_tf2xla_fallback_", "use-tf2xla-fallback", "bool", + /*default=*/"false", + "Use TF2XLA fallback for legalization">, + Option<"device_type_", "device-type", "std::string", + /*default=*/"\"INVALID_DEVICE_TYPE\"", + "The device type used by TF2XLA fallback. Must be specified if " + "use-tf2xla-fallback is true, otherwise not used">, + Option<"prefer_tf2xla_", "prefer-tf2xla", "bool", + /*default=*/"false", + "Prioritize tf2xla fallback legalization over MLIR legalization " + "patterns">, + ]; + + let constructor = "mlir::mhlo::createLegalizeTFPass()"; + let dependentDialects = [ + "arith::ArithDialect", + "chlo::ChloDialect", + "func::FuncDialect", + "mhlo::MhloDialect", + "quant::QuantDialect", + "shape::ShapeDialect", + "sparse_tensor::SparseTensorDialect", + "stablehlo::StablehloDialect" + ]; +} + +def LegalizeTFCollective : Pass<"xla-legalize-tf-collective", "ModuleOp"> { + let summary = "Legalize TF/XLA collective ops (TensorFlow dialect) to the HLO dialect"; + + let constructor = "mlir::mhlo::CreateLegalizeTFCollectivePass()"; + let dependentDialects = ["mhlo::MhloDialect", "sparse_tensor::SparseTensorDialect"]; +} + +def VerifyTFXLALegalization : Pass<"tfxla-verify-legalization", "mlir::func::FuncOp"> { + let summary = "Verifies that all TF ops have been legalized to XLA."; + + let description = [{"Ensures that all Tensorflow ops have been legalized to " + "XLA and reports an error about which op has not been" + "legalized. This pass does not transform any ops and is just" + " a verification pass to ensure invariants are true."}]; + + let options = [ + Option<"legalize_chlo_", "legalize-chlo", "bool", /*default=*/"true", + "Legalizes intermediate chlo ops to hlo"> + ]; + + let constructor = "mlir::hlo::CreateVerifyTFXLALegalizationPass()"; +} + +def TFXLADeviceSpecificTransforms : Pass<"tfxla-device-specific-transforms", + "mlir::func::FuncOp"> { + let summary = "Transforms ops that require device context into device independent TF Ops."; + + let description = [{"Transforms device specific ops into device independent" + "ops."}]; + + let options = [ + Option<"device_type_", "device-type", "std::string", + /*default=*/"\"INVALID_DEVICE_TYPE\"", + "The device type being targeted.">, + ]; + + let constructor = "mlir::mhlo::CreateTFXLADeviceSpecificTransformsPass()"; +} + +def InfeedOpsXlaAdjustLayout : Pass<"infeed-ops-xla-adjust-layout", + "mlir::func::FuncOp"> { + let summary = "Adjusts Infeed ops layout for XLA."; + + let description = [{"Adjust layouts so infeed send & receive use the same " + "format."}]; + + let constructor = "mlir::mhlo::CreateInfeedsOpsXlaAdjustLayoutPass()"; + + let dependentDialects = ["mhlo::MhloDialect"]; + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..3f243e54e8ff2abc1c62db138b7e2a63b4e743db --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf2xla/transforms/xla_legalize_tf_test.cc @@ -0,0 +1,117 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/deserialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/monitoring/cell_reader.h" + +namespace tensorflow { +namespace { + +using ::mlir::MLIRContext; +using ::mlir::ModuleOp; +using ::mlir::OwningOpRef; +using ::mlir::PassManager; +using ::tensorflow::monitoring::testing::CellReader; + +absl::StatusOr> GetMlirModuleFromString( + absl::string_view module_string, MLIRContext* context) { + mlir::DialectRegistry mlir_registry; + RegisterAllTensorFlowDialects(mlir_registry); + context->appendDialectRegistry(mlir_registry); + + OwningOpRef mlir_module; + auto status = + tensorflow::DeserializeMlirModule(module_string, context, &mlir_module); + if (!status.ok()) { + return status; + } + return mlir_module; +} + +bool BuildAndRunPipeline(absl::string_view module_string, + const std::function& passes) { + mlir::registerPassManagerCLOptions(); + MLIRContext context; + + OwningOpRef module = + GetMlirModuleFromString(module_string, &context).value(); + + PassManager pm(&context); + + if (mlir::failed(mlir::applyPassManagerCLOptions(pm))) return false; + passes(&pm); + + return pm.run(module.get()).succeeded(); +} + +std::function legalizeTFPasses() { + return [](PassManager* pm) { + pm->addPass(mlir::mhlo::createLegalizeTFPass( + /* legalize_chlo=*/true, llvm::StringRef("gpu/xpu"), + /* prefer_tf2xla=*/false)); + }; +} + +TEST(XlaLegalizeTest, IllegalOp) { + constexpr char kMlirIllegalOpStr[] = R"( + module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> tensor<1xi32> { + %0 = "tf.DoesntExist"() : () -> tensor<1xi32> + func.return %0 : tensor<1xi32> + } + })"; + CellReader legalize_failure_count( + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_pass_count"); + + auto status = BuildAndRunPipeline(kMlirIllegalOpStr, legalizeTFPasses()); + + EXPECT_TRUE(status); + EXPECT_EQ(legalize_failure_count.Read("tf.DoesntExist", "Unknown"), 1); +} + +TEST(XlaLegalizeTest, LegalOp) { + // We expect legalization to fail for legal op with dynamic shapes: + static constexpr char kMlirLegalOpStr[] = R"( + func.func @infeed_dequeue_tuple_dynamic_error() -> (tensor<3x3xf32>, tensor<4x?xf32>) { + %0:2 = "tf.InfeedDequeueTuple"() : () -> (tensor<3x3xf32>, tensor<4x?xf32>) func.return %0#0, %0#1 : tensor<3x3xf32>, tensor<4x?xf32> + })"; + CellReader legalize_failure_count( + "/tensorflow/core/tf2xla/v1/mlir_failed_xla_legalize_tf_pass_count"); + + auto status = BuildAndRunPipeline(kMlirLegalOpStr, legalizeTFPasses()); + + EXPECT_TRUE(status); + EXPECT_EQ(legalize_failure_count.Read("tf.InfeedDequeueTuple", "Unknown"), 1); +} +} // namespace + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_opt_main.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_opt_main.cc new file mode 100644 index 0000000000000000000000000000000000000000..492c9ef886120c1bc07e34c50faed17b442de65b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_opt_main.cc @@ -0,0 +1,69 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/InitAllPasses.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Tools/mlir-opt/MlirOptMain.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow//compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h" +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/quantization/stablehlo/passes/bridge/passes.h" +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/runtime_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/sparsecore/sparsecore_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/test_passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mlprogram_util.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/clustering_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/internal/passes/mlir_to_graph_passes.h" +#include "tensorflow/compiler/mlir/tf2xla/transforms/passes.h" +#include "xla/mlir_hlo/mhlo/transforms/passes.h" + +int main(int argc, char** argv) { + tensorflow::InitMlir y(&argc, &argv); + + mlir::registerAllPasses(); + mlir::registerTransformsPasses(); + mlir::registerTensorFlowPasses(); + mlir::TFDevice::registerTensorFlowDevicePasses(); + mlir::tf_saved_model::registerTensorFlowSavedModelPasses(); + mlir::mhlo::registerAllMhloPasses(); + + // These are in compiler/mlir/tf2xla and not part of the above MHLO passes. + mlir::mhlo::registerLegalizeTfPasses(); + mlir::mhlo::registerTfXlaPasses(); + mlir::quant::stablehlo::registerBridgePasses(); + tensorflow::tf2xla::internal::registerTFXLABridgeClusteringPasses(); + tensorflow::tf2xla::internal::registerTFXLABridgeMlirToGraphPasses(); + mlir::tf_test::registerTensorFlowTestPasses(); + tensorflow::RegisterConvertMlirToXlaHloPipelineWithDefaults(); + tensorflow::RegisterGraphOptimizationPasses(); + tensorflow::RegisterMlProgramPasses(); + mlir::TFTPU::registerRuntimeLoweringPasses(); + mlir::TFDevice::registerSparseCorePasses(); + + tensorflow::tfrt_compiler::RegisterTPULowerClusterToRuntimeOpsPassPipeline(); + tensorflow::tfrt_compiler:: + RegisterNonTPULowerClusterToRuntimeOpsPassPipeline(); + + mlir::DialectRegistry registry; + mlir::RegisterCommonToolingDialects(registry); + + return failed( + mlir::MlirOptMain(argc, argv, "TensorFlow pass driver\n", registry)); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_reduce_main.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_reduce_main.cc new file mode 100644 index 0000000000000000000000000000000000000000..68dd105e5b44baae2af84358388dde1e78ce8dbe --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_reduce_main.cc @@ -0,0 +1,72 @@ +/* Copyright 2021 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Reducer/ReductionPatternInterface.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Tools/mlir-reduce/MlirReduceMain.h" // from @llvm-project +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/register_common_dialects.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace { + +struct CollapseAndErase : public mlir::OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + mlir::LogicalResult matchAndRewrite( + mlir::TF::IdentityOp op, mlir::PatternRewriter& rewriter) const override { + auto arg = op.getInput().getDefiningOp(); + if (!arg) return mlir::failure(); + + rewriter.replaceOpWithNewOp(op, op.getType(), + arg.getInput()); + if (arg.use_empty()) { + rewriter.eraseOp(arg); + } + return mlir::success(); + } +}; + +struct TFReductionPatternInterface + : public mlir::DialectReductionPatternInterface { + public: + explicit TFReductionPatternInterface(mlir::Dialect *dialect) + : DialectReductionPatternInterface(dialect) {} + + void populateReductionPatterns( + mlir::RewritePatternSet &patterns) const final { + patterns.add(patterns.getContext()); + } +}; + +} // namespace + +int main(int argc, char *argv[]) { + tensorflow::InitMlir y(&argc, &argv); + + mlir::DialectRegistry registry; + mlir::RegisterCommonToolingDialects(registry); + + registry.addExtension( + +[](mlir::MLIRContext *ctx, mlir::TF::TensorFlowDialect *dialect) { + dialect->addInterfaces(); + }); + + mlir::MLIRContext context(registry); + + return failed(mlirReduceMain(argc, argv, context)); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_translate_main.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_translate_main.cc new file mode 100644 index 0000000000000000000000000000000000000000..0c1707cd19db8b3ab04fa65b7762d2a15e3dae12 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tf_mlir_translate_main.cc @@ -0,0 +1,181 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/strings/str_split.h" +#include "absl/types/span.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/ToolOutputFile.h" +#include "mlir/IR/AsmState.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/ToolUtilities.h" // from @llvm-project +#include "mlir/Tools/mlir-translate/Translation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h" +#include "tensorflow/compiler/mlir/tools/tf_mlir_translate_cl.h" +#include "tensorflow/core/platform/init_main.h" + +// NOLINTNEXTLINE +static llvm::cl::opt input_filename(llvm::cl::Positional, + llvm::cl::desc(""), + llvm::cl::init("-")); + +// NOLINTNEXTLINE +static llvm::cl::opt output_filename( + "o", llvm::cl::desc("Output filename"), llvm::cl::value_desc("filename"), + llvm::cl::init("-")); + +// NOLINTNEXTLINE +static llvm::cl::opt splitInputFile( + "split-input-file", + llvm::cl::desc("Split the input file into pieces and process each chunk " + "independently"), + llvm::cl::init(false)); + +// NOLINTNEXTLINE +static llvm::cl::opt import_saved_model_object_graph( + "savedmodel-objectgraph-to-mlir", + llvm::cl::desc( + "Import a saved model's object graph to its MLIR representation"), + llvm::cl::value_desc("dir")); + +// NOLINTNEXTLINE +static llvm::cl::opt import_saved_model_signature_defs( + "savedmodel-signaturedefs-to-mlir", + llvm::cl::desc( + "Import a saved model's SignatureDefs to their MLIR representation"), + llvm::cl::value_desc("dir")); + +// NOLINTNEXTLINE +static llvm::cl::opt import_saved_model_signature_defs_lite( + "savedmodel-signaturedefs-to-mlir-lite", + llvm::cl::desc("Import a saved model's SignatureDefs to their MLIR " + "representation without any graph transformation"), + llvm::cl::value_desc("dir")); + +// NOLINTNEXTLINE +static llvm::cl::opt saved_model_tags( + "tf-savedmodel-tags", + llvm::cl::desc("Tags used to indicate which MetaGraphDef to import, " + "separated by ','"), + llvm::cl::init("serve")); + +// NOLINTNEXTLINE +static llvm::cl::opt saved_model_exported_names( + "tf-savedmodel-exported-names", + llvm::cl::desc("Names to export from SavedModel, separated by ','. Empty " + "(the default) means export all."), + llvm::cl::init("")); + +int main(int argc, char** argv) { + tensorflow::InitMlir y(&argc, &argv); + + // Add flags for all the registered translations. + llvm::cl::opt + requested_translation("", llvm::cl::desc("Translation to perform")); + mlir::registerAsmPrinterCLOptions(); + llvm::cl::ParseCommandLineOptions(argc, argv, "TF MLIR translation driver\n"); + + if (!import_saved_model_object_graph && !import_saved_model_signature_defs && + !import_saved_model_signature_defs_lite && !requested_translation) { + llvm::errs() << "error: need to specify one translation to perform\n"; + return 1; + } else if (import_saved_model_object_graph + + import_saved_model_signature_defs + + import_saved_model_signature_defs_lite + + (requested_translation != nullptr) > + 1) { + llvm::errs() + << "error: cannot specify more than one translation to perform\n"; + return 1; + } + + std::string error_message; + auto output = mlir::openOutputFile(output_filename, &error_message); + if (!output) { + llvm::errs() << error_message << "\n"; + return 1; + } + + std::unordered_set tags = absl::StrSplit(saved_model_tags, ','); + std::vector exported_names_vector = + absl::StrSplit(saved_model_exported_names, ',', absl::SkipEmpty()); + absl::Span exported_names(exported_names_vector); + + if (import_saved_model_object_graph) { + mlir::MLIRContext context; + auto module_or = tensorflow::SavedModelObjectGraphToMlirImport( + input_filename, tags, exported_names, &context); + if (!module_or.status().ok()) return 1; + + module_or.value()->print(output->os()); + } else if (import_saved_model_signature_defs) { + mlir::MLIRContext context; + tensorflow::MLIRImportOptions import_options; + import_options.upgrade_legacy = upgrade_legacy; + auto module_or = tensorflow::SavedModelSignatureDefsToMlirImport( + input_filename, tags, exported_names, &context, import_options); + if (!module_or.status().ok()) return 1; + + module_or.value()->print(output->os()); + } else if (import_saved_model_signature_defs_lite) { + mlir::MLIRContext context; + tensorflow::MLIRImportOptions import_options; + import_options.upgrade_legacy = upgrade_legacy; + auto module_or = tensorflow::SavedModelSignatureDefsToMlirImportLite( + input_filename, tags, exported_names, &context, import_options); + if (!module_or.status().ok()) return 1; + + module_or.value()->print(output->os()); + } else { + auto input = mlir::openInputFile(input_filename, &error_message); + + if (!input) { + llvm::errs() << error_message << "\n"; + return 1; + } + + // Processes the memory buffer with a new MLIRContext. + auto processBuffer = [&](std::unique_ptr ownedBuffer, + llvm::raw_ostream& os) { + auto sourceMgr = std::make_shared(); + sourceMgr->AddNewSourceBuffer(std::move(ownedBuffer), llvm::SMLoc()); + mlir::MLIRContext context; + mlir::SourceMgrDiagnosticHandler diagnostic_handler(*sourceMgr, &context); + return (*requested_translation)(sourceMgr, os, &context); + }; + + if (splitInputFile) { + if (failed(mlir::splitAndProcessBuffer(std::move(input), processBuffer, + output->os()))) + return 1; + } else { + if (failed(processBuffer(std::move(input), output->os()))) return 1; + } + } + + output->keep(); + return 0; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..80475858c7078519f7b1624b5dafd92ce3e44092 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/BUILD @@ -0,0 +1,381 @@ +load( + "@llvm-project//mlir:tblgen.bzl", + "gentbl_cc_library", + "td_library", +) +load("@xla//third_party/rules_python/python:py_library.bzl", "py_library") +load( + "//tensorflow:tensorflow.bzl", + "tf_cc_binary", + "tf_cc_test", +) +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable", "tf_py_strict_test", "tf_python_pybind_extension") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") +load("//tensorflow/compiler/mlir/tfr:build_defs.bzl", "gen_op_libraries") +load("//tensorflow/core/platform:rules_cc.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + ":friends", + ], + licenses = ["notice"], +) + +package_group( + name = "friends", + packages = [ + "//tensorflow/c/...", + "//tensorflow/compiler/...", + # Allow visibility from the mlir language server. + "//learning/brain/mlir/mlir_lsp_server/...", + ], +) + +td_library( + name = "tfr_ops_td_files", + srcs = [ + "ir/tfr_ops.td", + ], + compatible_with = get_compatible_with_portable(), + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:CallInterfacesTdFiles", + "@llvm-project//mlir:ControlFlowInterfacesTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:QuantizationOpsTdFiles", + "@llvm-project//mlir:ShapeOpsTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "tfr_ops_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = { + "ir/tfr_ops.h.inc": ["-gen-op-decls"], + "ir/tfr_ops.cc.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "ir/tfr_ops.td", + deps = [ + ":tfr_ops_td_files", + ], +) + +gentbl_cc_library( + name = "tfr_decompose_inc_gen", + compatible_with = get_compatible_with_portable(), + tbl_outs = {"passes/generated_decompose.inc": ["-gen-rewriters"]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "passes/decompose_patterns.td", + deps = [ + ":tfr_ops_td_files", + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:FuncTdFiles", + ], +) + +cc_library( + name = "tfr", + srcs = [ + "ir/tfr_ops.cc", + "ir/tfr_ops.cc.inc", + ], + hdrs = [ + "ir/tfr_ops.h", + "ir/tfr_ops.h.inc", + "ir/tfr_types.h", + ], + deps = [ + ":tfr_ops_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_attributes", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ControlFlowInterfaces", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:QuantOps", + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "utils", + srcs = [ + "utils/utils.cc", + ], + hdrs = [ + "utils/utils.h", + ], + deps = [ + ":tfr", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "passes", + srcs = [ + "passes/canonicalize.cc", + "passes/decompose.cc", + "passes/generated_decompose.inc", + "passes/raise_to_tf.cc", + "passes/rewrite_quantized_io.cc", + ], + hdrs = [ + "passes/passes.h", + ], + deps = [ + ":tfr", + ":utils", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:lib", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AffineUtils", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:QuantOps", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFToControlFlow", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], + alwayslink = 1, +) + +tf_cc_binary( + name = "tfr-opt", + srcs = ["passes/tfr_opt.cc"], + deps = [ + ":passes", + ":tfr", + "//tensorflow/compiler/mlir:init_mlir", + "//tensorflow/compiler/mlir:passes", + "//tensorflow/compiler/mlir/quantization/common/ir:QuantOps", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:MlirOptLib", + "@llvm-project//mlir:QuantOps", + "@llvm-project//mlir:RegisterAllPasses", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:ShapeDialect", + ], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + ":tfr-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) + +cc_library( + name = "tfr_decompose_ctx", + srcs = ["integration/tfr_decompose_ctx.cc"], + hdrs = ["integration/tfr_decompose_ctx.h"], + deps = [ + ":passes", + ":tfr", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_attr", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/core:framework", + "//tensorflow/core:framework_types_hdr", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:ShapeDialect", + ], +) + +tf_cc_test( + name = "tfr_decompose_ctx_test", + srcs = ["integration/tfr_decompose_ctx_test.cc"], + deps = [ + ":tfr_decompose_ctx", + "//tensorflow/core:framework", + "//tensorflow/core:ops", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "@com_google_absl//absl/types:span", + "@llvm-project//mlir:IR", + "@xla//xla/hlo/testlib:test", + ], +) + +tf_python_pybind_extension( + name = "tfr_wrapper", + srcs = ["python/tfr_wrapper.cc"], + enable_stub_generation = True, + pytype_srcs = [ + "tfr_wrapper.pyi", + ], + starlark_only = True, + visibility = [ + "//tensorflow/python:__pkg__", + ], + deps = [ + ":tfr", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/python/lib/core:pybind11_lib", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:ShapeDialect", + "@pybind11", + ], +) + +py_library( + name = "composite", + srcs = ["python/composite.py"], + strict_deps = True, +) + +py_library( + name = "tfr_gen", + srcs = ["python/tfr_gen.py"], + strict_deps = True, + deps = [ + ":tfr_wrapper", + "//tensorflow:tensorflow_py", # buildcleaner: keep + "//tensorflow/core:protos_all_py", + "//tensorflow/python/autograph/converters:control_flow", + "//tensorflow/python/autograph/converters:return_statements", + "//tensorflow/python/autograph/impl:api", + "//tensorflow/python/autograph/pyct:anno", + "//tensorflow/python/autograph/pyct:cfg", + "//tensorflow/python/autograph/pyct:qual_names", + "//tensorflow/python/autograph/pyct:transformer", + "//tensorflow/python/autograph/pyct:transpiler", + "//tensorflow/python/autograph/pyct/static_analysis:activity", + "//tensorflow/python/autograph/pyct/static_analysis:reaching_definitions", + "//tensorflow/python/autograph/pyct/static_analysis:reaching_fndefs", + "//tensorflow/python/autograph/pyct/static_analysis:type_inference", + "//tensorflow/python/framework:dtypes", + "//tensorflow/python/framework:load_library", + "//tensorflow/python/framework:op_def_registry", + "//tensorflow/python/platform:tf_logging", + "//tensorflow/python/util:tf_inspect", + "@pypi//gast", + ], +) + +tf_py_strict_test( + name = "tfr_gen_test", + size = "medium", + srcs = ["python/tfr_gen_test.py"], + deps = [ + ":composite", + ":tfr_gen", + "//tensorflow/compiler/mlir/python/mlir_wrapper:filecheck_wrapper", + "//tensorflow/compiler/mlir/tfr/resources:test_ops", + "//tensorflow/python/framework:dtypes", + "//tensorflow/python/ops:array_ops", + "//tensorflow/python/ops:array_ops_gen", + "//tensorflow/python/ops:math_ops", + "//tensorflow/python/ops:math_ops_gen", + "//tensorflow/python/platform:client_testlib", + ], +) + +py_library( + name = "op_reg_gen", + srcs = ["python/op_reg_gen.py"], + strict_deps = True, + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/python/autograph/pyct:transformer", + "//tensorflow/python/autograph/pyct:transpiler", + "//tensorflow/python/framework:op_def_registry", + "//tensorflow/python/util:tf_inspect", + "@pypi//gast", + ], +) + +tf_py_strict_test( + name = "op_reg_gen_test", + size = "small", + srcs = ["python/op_reg_gen_test.py"], + deps = [ + ":composite", + ":op_reg_gen", + "//tensorflow/compiler/mlir/python/mlir_wrapper:filecheck_wrapper", + "//tensorflow/python/platform:client_testlib", + ], +) + +py_library( + name = "test_utils", + srcs = ["python/test_utils.py"], + strict_deps = True, + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/python/eager:backprop", + "//tensorflow/python/framework:test_lib", + "//tensorflow/python/platform:client_testlib", + ], +) + +gen_op_libraries( + name = "one_op", + src = "define_op_template.py", + deps = [ + "//tensorflow/python/platform:flags", + "@absl_py//absl:app", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/README.md b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c11f8720d8b9edccf6bd9eab084e80d34c474fd7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/README.md @@ -0,0 +1,168 @@ +# Composable Tensorflow + +## Composable Tensorflow + +Composable TensorFlow (TF) is the framework for defining portable TF ops with +composition in the authoring language. + +The set of standard TF ops is currently open. New ops are defined for special +purposes but it is hard to make them work end-to-end: The op +needs to be handled separately by several backends (tf2xla bridge, tflite +converter, CPU kernels, etc.). Writing shape functions and gradients for these +ops is extremely difficult. `tf.function` makes some parts of the implementation +simpler, but it introduces runtime overhead and it cannot easily be used to +apply dedicated optimizations to op kernels. + +The composable TF framework allows the user to define portable TF ops as +compositions of other TF ops. It translates a Python function used to define the +composition directly into a portable IR at build time, and uses it to expand the +composite op in the TF program during compilation / execution. By using this +expansion mechanism, new op are readily available on different platforms without +extra work. Moreover, since the expansion is optional, the backend can easily +treat it as a monolithic op when needed, for instance to apply optimizations or +associate it with a custom kernel. + +### Benefits + +Using the Composable TF API to define a new op and its composition can bring the +following benefits: + +* *Automatic backend support*: As long as it is composed of ops supported by + the backend, the new op is automatically supported (as a `tf.function` + alternative); +* *Reduced tracing overhead*: Unlike `tf.function`, the composition function + is compiled at build time, hence TF only needs to trace a single op to build + the `graph`; +* *Easy fused op/kernel optimization*: Even if it has complex semantics, the + new op is presented as a single node in the graph, thus optimization passes + and kernels can easily be specialized to this op for better performance. +* *Automatic shape/type inference support*: No shape functions are required + for the new op; +* *Automatic gradient support (WIP)*: The user doesn't need to author gradient + a function of the op for training. + +### Use Cases + +* (Portability) User wants to add a new op and run this op on different + platforms (CPU, TPU, TFLite, etc.) to be portable. + + * *Solution*: The user should define the new op as a composition. The ops + used inside the composition should have support for these platforms. + These ops can also be composite ops. + +* (Performance) User defines a custom kernel for a regular structure (i.e. + LSTM), but it is hard to add the logic to fuse the individual ops to target + this kernel in the inference graph. + + * *Solution*: The user should define a new TF op, which corresponds to the + fused kernel, with composition, and use this op to build the model for + both training and inference. For the platforms where a fused kernel is + not available, the execution will use the composition instead. + +## Gradient +(TODO) + +## Authoring Op Composition in Python + +The composable TF provides a single API to define a new op with its composition +at the same time. For example, the following code defines a new +`FusedFullyConnected` op, which have `MatMul`, `Add` and some +`activation function` (specified by an op attribute) fused. + + +```python +import tensorflow as tf + +@Composite( + 'FusedFullyConnected', + inputs=['input_: T', 'filter_: T', 'bias: T'], + attrs=['act: {"", "RELU", "RELU6", "TANH"} = ""'], + derived_attrs=['T: {float, int8}'], + outputs=['o: T']) +def _composite_fully_connected(input_, filter_, bias, act): + res = tf.raw_ops.MatMul( + a=input_, b=filter_, transpose_a=False, transpose_b=True) + res = tf.raw_ops.Add(x=res, y=bias) + if act == 'RELU': + return tf.raw_ops.Relu(features=res) + elif act == 'RELU6': + return tf.raw_ops.Relu6(features=res) + elif act == 'TANH': + return tf.raw_ops.Tanh(x=res) + else: + return res + +``` + +Besides defining new ops, composition can be specified for an existing op +for portability. The following code defines the semantics of `AddNOp`: + +```python +@Composite('AddNOp') +def _my_op_c(ins): + N = len(ins) + if N == 1: + return ins[0] + sum = ins[0] + for i in range(1, N): + sum += ins[i] + return sum +``` + +Utilities have been built to compile the Python composition functions down to +the backend IR. The project also provides a set of graph optimization passes to +expand the composite ops in the graph by using the input backend IR. These +passes have been added to the TF +[common runtime](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/common_runtime) +for graph execution and +[eager runtime](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/common_runtime/eager) +for eager execution. + +## Compiling Op Composition + +### Ahead-Of-Time (AOT) mode + +Like the op kernels, the op composition can be pre-compiled to the backend IR +so the decomposition can be invoked at runtime. A Python [define_op_template.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/mlir/tfr/define_op_template.py) +file is provided as an example to build composite ops in the users project +directory. All the targets required to build the new ops are created by the +following target: + + +```BUILD +load("//tensorflow/compiler/mlir/tfr:build_defs.bzl", "gen_op_libraries") + +gen_op_libraries( + name = "test_ops", + src = "define_op_template.py", + deps = [ + "//third_party/py/tensorflow", + ], +) +``` + +More composite op definitions and usages are here included in the +[examples](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/mlir/tfr/examples) +directory. + +### Just-In-Time (JIT) mode +(TODO) + +## Known Limitations + +* `while` statement +* condition of `if` statement couldn't be a tensor + +## RFC + +This project is an alternative implementation of +[RFC:Standardizing composite ops in tensorflow to support efficient inference](https://github.com/tensorflow/community/blob/master/rfcs/20190610-standardizing-composite_ops.md). +This project doesn't rely on the tracing functionality provided by `tf.function` +to avoid all its pitfalls and it helps to build more general transformations in +the backends. + +## Team + +* Feng Liu +* Dan Moldovan + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/build_defs.bzl b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/build_defs.bzl new file mode 100644 index 0000000000000000000000000000000000000000..331fe133f753449e444d16589fbbfc9fac1484ab --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/build_defs.bzl @@ -0,0 +1,165 @@ +"""BUILD extension for TF composition project.""" + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_ml_toolchain//py/rules_pywrap:pywrap.default.bzl", "use_pywrap_rules") +load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary") +load("@xla//third_party/rules_python/python:py_library.bzl", "py_library") +load("//tensorflow:tensorflow.bzl", "tf_custom_op_library", "tf_gen_op_wrapper_py") +load("//tensorflow:tensorflow.default.bzl", "tf_custom_op_py_library") + +# TODO(b/356020232): cleanup use_pywrap_rules once migration is done +def gen_op_libraries( + name, + src, + deps = [], + tags = [], + test = False): + """gen_op_libraries() generates all cc and py libraries for composite op source. + + Args: + name: used as the name component of all the generated libraries. + src: File contains the composite ops. + deps: Libraries the 'src' depends on. + tags: + test: + """ + if not src.endswith(".py") or name == src[:-3]: + fail("'src' %s conflicts with op Python wrapper. Rename it to be different from 'name'." % src) + + py_deps = [] + if use_pywrap_rules(): + py_deps = ["//tensorflow/python:_pywrap_tensorflow"] + + py_deps += [ + "//tensorflow/compiler/mlir/tfr:op_reg_gen", + "//tensorflow/compiler/mlir/tfr:tfr_gen", + "//tensorflow/compiler/mlir/tfr:composite", + ] + deps + + gen_op_lib_exec = src[:-3] # Strip off the .py + py_binary( + name = gen_op_lib_exec, + srcs = [src], + srcs_version = "PY3", + python_version = "PY3", + deps = py_deps, + ) + + registered_op = "registered_" + name + native.genrule( + name = registered_op, + srcs = [], + outs = [name + ".inc.cc"], + cmd = + "$(location %s) --output=$@ --gen_register_op=true" % gen_op_lib_exec, + tools = [":" + gen_op_lib_exec], + tags = tags, + ) + cc_library( + name = name + "_cc", + testonly = test, + srcs = [":" + registered_op], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + ], + alwayslink = 1, + ) + + tf_custom_op_library( + name = name + ".so", + srcs = [":" + registered_op], + ) + + tf_gen_op_wrapper_py( + name = "gen_" + name, + out = "gen_" + name + ".py", + py_lib_rule = py_library, + deps = [ + ":%s_cc" % name, + ], + extra_py_deps = [ + "//tensorflow/python:pywrap_tfe", + "//tensorflow/python/util:dispatch", + "//tensorflow/python/util:deprecation", + "//tensorflow/python/util:tf_export", + ] + py_deps, + ) + + tf_custom_op_py_library( + name = name, + dso = [":%s.so" % name], + kernels = [":%s_cc" % name], + srcs_version = "PY3", + # copybara:uncomment(OSS version passes this to py_library) lib_rule = py_library, + deps = [ + ":gen_%s" % name, + ], + ) + + # Link the register op and rebuild the binary + gen_tfr_lib_exec = gen_op_lib_exec + "_with_op_library" + py_binary( + name = gen_tfr_lib_exec, + main = src, + srcs = [src], + python_version = "PY3", + srcs_version = "PY3", + deps = py_deps + [":%s" % name], + ) + + native.genrule( + name = name + "_mlir", + srcs = [], + outs = [name + ".mlir"], + cmd = + "$(location %s) --output=$@ --gen_register_op=false" % gen_tfr_lib_exec, + tools = [":" + gen_tfr_lib_exec], + tags = tags, + ) + + py_library( + name = name + "_py", + srcs = [src], + srcs_version = "PY3", + deps = py_deps, + ) + +def gen_op_bindings(name): + cc_library( + name = name + "_ops_cc", + srcs = [name + "_ops.cc"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + ], + alwayslink = 1, + ) + + tf_custom_op_library( + name = name + "_ops.so", + srcs = [name + "_ops.cc"], + ) + + tf_gen_op_wrapper_py( + name = "gen_" + name + "_ops", + out = "gen_" + name + "_ops.py", + py_lib_rule = py_library, + deps = [":" + name + "_ops_cc"], + extra_py_deps = [ + "//tensorflow/python:pywrap_tfe", + "//tensorflow/python/util:dispatch", + "//tensorflow/python/util:deprecation", + "//tensorflow/python/util:tf_export", + ], + ) + + tf_custom_op_py_library( + name = name + "_ops", + dso = [":" + name + "_ops.so"], + kernels = [":" + name + "_ops_cc"], + # copybara:uncomment(OSS version passes this to py_library) lib_rule = py_library, + deps = [":gen_" + name + "_ops"], + ) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/define_op_template.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/define_op_template.py new file mode 100644 index 0000000000000000000000000000000000000000..d290726585bd50cab879269213f3a85e056f475d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/define_op_template.py @@ -0,0 +1,60 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A template to define composite ops.""" + +# pylint: disable=g-direct-tensorflow-import + +import os +import sys + +from absl import app +from tensorflow.compiler.mlir.tfr.python.composite import Composite +from tensorflow.compiler.mlir.tfr.python.op_reg_gen import gen_register_op +from tensorflow.compiler.mlir.tfr.python.tfr_gen import tfr_gen_from_module +from tensorflow.python.platform import flags + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + 'output', None, + 'Path to write the genereated register op file and MLIR file.') + +flags.DEFINE_bool('gen_register_op', True, + 'Generate register op cc file or tfr mlir file.') + +flags.mark_flag_as_required('output') + + +@Composite('TestRandom', derived_attrs=['T: numbertype'], outputs=['o: T']) +def _composite_random_op(): + pass + + +def main(_): + if FLAGS.gen_register_op: + assert FLAGS.output.endswith('.cc') + generated_code = gen_register_op(sys.modules[__name__], '_composite_') + else: + assert FLAGS.output.endswith('.mlir') + generated_code = tfr_gen_from_module(sys.modules[__name__], '_composite_') + + dirname = os.path.dirname(FLAGS.output) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(FLAGS.output, 'w') as f: + f.write(generated_code) + + +if __name__ == '__main__': + app.run(main=main) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.cc new file mode 100644 index 0000000000000000000000000000000000000000..f45d6fc5aa38c99a5dd656dc281cfec70e42df34 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.cc @@ -0,0 +1,247 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_attr.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/compiler/mlir/tfr/passes/passes.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_def.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/path.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/stringpiece.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/env_var.h" + +namespace tensorflow { +namespace tfr { + +const char* const kTFRLibEnv = "TF_MLIR_TFR_LIB_DIR"; + +absl::StatusOr> TFRDecomposeContext::Get( + mlir::MLIRContext* mlir_ctx) { + Env* env = Env::Default(); + std::string tfr_lib_dir; + TF_RETURN_IF_ERROR(ReadStringFromEnvVar( + kTFRLibEnv, "tensorflow/compiler/mlir/tfr/resources", &tfr_lib_dir)); + std::string composite_mlir_dir = + io::JoinPath(env->GetRunfilesDir(), tfr_lib_dir); + std::vector files; + TF_RETURN_IF_ERROR(env->GetChildren(composite_mlir_dir, &files)); + if (files.empty()) { + return absl::InternalError(absl::StrCat( + "Failed to find the decomposition lib from path ", composite_mlir_dir)); + } + std::string tfr_raw_text; + for (const auto& file : files) { + std::string fullpath = io::JoinPath(composite_mlir_dir, file); + if (env->MatchPath(fullpath, io::JoinPath(composite_mlir_dir, "*.mlir"))) { + std::string text; + TF_RETURN_IF_ERROR(ReadFileToString(env, fullpath, &text)); + tfr_raw_text.append(text); + } + } + + auto ctx = TFRDecomposeContext::GetFromText(tfr_raw_text, mlir_ctx); + if (!ctx) { + return absl::InternalError(absl::StrCat( + "Failed to load the imported decomposition lib: ", tfr_raw_text)); + } + return ctx; +} + +std::unique_ptr TFRDecomposeContext::GetFromText( + absl::string_view tfr_raw_text, mlir::MLIRContext* mlir_ctx) { + mlir_ctx->allowUnregisteredDialects(/*allow=*/true); + // Load dialects involved in the conversion + mlir::DialectRegistry registry; + // clang-format off + registry.insert(); + // clang-format on + mlir::func::registerAllExtensions(registry); + mlir_ctx->appendDialectRegistry(registry); + mlir_ctx->loadAllAvailableDialects(); + + // Load the TFR functions in a mlir::ModuleOp + auto memory_buffer = llvm::MemoryBuffer::getMemBuffer( + llvm::StringRef(tfr_raw_text.data(), tfr_raw_text.size())); + llvm::SourceMgr source_mgr; + source_mgr.AddNewSourceBuffer(std::move(memory_buffer), llvm::SMLoc()); + mlir::OwningOpRef module = + mlir::parseSourceFile(source_mgr, mlir_ctx); + // The MLIRContext owns the module + auto module_op = module.release(); + + // Create the context + return std::make_unique(module_op); +} + +absl::StatusOr TFRDecomposeContext::ExpandNode( + const NodeDef& node_def, absl::string_view func_name) { + const OpDef* op_def; + TF_RETURN_IF_ERROR(OpRegistry::Global()->LookUpOpDef(node_def.op(), &op_def)); + DataTypeVector input_dtys, output_dtys; + TF_RETURN_IF_ERROR(InputTypesForNode(node_def, *op_def, &input_dtys)); + TF_RETURN_IF_ERROR(OutputTypesForNode(node_def, *op_def, &output_dtys)); + + mlir::MLIRContext* context = tfr_module_.getContext(); + llvm::SmallVector input_tys, output_tys; + mlir::Builder builder(context); + for (auto ty : input_dtys) { + mlir::Type elt_ty; + TF_RETURN_IF_ERROR(ConvertDataType(ty, builder, &elt_ty)); + mlir::TensorType mlir_ty = mlir::UnrankedTensorType::get(elt_ty); + input_tys.push_back(mlir_ty); + } + for (auto ty : output_dtys) { + mlir::Type elt_ty; + TF_RETURN_IF_ERROR(ConvertDataType(ty, builder, &elt_ty)); + mlir::TensorType mlir_ty = mlir::UnrankedTensorType::get(elt_ty); + output_tys.push_back(mlir_ty); + } + llvm::SmallVector attrs; + for (const auto& attr : node_def.attr()) { + TF_ASSIGN_OR_RETURN(auto mlir_attr, + ConvertAttributeValue(attr.second, &builder)); + attrs.push_back({mlir::StringAttr::get(context, attr.first), mlir_attr}); + } + + mlir::Location loc = mlir::UnknownLoc::get(context); + mlir::ModuleOp module = mlir::ModuleOp::create(loc); + mlir::FunctionType func_type = + mlir::FunctionType::get(context, input_tys, output_tys); + llvm::StringRef func_name_str(func_name.data(), func_name.size()); + auto func = mlir::func::FuncOp::create(loc, func_name_str, func_type, {}); + module.push_back(func); + func.addEntryBlock(); + mlir::OpBuilder op_builder(func.getBody()); + + // Create the TF op + const std::string tf_op_full_name = absl::StrCat("tf.", node_def.op()); + mlir::OperationState op_state(loc, tf_op_full_name); + op_state.addOperands(func.getArguments()); + op_state.addTypes(output_tys); + op_state.addAttributes(attrs); + mlir::Operation* tf_op = op_builder.create(op_state); + mlir::func::ReturnOp::create(op_builder, loc, tf_op->getResults()); + + // Run the decompose passes on the module + TF_RETURN_IF_ERROR(DecomposeGraph(module)); + + // Export the result as a FunctionDef. + FunctionDef func_def; + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::ConvertMlirFunctionToFunctionLibraryDef( + func, export_confs_, &func_def)); + module.erase(); + return func_def; +} + +absl::Status TFRDecomposeContext::DecomposeGraph(mlir::ModuleOp user_module) { + // Call the decompose passes by using the external symbol table. + if (failed(pm_.run(user_module))) { + return absl::InternalError("Failed to run the decompose passes."); + } + return absl::OkStatus(); +} + +// Constructor of the decompose context. +TFRDecomposeContext::TFRDecomposeContext(mlir::ModuleOp tfr_module) + : tfr_module_(tfr_module), pm_(tfr_module_.getContext()) { + mlir::OpPassManager& func_pm = pm_.nest(); + + // Prepare the imported graph. + func_pm.addPass(mlir::CreateExecutorDialectToFunctionalConversionPass()); + + // Run TFR lowering, inlining and raising to tf. + func_pm.addPass(mlir::TFR::CreateDecomposeTFOpsPass(tfr_module_)); + func_pm.addPass(mlir::TFR::CreateRaiseToTFOpsPass( + tfr_module_, /*materialize_derived_attrs=*/true)); + + // Prepare to be exported. + func_pm.addPass(mlir::CreateFunctionalToExecutorDialectConversionPass()); + pm_.addPass(mlir::CreateBreakUpIslandsPass()); +} + +void TFRDecomposeContext::Destroy() { tfr_module_.erase(); } + +absl::StatusOr ExpandNode(const NodeDef& node_def, + absl::string_view func_name) { + mlir::MLIRContext mlir_ctx; + TF_ASSIGN_OR_RETURN(auto ctx, TFRDecomposeContext::Get(&mlir_ctx)); + return ctx->ExpandNode(node_def, func_name); +} + +absl::Status DecomposeGraph(mlir::ModuleOp user_module) { + mlir::MLIRContext* mlir_ctx = user_module.getContext(); + TF_ASSIGN_OR_RETURN(auto ctx, TFRDecomposeContext::Get(mlir_ctx)); + return ctx->DecomposeGraph(user_module); +} + +} // namespace tfr +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.h new file mode 100644 index 0000000000000000000000000000000000000000..c27bfe4b1b0591be40490db7cc085a03db4db4aa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.h @@ -0,0 +1,87 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFR_INTEGRATION_TFR_DECOMPOSE_CTX_H_ +#define TENSORFLOW_COMPILER_MLIR_TFR_INTEGRATION_TFR_DECOMPOSE_CTX_H_ + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/stringpiece.h" + +namespace tensorflow { +namespace tfr { + +extern const char* const kTFRLibEnv; + +using tsl::StatusOr; + +// An wrapper for all the objects used to decompose a module (graph mode) and +// node_def (eager mode). Note that this class owns the decomposition library. +class TFRDecomposeContext { + public: + // The entry function to get a decompose context. All the required passes have + // been initialized. + static absl::StatusOr> Get( + mlir::MLIRContext* mlir_ctx); + + // Constructor of the decompose context. To share the decompose library, the + // whole decompose TFR function library is loaded. + explicit TFRDecomposeContext(mlir::ModuleOp tfr_module); + + // Constructs the decompose context from the tfr text module and the mlir + // context. The tfr text module is added to the mlir context. + static std::unique_ptr GetFromText( + absl::string_view tfr_raw_text, mlir::MLIRContext* mlir_ctx); + + // Decomposes the op in the NodeDef to a set of primitive ops according to the + // decompose library in the context. Wrap the decomposed result in a + // FunctionDef. + absl::StatusOr ExpandNode(const NodeDef& node_def, + absl::string_view func_name); + + // Runs the decompose passes on the user_module. + absl::Status DecomposeGraph(mlir::ModuleOp user_module); + + // Erases the tfr_module created. + void Destroy(); + + private: + mlir::ModuleOp tfr_module_; + mlir::PassManager pm_; + + GraphExportConfig export_confs_; +}; + +// Decomposes the NodeDef to a set of primitive ops according to the decompose +// library loaded. Wrap the decomposed result in a FunctionDef. +absl::StatusOr ExpandNode(const NodeDef& node_def, + absl::string_view func_name); + +// Decomposes the ops in the ModuleOp to a set of primitive ops according to +// decompose library in the context. +absl::Status DecomposeGraph(mlir::ModuleOp user_module); + +} // namespace tfr +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFR_INTEGRATION_TFR_DECOMPOSE_CTX_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a5b65a9deb0d1d0ad80dfca48d6d3f7fff6f8f83 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx_test.cc @@ -0,0 +1,159 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfr/integration/tfr_decompose_ctx.h" + +#include +#include +#include +#include + +#include "absl/types/span.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "xla/hlo/testlib/test.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/test.h" + +using testing::ElementsAreArray; +using testing::Test; +using NodeAndType = std::pair; + +namespace tensorflow { +namespace { + +REGISTER_OP("MyAddN") + .Input("inputs: N * T") + .Output("sum: T") + .Attr("N: int >= 1") + .Attr("T: {numbertype, variant}") + .SetIsCommutative() + .SetIsAggregate() + .SetShapeFn(shape_inference::UnchangedShape); + +REGISTER_OP("RiscAddDummy") + .Input("x: T") + .Input("y: T") + .Output("z: T") + .Attr( + "T: {bfloat16, half, float, double, uint8, int8, int16, int32, int64, " + "complex64, complex128, string}") + .SetShapeFn(shape_inference::UnchangedShape); + +constexpr char tfr_raw_text[] = R"( + +tfr.func @tf__my_add_n(%values: !tfr.tensor_list, + %n: i64 {tfr.name="N"}) -> !tfr.tensor { + %index = arith.constant 0 : index + %cst = arith.constant 1 : i64 + %eq = arith.cmpi "eq", %n, %cst : i64 + %v1 = tfr.get_element %values[%index] : (!tfr.tensor_list, index) -> !tfr.tensor + %res = scf.if %eq -> !tfr.tensor { + scf.yield %v1 : !tfr.tensor + } else { + %step = arith.index_cast %cst : i64 to index + %end = arith.index_cast %n : i64 to index + %reduce = scf.for %i = %step to %end step %step iter_args(%reduce_iter=%v1) -> !tfr.tensor { + %v = tfr.get_element %values[%i] : (!tfr.tensor_list, index) -> !tfr.tensor + %reduce_next = tfr.call @tf__risc_add_dummy(%reduce_iter, %v) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + scf.yield %reduce_next : !tfr.tensor + } + scf.yield %reduce : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +tfr.func @tf__my_add_n_(!tfr.tensor_list, i64 {tfr.name="N"}) -> !tfr.tensor attributes{N,T} +tfr.func @tf__risc_add_dummy_(!tfr.tensor, !tfr.tensor) -> !tfr.tensor attributes{T} +)"; + +class TFRDecomposeContextTest : public Test { + protected: + void SetUp() override { + test_ctx_ = tfr::TFRDecomposeContext::GetFromText(tfr_raw_text, &ctx_); + } + + void TearDown() override { test_ctx_->Destroy(); } + + mlir::MLIRContext ctx_; + std::unique_ptr test_ctx_; +}; + +std::vector NodesSequenceOf(const FunctionDef& graph) { + std::vector nodes; + for (auto& node : graph.node_def()) { + nodes.push_back({node.op(), node.attr().at("T").type()}); + } + return nodes; +} + +TEST_F(TFRDecomposeContextTest, FLOAT_1_ins) { + std::vector src_list; + src_list.emplace_back("input", 0, DT_FLOAT); + NodeDef test_node; + auto status = NodeDefBuilder("float_add", "MyAddN") + .Input(src_list) + .Finalize(&test_node); + EXPECT_TRUE(status.ok()); + auto decomposed = test_ctx_->ExpandNode(test_node, "test"); + EXPECT_TRUE(decomposed.ok()); + std::vector expected_results{{"Identity", DT_FLOAT}}; + EXPECT_THAT(NodesSequenceOf(decomposed.value()), + ElementsAreArray(expected_results)); +} + +TEST_F(TFRDecomposeContextTest, FLOAT_3_ins) { + std::vector src_list; + src_list.emplace_back("in0", 0, DT_FLOAT); + src_list.emplace_back("in1", 0, DT_FLOAT); + src_list.emplace_back("in2", 0, DT_FLOAT); + NodeDef test_node; + auto status = NodeDefBuilder("float_add_3", "MyAddN") + .Input(src_list) + .Finalize(&test_node); + EXPECT_TRUE(status.ok()); + auto decomposed = test_ctx_->ExpandNode(test_node, "test"); + EXPECT_TRUE(decomposed.ok()); + + std::vector expected_results{{"RiscAddDummy", DT_FLOAT}, + {"RiscAddDummy", DT_FLOAT}}; + EXPECT_THAT(NodesSequenceOf(decomposed.value()), + ElementsAreArray(expected_results)); +} + +TEST_F(TFRDecomposeContextTest, INT32_3_ins) { + std::vector src_list; + src_list.emplace_back("in0", 0, DT_INT32); + src_list.emplace_back("in1", 0, DT_INT32); + src_list.emplace_back("in2", 0, DT_INT32); + NodeDef test_node; + auto status = + NodeDefBuilder("int_add", "MyAddN").Input(src_list).Finalize(&test_node); + EXPECT_TRUE(status.ok()); + auto decomposed = test_ctx_->ExpandNode(test_node, "test"); + EXPECT_TRUE(decomposed.ok()); + + std::vector expected_results{{"RiscAddDummy", DT_INT32}, + {"RiscAddDummy", DT_INT32}}; + EXPECT_THAT(NodesSequenceOf(decomposed.value()), + ElementsAreArray(expected_results)); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..33618687ceef8c3c345b2ec15162bd6223260be1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.cc @@ -0,0 +1,1037 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" + +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/QuantTypes.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypeInterfaces.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/FunctionImplementation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_types.h" + +namespace mlir { + +namespace TFR { + +//===----------------------------------------------------------------------===// +// InlinerInterface +//===----------------------------------------------------------------------===// + +namespace { +/// This class defines the interface for inlining within the TFR dialect. +class TFRInlinerInterface : public DialectInlinerInterface { + using DialectInlinerInterface::DialectInlinerInterface; + + public: + // Allow all call operations to be inlined. + bool isLegalToInline(Operation *call, Operation *callable, + bool wouldBeCloned) const final { + return true; + } + // Returns true if the given region 'src' can be inlined into the region + // 'dest' that is attached to an operation registered to the current dialect. + bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned, + IRMapping &) const final { + return true; + } + + // Returns true if the given operation 'op', that is registered to this + // dialect, can be inlined into the region 'dest' that is attached to an + // operation registered to the current dialect. + bool isLegalToInline(Operation *op, Region *dest, bool wouldBeCloned, + IRMapping &) const final { + return true; + } + + // Handle the given inlined terminator by replacing it with a new operation + // as necessary. Required when the region has only one block. + void handleTerminator(Operation *op, ValueRange valuesToRepl) const final { + auto retValOp = dyn_cast(op); + if (!retValOp) return; + + for (auto ret_value : llvm::zip(valuesToRepl, retValOp.getOperands())) { + std::get<0>(ret_value).replaceAllUsesWith(std::get<1>(ret_value)); + } + } + + // Attempts to materialize a conversion for a type mismatch between a call + // from this dialect, and a callable region. This method should generate an + // operation that takes 'input' as the only operand, and produces a single + // result of 'resultType'. If a conversion can not be generated, nullptr + // should be returned. + Operation *materializeCallConversion(OpBuilder &builder, Value input, + Type result_type, + Location conversion_loc) const final { + if (!isa(input.getType()) || !isa(result_type)) { + return nullptr; + } + auto input_itype = llvm::cast(input.getType()); + auto result_itype = llvm::cast(result_type); + if (input_itype.getWidth() == result_itype.getWidth()) return nullptr; + if (input_itype.getWidth() > result_itype.getWidth()) { + return arith::TruncIOp::create(builder, conversion_loc, result_type, + input); + } else { + return arith::ExtSIOp::create(builder, conversion_loc, result_type, + input); + } + } +}; +} // namespace + +//===----------------------------------------------------------------------===// +// TFR Dialect +//===----------------------------------------------------------------------===// + +TFRDialect::TFRDialect(MLIRContext *context) + : Dialect(/*name=*/"tfr", context, TypeID::get()) { + // TFR depends on TensorFlow for its canonicalization + context->getOrLoadDialect(); + + addTypes(); + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.cc.inc" + >(); + + addInterfaces(); +} + +Operation *TFRDialect::materializeConstant(OpBuilder &builder, Attribute value, + Type type, Location loc) { + if (arith::ConstantOp::isBuildableWith(value, type)) + return arith::ConstantOp::create(builder, loc, type, + llvm::cast(value)); + if (func::ConstantOp::isBuildableWith(value, type)) + return func::ConstantOp::create(builder, loc, type, + llvm::cast(value)); + return nullptr; +} + +bool TFRType::classof(Type type) { + return llvm::isa(type.getDialect()); +} + +//===----------------------------------------------------------------------===// +// Custom op methods +//===----------------------------------------------------------------------===// + +void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) { + // Direct call. + if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) { + auto symRef = callee.get(); + return setCalleeAttr(cast(symRef)); + } + // Indirect call, callee Value is the first operand. + return setOperand(0, callee.get()); +} + +LogicalResult ConstantTensorOp::verify() { + ConstantTensorOp op = *this; + auto input_type = op.getArg().getType(); + auto output_type = op.getOut().getType(); + + if (auto output_tensor_type = llvm::dyn_cast(output_type)) { + return success(); + } + + auto output_tensor_type = llvm::dyn_cast(output_type); + if (!output_tensor_type || !output_tensor_type.hasStaticShape()) { + op.emitError("output type should be static and ranked."); + return failure(); + } + + if (output_tensor_type.getRank() == 0) { + bool same_scalar = output_tensor_type.getElementType() == input_type; + if (!same_scalar) { + op.emitError("input and output should have the same scalar types."); + } + return success(same_scalar); + } + + if (auto input_vector_type = llvm::dyn_cast(input_type)) { + bool same_element_type = output_tensor_type.getElementType() == + input_vector_type.getElementType(); + bool same_shape = + output_tensor_type.getShape() == input_vector_type.getShape(); + if (!same_element_type || !same_shape) { + op.emitError("input and output should have same shape and element type."); + } + return success(same_element_type && same_shape); + } + + op.emitError("input can not be converted to an output tensor."); + return failure(); +} + +LogicalResult TFRFuncOp::verify() { + TFRFuncOp func = *this; + // Collect all attribute names used by the tensor and tensor list arguments + // and returns. Also, collect the names of all the attribute arguments as the + // defined list. Later on, the used attribute names will be verified to be in + // the defined list. + llvm::SmallVector input_used_attrs, output_used_attrs; + + // While scanning the arguments, record the start/end indices of each argument + // type, so the order can be verified as well. + // TODO(fengliuai): the attribute arguments with default values need to be + // at the end? + int first_tensor = -1, last_tensor = -1, first_tensor_list = -1, + last_tensor_list = -1, first_attr = -1; + for (auto arg : llvm::enumerate(func.getFunctionType().getInputs())) { + Type arg_type = arg.value(); + + if (auto tensor = llvm::dyn_cast(arg_type)) { + if (first_tensor == -1) { + first_tensor = arg.index(); + } + last_tensor = arg.index(); + auto used = tensor.getAttrKeys(); + input_used_attrs.append(used.begin(), used.end()); + continue; + } + + if (auto tensor_list = llvm::dyn_cast(arg_type)) { + if (first_tensor_list == -1) { + first_tensor_list = arg.index(); + } + last_tensor_list = arg.index(); + auto used = tensor_list.getAttrKeys(); + input_used_attrs.append(used.begin(), used.end()); + continue; + } + + if (!isa(arg_type)) { + if (first_attr == -1) { + first_attr = arg.index(); + } + auto name = + func.getArgAttrOfType(arg.index(), kAttrArgumentNameAttr); + if (!name) { + func.emitError( + llvm::Twine(arg.index()) + + " attribute argument doesn't have a tfr.name attribute."); + return failure(); + } + continue; + } + + func.emitError("Builtin TensorType isn't allowed as the argument."); + return failure(); + } + + // Collect all the undefined attributes used in the inputs. + llvm::SmallVector undefined_attrs; + for (auto attr : input_used_attrs) { + if (!func->getAttr(attr.getValue())) { + undefined_attrs.push_back(attr); + } + } + + // Verify the argument order: tensors, tensor list, attributes; and also + // verify there is at most one tensor list argument. + if (first_attr != -1 && + (first_attr < last_tensor_list || first_attr < last_tensor)) { + func.emitError( + "tfr.tensor/tfr.tensor_list argument should be before non tensor " + "arguments."); + return failure(); + } + // The order between tensor arguments and tensor list arguments and the number + // of tensor list arguments are verified only when they couldn't be determined + // by the attributes. + if (!undefined_attrs.empty()) { + if (first_tensor_list != -1 && first_tensor_list < last_tensor) { + func.emitError( + "tfr.tensor argument should be before tfr.tensor_list argument."); + return failure(); + } + if (first_tensor_list != last_tensor_list) { + func.emitError("More than one tfr.tensor_list argument isn't allowed."); + return failure(); + } + } + + // Verify the result order: tensor, tensor list, and also verify at most one + // tensor list result. + int undefined_input_attrs_number = undefined_attrs.size(); + bool seen_tensor_list = false, has_tensor_list_order_error = false, + has_multiple_tensor_lists_error = false; + for (auto result_type : func.getFunctionType().getResults()) { + if (auto tensor = llvm::dyn_cast(result_type)) { + if (seen_tensor_list) { + has_tensor_list_order_error = true; + } else { + auto used = tensor.getAttrKeys(); + output_used_attrs.append(used.begin(), used.end()); + } + continue; + } + + if (auto tensor_list = llvm::dyn_cast(result_type)) { + if (seen_tensor_list) { + has_multiple_tensor_lists_error = true; + } else { + seen_tensor_list = true; + auto used = tensor_list.getAttrKeys(); + output_used_attrs.append(used.begin(), used.end()); + } + continue; + } + + func.emitError( + "None tfr.tensor/tfr.tensor_list results aren't allowed as a " + "result."); + return failure(); + } + + // Collect all the undefined attributes used in the outputs. + for (auto attr : output_used_attrs) { + if (!func->getAttr(attr.getValue())) { + undefined_attrs.push_back(attr); + } + } + + // Verify there are no tensor/tensor list order error and multiple tensor + // list arguments error. + if (undefined_input_attrs_number != undefined_attrs.size()) { + if (has_tensor_list_order_error) { + func.emitError( + "tfr.tensor result should be before tfr.tensor_list result."); + return failure(); + } else if (has_multiple_tensor_lists_error) { + func.emitError("More than one tfr.tensor_list result isn't allowed."); + return failure(); + } + } + + // TODO(fengliuai): We might want to refine this constraint because the + // tensor element type can be derived. + if (!undefined_attrs.empty()) { + llvm::SmallVector attr_names(undefined_attrs.size()); + std::transform(undefined_attrs.begin(), undefined_attrs.end(), + attr_names.begin(), + [](StringAttr attr) { return attr.getValue().str(); }); + func.emitError(llvm::Twine("Undefined attributes are used: ", + llvm::join(attr_names, ","))); + return failure(); + } + + return success(); +} + +ParseResult TFRFuncOp::parse(OpAsmParser &parser, OperationState &result) { + auto build_func_type = + [](Builder &builder, ArrayRef arg_types, ArrayRef results, + function_interface_impl::VariadicFlag, + std::string &) { return builder.getFunctionType(arg_types, results); }; + return function_interface_impl::parseFunctionOp( + parser, result, /*allowVariadic=*/false, + getFunctionTypeAttrName(result.name), build_func_type, + getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name)); +} + +void TFRFuncOp::print(OpAsmPrinter &p) { + function_interface_impl::printFunctionOp( + p, *this, /*isVariadic=*/false, getFunctionTypeAttrName(), + getArgAttrsAttrName(), getResAttrsAttrName()); +} + +} // namespace TFR +} // namespace mlir + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.cc.inc" + +namespace mlir { +namespace TFR { +namespace { +class ConvertConstToTensorConst : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(ConstantTensorOp cst_tensor_op, + PatternRewriter &rewriter) const override { + Location loc = cst_tensor_op.getLoc(); + Type out_type = cst_tensor_op.getType(); + Operation *new_cst = nullptr; + + ArrayAttr array; + if (matchPattern(cst_tensor_op.getArg(), m_Constant(&array))) { + llvm::DenseSet all_types; + for (auto it : array) { + TypedAttr typed_attr = llvm::dyn_cast(it); + if (!typed_attr) return failure(); + all_types.insert(typed_attr.getType()); + } + if (all_types.size() != 1) return failure(); + ShapedType new_out_type = RankedTensorType::get( + {static_cast(array.size())}, *all_types.begin()); + DenseElementsAttr attr = + DenseElementsAttr::get(new_out_type, array.getValue()); + new_cst = TF::ConstOp::create(rewriter, loc, new_out_type, attr); + if (isa(out_type)) { + new_cst = + CastOp::create(rewriter, loc, out_type, new_cst->getResult(0)); + } + rewriter.replaceOp(cst_tensor_op, new_cst->getResult(0)); + return success(); + } + + TypedAttr scalar; + if (matchPattern(cst_tensor_op.getArg(), m_Constant(&scalar))) { + Type new_out_type = RankedTensorType::get({}, scalar.getType()); + new_cst = TF::ConstOp::create(rewriter, loc, new_out_type, scalar); + if (isa(out_type)) { + new_cst = + CastOp::create(rewriter, loc, out_type, new_cst->getResult(0)); + } + rewriter.replaceOp(cst_tensor_op, new_cst->getResult(0)); + return success(); + } + return failure(); + } +}; + +inline bool isQuantizedType(Type type) { + auto tensor_type = llvm::dyn_cast(type); + return (tensor_type && + isa(tensor_type.getElementType())); +} + +class RemoveRedundantCast : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(CastOp cast_op, + PatternRewriter &rewriter) const override { + auto preceding_cast = + llvm::dyn_cast_or_null(cast_op.getArg().getDefiningOp()); + if (!preceding_cast) { + return failure(); + } + Value input = preceding_cast.getArg(); + Type input_type = input.getType(); + Type output_type = cast_op.getType(); + + // Preserve quantization information for intermediate tensors. + auto intermediate_type = preceding_cast.getType(); + if (isQuantizedType(intermediate_type) || isQuantizedType(output_type)) { + return failure(); + } + + auto input_tensor_type = llvm::dyn_cast(input_type); + auto output_tensor_type = llvm::dyn_cast(output_type); + if (!input_tensor_type || !output_tensor_type) { + return failure(); + } + + // Canonicalize two tfr.cast pairs with different element type to + // two tfr.casts with the same element type followed by a tf.Cast. + if ((input_tensor_type.getElementType() != + output_tensor_type.getElementType()) && + !isQuantizedType(input_type) && !isQuantizedType(output_type)) { + auto new_tfr_cast = TFR::CastOp::create( + rewriter, cast_op.getLoc(), + output_tensor_type.clone(input_tensor_type.getElementType()), + cast_op.getArg()); + rewriter.replaceOpWithNewOp(cast_op, output_type, + new_tfr_cast); + return success(); + } + + // If the two types are the same, the back-to-back tfr.cast ops can be + // removed. + if (input_type == output_type || isa(output_type)) { + rewriter.replaceOp(cast_op, {input}); + return success(); + } + + // If the rank of the input tensor isn't ranked, we replace the pair + // with tf.EnsureShape op so it can be removed after shape inference or + // confirmed at runtime. + if (isa(input_type)) { + auto shape = llvm::cast(output_type).getShape(); + auto shape_attr = TF::ShapeAttr::get(rewriter.getContext(), shape); + rewriter.replaceOpWithNewOp(cast_op, output_type, + input, shape_attr); + return success(); + } + + return failure(); + } +}; + +class GetTensorShape : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(GetShapeOp shape_op, + PatternRewriter &rewriter) const override { + Operation *preceding_op = shape_op.getArg().getDefiningOp(); + if (auto cast_op = llvm::dyn_cast_or_null(preceding_op)) { + // replace this pair by shape.shape_of, so the folding works. + rewriter.replaceOpWithNewOp(shape_op, cast_op.getArg()); + return success(); + } + return failure(); + } +}; + +class RemoveRedundantGetElement : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(GetElementOp ge_op, + PatternRewriter &rewriter) const override { + IntegerAttr index; + if (!matchPattern(ge_op.getIndex(), m_Constant(&index))) { + return failure(); + } + auto preceding_build_list = llvm::dyn_cast_or_null( + ge_op.getTensorList().getDefiningOp()); + if (!preceding_build_list || + preceding_build_list.getNumOperands() <= index.getInt()) { + return failure(); + } + Value input = preceding_build_list.getOperand(index.getInt()); + Type output_type = ge_op.getType(); + if (input.getType() != output_type && + !isa(output_type)) { + return failure(); + } + rewriter.replaceOp(ge_op, {input}); + return success(); + } +}; + +class RemoveRedundantGetLength : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(GetLengthOp gl_op, + PatternRewriter &rewriter) const override { + auto preceding_build_list = llvm::dyn_cast_or_null( + gl_op.getTensorList().getDefiningOp()); + if (!preceding_build_list) { + return failure(); + } + int64_t num_tensors = preceding_build_list.getNumOperands(); + rewriter.replaceOpWithNewOp( + gl_op, rewriter.getIndexAttr(num_tensors)); + return success(); + } +}; + +class BuildConstantListAsAttr : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(BuildListOp bl_op, + PatternRewriter &rewriter) const override { + SmallVector array_list; + array_list.reserve(bl_op.getNumOperands()); + for (const auto &operand : bl_op.getOperands()) { + Attribute array_elt; + if (!matchPattern(operand, m_Constant(&array_elt))) { + return failure(); + } + array_list.push_back(array_elt); + } + auto array_attr = rewriter.getArrayAttr(array_list); + rewriter.replaceOpWithNewOp(bl_op, array_attr); + return success(); + } +}; + +quant::QuantizedType getQuantizedElementType(CastOp cast_op) { + if (!cast_op || !cast_op.getInputElementType()) { + return {}; + } + return llvm::dyn_cast( + llvm::cast(cast_op.getInputElementType()).getValue()); +} + +class RemoveRawDataOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(TFRQuantRawDataOp raw_data_op, + PatternRewriter &rewriter) const override { + auto preceding_op = raw_data_op.getInput().getDefiningOp(); + if (isa(preceding_op)) { + return rewritePrecedingListOp(raw_data_op, rewriter); + } + + auto preceding_cast = dyn_cast_or_null(preceding_op); + if (!preceding_cast || !getQuantizedElementType(preceding_cast)) { + return failure(); + } + // If there are redundant casts, hoist output of raw data op originating op. + if (preceding_cast.getArg().getDefiningOp()) { + auto redundant_cast = preceding_cast.getArg().getDefiningOp(); + if (!redundant_cast || redundant_cast.getArg().getType() != + preceding_cast.getOut().getType()) { + return failure(); + } + raw_data_op.getOutput().replaceAllUsesWith(redundant_cast.getArg()); + } else { + // If the argument of cast op is input, then simply remove the RawData op. + raw_data_op.getOutput().replaceAllUsesWith(preceding_cast.getOut()); + } + return success(); + } + + LogicalResult rewritePrecedingListOp(TFRQuantRawDataOp raw_data_op, + PatternRewriter &rewriter) const { + llvm::SmallVector new_list_values; + auto preceding_list = raw_data_op.getInput().getDefiningOp(); + for (Value operand : preceding_list.getTensors()) { + auto preceding_cast = operand.getDefiningOp(); + if (!preceding_cast || !getQuantizedElementType(preceding_cast)) { + return failure(); + } + + // This function currently only supports the case with redundant casts. + auto redundant_cast = preceding_cast.getArg().getDefiningOp(); + if (!redundant_cast || redundant_cast.getArg().getType() != + preceding_cast.getOut().getType()) { + return failure(); + } + + new_list_values.push_back(redundant_cast.getArg()); + } + + auto new_list = + BuildListOp::create(rewriter, raw_data_op.getLoc(), + preceding_list.getType(), new_list_values); + raw_data_op.getOutput().replaceAllUsesWith(new_list.getOut()); + return success(); + } +}; + +class RemoveQParamsOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(TFRQuantQParamsOp qparams_op, + PatternRewriter &rewriter) const override { + auto cast_op = dyn_cast(qparams_op.getInput().getDefiningOp()); + auto cast_qtype = getQuantizedElementType(cast_op); + if (!cast_qtype) { + return failure(); + } + + TF::ConstOp scale_op; + TF::ConstOp zp_op; + + // Reads quantization parameters from the quantized type, and converts + // them to constants. + rewriter.setInsertionPoint(qparams_op); + Location loc = qparams_op->getLoc(); + if (auto qtype = llvm::dyn_cast(cast_qtype)) { + scale_op = TF::ConstOp::create( + rewriter, loc, RankedTensorType::get({}, rewriter.getF32Type()), + rewriter.getF32FloatAttr(qtype.getScale())); + zp_op = TF::ConstOp::create( + rewriter, loc, RankedTensorType::get({}, rewriter.getI32Type()), + rewriter.getI32IntegerAttr(qtype.getZeroPoint())); + } else if (auto qtype = llvm::dyn_cast( + cast_qtype)) { + SmallVector scales(qtype.getScales().begin(), + qtype.getScales().end()); + SmallVector zps(qtype.getZeroPoints().begin(), + qtype.getZeroPoints().end()); + const size_t num_channels = qtype.getScales().size(); + + auto scales_type = RankedTensorType::get( + {static_cast(num_channels)}, rewriter.getF32Type()); + auto scales_attr = + DenseElementsAttr::get(scales_type, llvm::ArrayRef(scales)); + scale_op = TF::ConstOp::create(rewriter, loc, scales_attr); + + auto zps_type = RankedTensorType::get( + {static_cast(num_channels)}, rewriter.getI32Type()); + auto zps_attr = DenseElementsAttr::get(zps_type, llvm::ArrayRef(zps)); + zp_op = TF::ConstOp::create(rewriter, loc, zps_attr); + } + if (!scale_op || !zp_op) { + return failure(); + } + auto scale_cast = CastOp::create( + rewriter, loc, qparams_op.getScale().getType(), scale_op.getOutput()); + auto zp_cast = CastOp::create(rewriter, loc, qparams_op.getZp().getType(), + zp_op.getOutput()); + + qparams_op.getScale().replaceAllUsesWith(scale_cast.getOut()); + qparams_op.getZp().replaceAllUsesWith(zp_cast.getOut()); + return success(); + } +}; + +// TODO(b/193731721): Migrate tfr_ builtin canonicalizations to LowerTFROpPass +class RemoveScaleFactorOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + // Replace quant_scale_factor with constant tensor equivalent to + // TFR_ConstantTensorOp ( + // ConstantOp (ConstAttr tfr.cast + // * in_scale[1] (filter scale): float scalar/vector + // (per-tensor vs per-channel) quantization, given by tf.Const -> tfr.cast + LogicalResult matchAndRewrite(TFRQuantScaleFactorOp scale_factor_op, + PatternRewriter &rewriter) const override { + auto out_scale_op = + scale_factor_op.getOutScale().getDefiningOp(); + if (!out_scale_op) { + return failure(); + } + const double out_scale = + llvm::cast(out_scale_op.getValue()).getValueAsDouble(); + + auto in_scales_op = + scale_factor_op.getInScales().getDefiningOp(); + if (!in_scales_op || in_scales_op.getNumOperands() != 2) { + // BuildListOp is variadic, but we require two values: input_scale + // and filter_scale. + return failure(); + } + + auto in_scale_op = in_scales_op.getOperand(0).getDefiningOp(); + if (!in_scale_op) { + return failure(); + } + + DenseFPElementsAttr in_scale_attr; + if (!matchPattern(in_scale_op.getArg(), m_Constant(&in_scale_attr)) || + in_scale_attr.size() != 1) { + return failure(); + } + const float in_scale = in_scale_attr.getValues()[0]; + auto filter_scale_op = in_scales_op.getOperand(1).getDefiningOp(); + if (!filter_scale_op) { + return failure(); + } + DenseFPElementsAttr filter_scale_attr; + if (!matchPattern(filter_scale_op.getArg(), + m_Constant(&filter_scale_attr))) { + return failure(); + } + + // The shape of scale_type is {} (rank 0) for per-tensor quantized tensor, + // and {num_channels} (rank 1) for per-channel quantized one. + auto scale_type = + llvm::dyn_cast(filter_scale_attr.getType()); + if (scale_type.getRank() != 0 && scale_type.getRank() != 1) { + return failure(); + } + SmallVector scale_factors; + scale_factors.reserve(filter_scale_attr.size()); + for (auto value : filter_scale_attr.getValues()) { + scale_factors.push_back(in_scale * value.convertToFloat() / out_scale); + } + rewriter.setInsertionPoint(scale_factor_op); + const Location loc = scale_factor_op->getLoc(); + auto result_scale_op = TF::ConstOp::create( + rewriter, loc, + DenseElementsAttr::get(scale_type, llvm::ArrayRef(scale_factors))); + auto result_scale_cast_op = CastOp::create( + rewriter, loc, scale_factor_op.getType(), result_scale_op.getOutput()); + scale_factor_op.getScaleFactor().replaceAllUsesWith( + result_scale_cast_op.getOut()); + return success(); + } +}; + +class RemoveRescaleOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + // Replace quant_rescale (input, scale, zp) with + // tf.Cast(tf.Round(tf.Cast(input, f32) * scale) + tf.Cast(zp, f32), i32) + LogicalResult matchAndRewrite(TFRQuantRescaleOp rescale_op, + PatternRewriter &rewriter) const override { + Value input = rescale_op.getInput(); + Value scale = rescale_op.getScale(); + Value zp = rescale_op.getZp(); + + const Location loc = rescale_op->getLoc(); + const auto result_types = rescale_op->getResultTypes(); + auto c_false = + arith::ConstantOp::create(rewriter, loc, rewriter.getBoolAttr(false)); + TypeAttr f32_attr = TypeAttr::get(rewriter.getF32Type()); + TFRAttrType output_type = TFRAttrType::get(rewriter.getContext()); + auto constant_f32_op = + ConstOp::create(rewriter, loc, output_type, f32_attr); + TypeAttr i32_attr = TypeAttr::get(rewriter.getI32Type()); + auto constant_i32_op = + ConstOp::create(rewriter, loc, output_type, i32_attr); + + IntegerAttr zp_attr; + if (!matchPattern(zp, m_Constant(&zp_attr))) { + return failure(); + } + rewriter.setInsertionPoint(zp.getDefiningOp()); + auto zp_tensor = TF::ConstOp::create( + rewriter, loc, RankedTensorType::get({}, zp.getType()), zp_attr); + auto zp_cast = + CastOp::create(rewriter, loc, rewriter.getType(), + zp_tensor.getOutput()); + + rewriter.setInsertionPoint(rescale_op); + auto cast_input_to_float_op = + CallOp::create(rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__cast"), + ArrayRef{input, constant_f32_op, c_false}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + auto input_x_scale_op = CallOp::create( + rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__mul"), + ArrayRef{cast_input_to_float_op.getResult(0), scale}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + auto round_rescaled_op = + CallOp::create(rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__round"), + ArrayRef{input_x_scale_op->getResult(0)}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + auto cast_zp_to_float_op = + CallOp::create(rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__cast"), + ArrayRef{zp_cast, constant_f32_op, c_false}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + auto recentered_op = + CallOp::create(rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__add"), + ArrayRef{round_rescaled_op->getResult(0), + cast_zp_to_float_op->getResult(0)}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + auto cast_output_to_i32 = CallOp::create( + rewriter, loc, result_types, + SymbolRefAttr::get(rewriter.getContext(), "tf__cast"), + ArrayRef{recentered_op->getResult(0), constant_i32_op, c_false}, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + rescale_op.getOutput().replaceAllUsesWith(cast_output_to_i32.getResult(0)); + return success(); + } +}; + +} // namespace + +void ConstantTensorOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void CastOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void GetShapeOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void GetElementOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void GetLengthOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void BuildListOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void TFRQuantRawDataOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void TFRQuantQParamsOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void TFRQuantRescaleOp::getCanonicalizationPatterns(RewritePatternSet &results, + MLIRContext *context) { + results.add(context); +} + +void TFRQuantScaleFactorOp::getCanonicalizationPatterns( + RewritePatternSet &results, MLIRContext *context) { + results.add(context); +} + +OpFoldResult TFR::EqualOp::fold(FoldAdaptor adaptor) { + auto operands = adaptor.getOperands(); + assert(operands.size() == 2 && "equal op has two operands"); + auto ctx = getContext(); + if (operands[0] == operands[1]) return BoolAttr::get(ctx, true); + return BoolAttr::get(ctx, false); +} + +OpFoldResult ConstOp::fold(FoldAdaptor adaptor) { + auto operands = adaptor.getOperands(); + (void)operands; + assert(operands.empty() && "constant has no operands"); + + // Return the held attribute value. + return getValue(); +} + +// CallableOpInterface +Region *TFRFuncOp::getCallableRegion() { + return isExternal() ? nullptr : &getBody().front(); +} + +//===----------------------------------------------------------------------===// +// Dialect type definitions +//===----------------------------------------------------------------------===// + +// Parses a TFR type. +// tfr_type ::= tensor_type | tensor_list_type | attr_type +// string_list ::= `[` string-literal (, string-literal)+ `]` +// tensor_type ::= `tensor` +// | `tensor<` (string-literal | string_list) '>' +// tensor_list_type ::= `tensor_list` +// | `tensor_list<` (string-literal | string_list) '>' +// attr_type ::= `attr` +Type TFRDialect::parseType(DialectAsmParser &parser) const { + Location loc = parser.getEncodedSourceLoc(parser.getNameLoc()); + MLIRContext *ctx = loc.getContext(); + + StringRef typeNameSpelling; + if (failed(parser.parseKeyword(&typeNameSpelling))) return {}; + llvm::SmallVector attrs; + if (succeeded(parser.parseOptionalLess())) { + bool l_square_parsed = false; + if (succeeded(parser.parseOptionalLSquare())) { + l_square_parsed = true; + } + + do { + StringRef attr; + if (failed(parser.parseKeyword(&attr))) return {}; + attrs.push_back(StringAttr::get(ctx, attr)); + } while (succeeded(parser.parseOptionalComma())); + + if (l_square_parsed && failed(parser.parseRSquare())) { + parser.emitError(parser.getNameLoc(), "expected ']'"); + } + + if (failed(parser.parseGreater())) { + parser.emitError(parser.getNameLoc(), "expected '>'"); + } + } + + if (typeNameSpelling == "tensor") { + return TFRTensorType::getChecked(attrs, loc); + } else if (typeNameSpelling == "tensor_list") { + return TFRTensorListType::getChecked(attrs, loc); + } else if (typeNameSpelling == "attr") { + return TFRAttrType::getChecked(loc, loc.getContext()); + } else { + parser.emitError(parser.getNameLoc(), "unknown type " + typeNameSpelling); + return {}; + } +} + +void TFRDialect::printType(Type type, DialectAsmPrinter &os) const { + llvm::ArrayRef attrs; + + if (isa(type)) { + os << "attr"; + return; + } + if (auto tensor_ty = llvm::dyn_cast(type)) { + attrs = tensor_ty.getAttrKeys(); + os << "tensor"; + } else if (auto tensor_list_ty = llvm::dyn_cast(type)) { + attrs = tensor_list_ty.getAttrKeys(); + os << "tensor_list"; + } else { + llvm_unreachable("Unhandled tfr type"); + } + + if (attrs.empty()) return; + os << "<"; + + if (attrs.size() > 1) { + os << "["; + } + + llvm::interleaveComma(attrs, os, + [&](StringAttr attr) { os << attr.getValue(); }); + + if (attrs.size() > 1) { + os << "]"; + } + os << ">"; +} + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..38959728a2284d15115a6303444f0cba5cecaf23 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.h @@ -0,0 +1,66 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_OPS_H_ + +#include "llvm/ADT/StringSet.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/Interfaces/CallInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/ControlFlowInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/FunctionInterfaces.h" // from @llvm-project +#include "mlir/Interfaces/InferTypeOpInterface.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace mlir { +namespace TFR { + +constexpr char kAttrArgumentNameAttr[] = "tfr.name"; +constexpr char kAttrArgumentDefaultAttr[] = "tfr.default"; +constexpr char kAttrArgumentTypeAttr[] = "tfr.type"; + +class TFRDialect : public Dialect { + public: + explicit TFRDialect(MLIRContext *context); + + static StringRef getDialectNamespace() { return "tfr"; } + + Operation *materializeConstant(OpBuilder &builder, Attribute value, Type type, + Location loc) override; + + // Parse a type registered to this dialect. + Type parseType(DialectAsmParser &parser) const override; + + // Prints a type registered to this dialect. + void printType(Type ty, DialectAsmPrinter &os) const override; +}; + +} // namespace TFR +} // namespace mlir + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..41048b249799e390fc0fde8580af465e64a06dcd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_ops.td @@ -0,0 +1,615 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is the operation definition file for TFR + +#ifndef DIALECT_TFR_OPS_ +#define DIALECT_TFR_OPS_ + +include "mlir/Dialect/Shape/IR/ShapeBase.td" +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/FunctionInterfaces.td" +include "mlir/Dialect/Quant/IR/QuantBase.td" +include "mlir/IR/SymbolInterfaces.td" +include "mlir/Interfaces/CallInterfaces.td" +include "mlir/Interfaces/ControlFlowInterfaces.td" +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td" + +//===----------------------------------------------------------------------===// +// Dialect +//===----------------------------------------------------------------------===// + +def TFR_Dialect : Dialect { + let name = "tfr"; + + let description = [{ + The TensorFlow Composition dialect. + }]; + + let cppNamespace = "::mlir::TFR"; +} + +//===----------------------------------------------------------------------===// +// Type classes +//===----------------------------------------------------------------------===// + +// tensor argument types +class TFR_Type : DialectType($_self)">, + "TFR " # name #" type">, + BuildableType<"$_builder.getType()">; +def TFR_TensorType : TFR_Type<"TFRTensor">; +def TFR_TensorListType : TFR_Type<"TFRTensorList">; +def TFR_AllTensorTypes : Type, "all tensor related types">; + +// attribute argument types +def TFR_AttrType : TFR_Type<"TFRAttr">; +def TFR_AttrScalarType: TypeAlias; +def TFR_AttrVectorType : VectorOfNonZeroRankOf<[TF_ElementType, TFR_AttrType]>; +def TFR_AllAttrTypes : Type, "all attribute related types">; + +// all allowed arguments types +def TFR_allowedArgType : Type, "allowed tfr.call operand types">; + +def TFR_allowedConstValues : Attr, "allowed tfr.constant value"> { + let storageType = "Attribute"; + let returnType = "Attribute"; + let convertFromStorage = "$_self"; + let constBuilderCall = "$0"; +} + +// all allowed result types +def TFR_allowedResultType : TypeAlias; + +// standard tensor type and tfr.tensor types can be casted to each other. +def TFR_singleTensorType : Type.predicate]>, "single tensor or tfr.tensor type">; + +// all allowed build list input types +def TFR_allowedBuiltListType : Type, "single tfr.tensor or tensor element type">; + +// all allowed build list result types +def TFR_allowedListResultType : Type, "tfr.tensor_list or tfr.attr type">; + +//===----------------------------------------------------------------------===// +// Op classes +//===----------------------------------------------------------------------===// + +class TFR_Op traits> : + Op; + +def TFR_CallOp : TFR_Op<"call", [CallOpInterface]> { + let description = [{ + The `call` operation represents a direct call to a function that is within + the same symbol scope as the callee. The operands and result types of the + call must match the specified function type. The callee is encoded as a + symbol reference attribute named "callee". + + Example: + + ```mlir + %2 = tfr.call @my_add(%0, %1) : (tfr.tensor, f32) -> tfr.tensor_list + ``` + + Note that the operands of the `call` operation can only be with tfr.tensor, + tfr.tensor_list, tfr.attr and mlir float and integer types. The results of + the `call` operation can only be with tfr.tensor and tfr.tensor_list types. + }]; + + let arguments = (ins + FlatSymbolRefAttr:$callee, + Variadic:$args, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs); + + let results = (outs + Variadic:$outs); + + let extraClassDeclaration = [{ + // Get the argument operands to the called function. + operand_range getArgOperands() { return getArgs(); } + MutableOperandRange getArgOperandsMutable() { + return getArgsMutable(); + } + + // Return the callee of this operation. + CallInterfaceCallable getCallableForCallee() { return getCalleeAttr(); } + // Sets the callee from the callable + void setCalleeFromCallable(CallInterfaceCallable callee); + }]; + + let assemblyFormat = [{ + $callee `(` $args `)` attr-dict `:` functional-type($args, results) + }]; +} + +def TFR_CastOp : TFR_Op<"cast", [Pure]> { + let description = [{ + The `cast` operation converts the operand with built-in tensor type to + tfr.tensor type, or vice versa. + + Example: + + ```mlir + %1 = tfr.cast(%0) : tensor -> !tfr.tensor + %3 = tfr.cast(%1) : !tfr.tensor -> tensor + ``` + }]; + + let arguments = (ins TFR_singleTensorType:$arg); + + let results = (outs TFR_singleTensorType:$out); + + let extraClassDeclaration = [{ + // Return element type of the input tensor type. Only available when the + // input is a MLIR built-in tensor type. + Attribute getInputElementType() { + if (auto ty = llvm::dyn_cast(getArg().getType())) { + return TypeAttr::get(ty.getElementType()); + } + return {}; + } + }]; + + let hasCanonicalizer = 1; +} + +def TFR_GetShapeOp : TFR_Op<"get_shape", [Pure]> { + let description = [{ + The `get_shape` operation gets the shape of a tfr.tensor and returns + !shape.shape type. + + Example: + + ```mlir + %1 = "tfr.get_shape"(%0) : !tfr.tensor -> !shape.shape + %1 = tfr.get_shape %0 -> !shape.shape + ``` + }]; + + let arguments = (ins TFR_TensorType:$arg); + + let results = (outs Shape_ShapeType:$out); + + let assemblyFormat = "$arg attr-dict `->` type($out)"; + + let hasCanonicalizer = 1; +} + +def TFR_GetElementTypeOp : TFR_Op<"get_element_type", [Pure]> { + let description = [{ + The `get_element_type` operation gets the element type of a tfr.tensor and + returns !tfr.attr. + + Example: + + ```mlir + %1 = "tfr.get_element_type"(%0) : !tfr.tensor -> !tfr.attr + %1 = tfr.get_element_type %0 -> !tfr.attr + ``` + }]; + + let arguments = (ins TFR_TensorType:$arg); + + let results = (outs TFR_AttrType:$out); + + let assemblyFormat = "$arg attr-dict `->` type($out)"; +} + +def TFR_EqualOp : TFR_Op<"equal", [Pure, SameTypeOperands]> { + let description = [{ + The `equal` operation compares the values of the tfr.attr type arguments. + The operation returns an i1 boolean indicating if the two values are the + same. + Example: + + ```mlir + %x = tfr.equal %lhs, %rhs -> i1 + %x = "tfr.equal"(%lhs, %rhs) : (!tfr.attr, !tfr.attr) -> i1 + ``` + }]; + + let arguments = (ins + TFR_AttrType:$lhs, + TFR_AttrType:$rhs + ); + let results = (outs BoolLike:$result); + + let hasFolder = 1; + + let assemblyFormat = "$lhs `,` $rhs attr-dict `->` type($result)"; +} + +def TFR_ConstOp : TFR_Op<"constant", [ConstantLike, Pure]> { + let description = [{ + The `attr` operation stores TF op's attribute, which doesn't support + arithmetic operations. + + Example: + + ```mlir + %1 = "tfr.constant"() { value: i32 } : () -> !tfr.attr + %2 = "tfr.constant"() { value: [i32, f32] } : () -> !tfr.attr + %3 = tfr.constant [i32, f32] -> !tfr.attr + %4 = tfr.constant f32 -> !tfr.attr + ``` + }]; + + let arguments = (ins TFR_allowedConstValues:$value); + + let results = (outs TFR_AttrType:$out); + + let hasFolder = 1; + + let builders = [ + OpBuilder<(ins "Attribute":$value), + [{ + auto* ctx = value.getContext(); + $_state.addAttribute("value", value); + $_state.addTypes(TFRAttrType::get(ctx)); + }]> + ]; + + let assemblyFormat = [{ + $value attr-dict `->` type($out) + }]; +} + +def TFR_ConstantTensorOp : TFR_Op<"constant_tensor", [Pure]> { + let description = [{ + The `constant_tensor` operation converts the operand with non-built-in + tensor type to built-in tensor type or tfr.tensor type. If it is built-in + tensor type, the shape shouldn't be changed during the conversion. + + Example: + + ```mlir + %1 = tfr.constant_tensor(%0) : f32 -> tensor + %3 = tfr.constant_tensor(%2) : vector<1xf32> -> tensor<1xf32> + ``` + }]; + + let arguments = (ins TFR_AllAttrTypes:$arg); + + let results = (outs TFR_singleTensorType:$out); + + let hasCanonicalizer = 1; + + let hasVerifier = 1; +} + +def TFR_GetElementOp : TFR_Op<"get_element", [Pure]> { + let description = [{ + The `get_element` operation extracts one tfr.tensor element from a + tfr.tensor_list. + + Example: + + ```mlir + %2 = tfr.get_element %1[%0] : (tfr.tensor, index) -> tfr.tensor + ``` + }]; + + let arguments = (ins + TFR_TensorListType:$tensor_list, + Index:$index); + + let results = (outs TFR_TensorType:$out); + + let hasCanonicalizer = 1; + + let assemblyFormat = [{ + $tensor_list `[` $index `]` attr-dict `:` + `(` type($tensor_list) `,` type($index) `)` `->` type($out) + }]; +} + +def TFR_BuildListOp : TFR_Op<"build_list", [Pure]> { + let description = [{ + The `build_list` operation builds a tensor list from a list of tensors, or + an tfr.attr from a list of scalars. + + Example: + + ```mlir + %3 = tfr.build_list(%2, %1, %0) : + (tfr.tensor, tfr.tensor, tfr.tensor) -> tfr.tensor_list + %3 = tfr.build_list(%2, %1, %0) : (i32, i32, i32) -> tfr.attr + ``` + }]; + + let arguments = (ins Variadic:$tensors); + + let results = (outs TFR_allowedListResultType:$out); + + let hasCanonicalizer = 1; +} + +def TFR_GetLengthOp : TFR_Op<"get_length", [Pure]> { + let description = [{ + The `get_length` operation returns the number of tensors for a + tfr.tensor_list. + + Example: + + ```mlir + %2 = tfr.get_length(%1) : tfr.tensor -> index + %2 = tfr.get_length %1 -> index + ``` + }]; + + let arguments = (ins TFR_TensorListType:$tensor_list); + + let results = (outs Index:$out); + + let hasCanonicalizer = 1; + + let assemblyFormat = [{ + $tensor_list attr-dict `->` type($out) + }]; +} + +//===----------------------------------------------------------------------===// +// Function related classes +//===----------------------------------------------------------------------===// + +def TFR_TFRFuncOp : TFR_Op<"func", [HasParent<"ModuleOp">, + FunctionOpInterface, + IsolatedFromAbove, Symbol]> { + let summary = "TFR Function defines a composition of other ops"; + + let description = [{ + Defines a function that can be used to decompose an TF function call to + the invocation of a set of other TF ops. + + Syntax: + + ``` + op ::= `tfr.func` visibility? symbol-ref-id `(` argument-list `)` (`->` + function-result-list)? function-attributes? region + ``` + + Example: + + ```mlir + tfr.func @foo(%arg0: !tfr.tensor, %arg1: !tfr.tensor_list, + %arg2: int {tfr.name="T", tfr.default=1}) + attributes {qux: "quux"} { + tfr.return + } + ``` + + Note the arguments are ordered by the following rule: + tfr.tensor > tfr.tensor_list > tfr.attr/i32/..., + and only one trfr.tensor_list argument is allowed. + }]; + + let arguments = (ins + TypeAttrOf:$function_type, + StrAttr:$sym_name, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + + let results = (outs); + + // When the regions is empty, the tfr.func is an external function and used + // to model the element type constraints of the tf op. Otherwise, there is one + // region containing the composition. + let regions = (region VariadicRegion:$body); + + let extraClassDeclaration = [{ + LogicalResult verifyType() { return success(); } + + mlir::Region *getCallableRegion(); + + /// Returns the argument types of this function. + ArrayRef getArgumentTypes() { return getFunctionType().getInputs(); } + + /// Returns the result types of this function. + ArrayRef getResultTypes() { return getFunctionType().getResults(); } + + // Get the names of all defined attributes, including both derived and + // non-derived ones. + llvm::StringSet<> getDefinedAttributeNames() { + llvm::StringSet<> all_attrs; + for (auto& attr : (*this)->getAttrs()) { + all_attrs.insert(attr.getName().strref()); + } + for (const auto& operand : llvm::enumerate(getArgumentTypes())) { + if (auto attr_name = getArgAttrOfType( + operand.index(), kAttrArgumentNameAttr)) { + all_attrs.insert(attr_name.getValue()); + } + } + return all_attrs; + } + }]; + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def TFR_TFRReturnOp : TFR_Op<"return", [HasParent<"TFRFuncOp">, Pure, + ReturnLike, Terminator]> { + let description = [{ + A terminator operation for regions that appear in the body of `tfr.func` + functions. The operands to the `tfr.return` are the result values returned + by an invocation of the `tfr.func`. + + Note that only the tfr.tensor and tfr.tensor_list can be returned. + }]; + + let arguments = (ins Variadic:$arguments); + + let assemblyFormat = "attr-dict ($arguments^ `:` type($arguments))?"; +} + +//===----------------------------------------------------------------------===// +// Quantization related operations +//===----------------------------------------------------------------------===// + +def TFR_TFRQuantActRangeOp : TFR_Op<"quant_act_range", [Pure]> { + let description = [{ + The `quant_act_range` returns the a pair of integers to indicate the fixed + range for the fused activation `act` with the quantization defined by the + `scale` and `zero point`. Currently, the allowed activations are + `NONE`, `RELU`, `RELU6` and `RELU_N1_TO_1`. + + Example: + + ```mlir + %3, %4 = tfr.quant_act_range(%2, %1, %0) : + (tfr.attr, float, i64) -> (tfr.tensor, tfr.tensor) + ``` + }]; + + let arguments = (ins + TFR_AttrType:$act, + F32:$scale, + I64:$zp); + + let results = (outs TFR_TensorType:$min, TFR_TensorType:$max); + + let assemblyFormat = [{ + `(` $act `,` $scale `,` $zp `)` attr-dict `:` functional-type(operands, results) + }]; +} + +def TFR_TFRQuantRescaleOp : TFR_Op<"quant_rescale", [Pure]> { + let description = [{ + The `quant_rescale` rescales the elements of the integer tensor by the + floating-point rescale factor. This op needs to be legalized to the preferred + operations of the backends. + + Example: + + ```mlir + %3 = tfr.quant_rescale(%2, %1, %0) : + (tfr.tensor, tfr.tensor, i64) -> (tfr.tensor) + ``` + }]; + + let arguments = (ins + TFR_TensorType:$input, + TFR_TensorType:$scale, + I64:$zp); + + let results = (outs TFR_TensorType:$output); + + let assemblyFormat = [{ + `(` $input `,` $scale `,` $zp `)` attr-dict `:` functional-type(operands, results) + }]; + + let hasCanonicalizer = 1; +} + +def TFR_TFRQuantRawDataOp : TFR_Op<"quant_raw_data", [ + Pure, + SameOperandsAndResultType]> { + let description = [{ + The `quant_raw_data` removes the quantization parameter from the intput + tensor(s). + + Example: + + ```mlir + %3 = tfr.quant_raw_data(%0) : (tfr.tensor) -> (tfr.tensor) + ``` + }]; + + let arguments = (ins TFR_AllTensorTypes:$input); + + let results = (outs TFR_AllTensorTypes:$output); + + let assemblyFormat = [{ + `(` $input `)` attr-dict `:` functional-type($input, results) + }]; + + let hasCanonicalizer = 1; +} + +def TFR_TFRQuantQParamsOp : TFR_Op<"quant_qparam", [Pure]> { + let description = [{ + The `quant_qparam` returns the quantization parameter of the input + tensors. + + Example: + + ```mlir + %3 = tfr.quant_qparam(%0) : (tfr.tensor) -> (float, tfr.tensor) + ``` + }]; + + let arguments = (ins TFR_TensorType:$input); + + let results = (outs TFR_TensorType:$scale, TFR_TensorType:$zp); + + let assemblyFormat = [{ + `(` $input `)` attr-dict `:` functional-type($input, results) + }]; + + let hasCanonicalizer = 1; +} + + +def TFR_TFRQuantScaleFactorOp : TFR_Op<"quant_scale_factor", [Pure]> { + let description = [{ + The `quant_scale_factor` computes the effective scale factor according to the + output scale and input scales. + + Example: + + ```mlir + %3 = tfr.quant_scale_factor(%0) : (f32, tfr.tensor_list) -> (tfr.tensor) + ``` + }]; + + let arguments = (ins + F32:$out_scale, + TFR_TensorListType:$in_scales); + + let results = (outs TFR_TensorType:$scale_factor); + + let assemblyFormat = [{ + `(` $out_scale `,` $in_scales `)` attr-dict `:` functional-type(operands, results) + }]; + + let hasCanonicalizer = 1; +} + +#endif // DIALECT_TFR_OPS_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_types.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_types.h new file mode 100644 index 0000000000000000000000000000000000000000..acccb50ca38b908b3b667f164ca083d5a889f869 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/ir/tfr_types.h @@ -0,0 +1,124 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_TYPES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_TYPES_H_ + +#include +#include + +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/TypeSupport.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project + +namespace mlir { +namespace TFR { + +class TFRType : public Type { + public: + using Type::Type; + + static bool classof(Type type); +}; + +namespace detail { + +struct TFRTypeStorage final + : public TypeStorage, + public llvm::TrailingObjects { + using KeyTy = ArrayRef; + + explicit TFRTypeStorage(unsigned num_attrs) : num_attrs(num_attrs) {} + + static TFRTypeStorage* construct(TypeStorageAllocator& allocator, KeyTy key) { + // Allocate a new storage instance. + auto byteSize = TFRTypeStorage::totalSizeToAlloc(key.size()); + auto rawMem = allocator.allocate(byteSize, alignof(TFRTypeStorage)); + auto result = ::new (rawMem) TFRTypeStorage(key.size()); + + // Copy in the string attributes into the trailing storage. + std::uninitialized_copy(key.begin(), key.end(), + result->getTrailingObjects()); + return result; + } + + bool operator==(const KeyTy& attrs) const { return attrs == GetAttrs(); } + + KeyTy GetAttrs() const { return {getTrailingObjects(), num_attrs}; } + + unsigned num_attrs; +}; + +template +class TFRTypeImpl : public Type::TypeBase { + public: + using Base = Type::TypeBase; + using TFRBase = TFRTypeImpl; + using Base::Base; + + static Derived get(ArrayRef attrs, MLIRContext* context) { + return Base::get(context, attrs); + } + + static Derived getChecked(ArrayRef attrs, Location loc) { + return Base::getChecked(loc, loc.getContext(), attrs); + } + static Derived getChecked(function_ref emitError, + MLIRContext* context, ArrayRef attrs) { + return Base::getChecked(emitError, context, attrs); + } + + static Derived get(MLIRContext* context) { return get({}, context); } + + // TODO(fengliuai): fix the implementation + static LogicalResult verify(function_ref emitError, + ArrayRef attrs) { + return success(); + } + + ArrayRef getAttrKeys() { return Base::getImpl()->GetAttrs(); } +}; +} // namespace detail + +class TFRTensorType : public detail::TFRTypeImpl { + public: + using TFRBase::TFRBase; + static constexpr StringLiteral name = "tfr.tensor"; + static std::string getTypeName() { return "TFRTensorType"; } +}; + +class TFRTensorListType : public detail::TFRTypeImpl { + public: + using TFRBase::TFRBase; + static constexpr StringLiteral name = "tfr.tensor_list"; + static std::string getTypeName() { return "TFRTensorListType"; } +}; + +class TFRAttrType : public Type::TypeBase { + public: + using Base::Base; + static constexpr StringLiteral name = "tfr.attr"; + static std::string getTypeName() { return "TFRAttrType"; } +}; + +} // namespace TFR +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TFR_IR_TFR_TYPES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/canonicalize.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/canonicalize.cc new file mode 100644 index 0000000000000000000000000000000000000000..0ef9b576978ff767ca6f5767662448696a46a65e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/canonicalize.cc @@ -0,0 +1,177 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/Support/raw_ostream.h" +#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h" // from @llvm-project +#include "mlir/Dialect/Affine/LoopUtils.h" // from @llvm-project +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Inliner.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/compiler/mlir/tfr/passes/passes.h" + +//===----------------------------------------------------------------------===// +// Canonicalization patterns for the scf.for and scf.if ops. They are used to +// optimize the control flow in the tfr function. Technically, both patterns +// should be upstreamed to be part of the op definition. +// TODO(fengliuai): sync with the llvm upstream for both patterns. +// +namespace mlir { +namespace TFR { + +namespace { + +class UnrollSCFForOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(scf::ForOp for_op, + PatternRewriter &rewriter) const override { + Location loc = for_op.getLoc(); + APInt lower_bound, upper_bound, step; + if (!matchPattern(for_op.getLowerBound(), m_ConstantInt(&lower_bound)) || + !matchPattern(for_op.getUpperBound(), m_ConstantInt(&upper_bound)) || + !matchPattern(for_op.getStep(), m_ConstantInt(&step))) { + return failure(); + } + uint64_t trip_count = (upper_bound - lower_bound).sdiv(step).getZExtValue(); + if (trip_count <= 0) return failure(); + + // TODO(fengliuai): use loopUnrollByFactor once the iter_arg is supported + + Block *single_block = for_op.getBody(); + IRMapping mapping; + Value iv = for_op.getInductionVar(); + for (auto iter_op : + llvm::zip(for_op.getRegionIterArgs(), for_op.getInitArgs())) { + mapping.map(std::get<0>(iter_op), std::get<1>(iter_op)); + } + mapping.map(iv, for_op.getLowerBound()); + for (auto i = 0; i < trip_count; ++i) { + if (!iv.use_empty()) { + // iv' = iv + step * i; + Value iter = arith::ConstantIndexOp::create(rewriter, loc, i); + Value step_cst = + arith::ConstantIndexOp::create(rewriter, loc, step.getSExtValue()); + Value stride = arith::MulIOp::create(rewriter, loc, step_cst, iter); + Value iv_unroll = + arith::AddIOp::create(rewriter, loc, mapping.lookup(iv), stride); + mapping.map(iv, iv_unroll); + } + + Operation *terminator_op; + for (auto it = single_block->begin(); it != single_block->end(); ++it) { + terminator_op = rewriter.clone(*it, mapping); + } + // Map the block arguments to the yield results. + for (auto iter_op : llvm::zip(for_op.getRegionIterArgs(), + terminator_op->getOperands())) { + mapping.map(std::get<0>(iter_op), std::get<1>(iter_op)); + } + rewriter.eraseOp(terminator_op); + } + SmallVector returned; + for (Value arg : for_op.getRegionIterArgs()) { + returned.push_back(mapping.lookup(arg)); + } + rewriter.replaceOp(for_op, returned); + return success(); + } +}; + +// TODO(fengliuai): up stream this pattern. +class SimplifySCFIfOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + LogicalResult matchAndRewrite(scf::IfOp if_op, + PatternRewriter &rewriter) const override { + // Then branch + if (matchPattern(if_op.getCondition(), m_NonZero())) { + return InlineRegion(if_op.getLoc(), rewriter, if_op, + &if_op.getThenRegion()); + } + + // Else branch + if (matchPattern(if_op.getCondition(), m_Zero())) { + if (if_op.getElseRegion().empty()) { + // Remove the op + rewriter.eraseOp(if_op); + return success(); + } else { + return InlineRegion(if_op.getLoc(), rewriter, if_op, + &if_op.getElseRegion()); + } + } + + // Not a constant condition + return failure(); + } + + private: + LogicalResult InlineRegion(Location loc, PatternRewriter &rewriter, + Operation *inline_point, Region *region) const; +}; + +LogicalResult SimplifySCFIfOp::InlineRegion(Location loc, + PatternRewriter &rewriter, + Operation *inline_point, + Region *region) const { + InlinerInterface interface(loc.getContext()); + InlinerConfig config; + if (failed(inlineRegion(interface, config.getCloneCallback(), region, + inline_point, {}, inline_point->getResults(), loc, + /*shouldCloneInlinedRegion=*/true))) { + return failure(); + } + + // If the inlining was successful then erase the scf.if op. + rewriter.eraseOp(inline_point); + return success(); +} + +} // namespace + +void populateCanonicalizationPatterns(func::FuncOp func, + RewritePatternSet &patterns) { + MLIRContext *context = func.getContext(); + mlir::Dialect *tf = context->getLoadedDialect(); + // Load all official canonicalization patterns. Here we skip the + // canonicalization of the ops in the tf dialect, because they couldn't + // propagate the attributes correctly. These optimization will be played by + // bridge. + func->walk([&](Operation *op) { + if (op->getDialect() != tf) { + op->getRegisteredInfo()->getCanonicalizationPatterns(patterns, context); + } + }); + patterns.add(context); +} + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose.cc new file mode 100644 index 0000000000000000000000000000000000000000..fbf0273a9428777929a72e700fd4168312d30661 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose.cc @@ -0,0 +1,370 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "mlir/Transforms/Inliner.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_types.h" +#include "tensorflow/compiler/mlir/tfr/passes/passes.h" +#include "tensorflow/compiler/mlir/tfr/utils/utils.h" +#include "tensorflow/core/lib/monitoring/counter.h" + +namespace tensorflow { +namespace { + +auto* tf_core_op_expansion_op_counter = + monitoring::Counter<1>::New("/tensorflow/core/op_expansion/op_counter", + "The number of composite op expanded.", "name"); +} + +void IncreaseOpExpansionExecuteCounterByOne(const std::string& op_name) { + tf_core_op_expansion_op_counter->GetCell(op_name)->IncrementBy(1); +} + +} // namespace tensorflow + +//===----------------------------------------------------------------------===// +// The pass to decompose unregistered TF ops with the TFR compose function. +// +namespace mlir { +namespace TFR { + +namespace { + +// Quantize the float value based on given scale and zero point attributes. +IntegerAttr Quantize(float value, Attribute scale_attr, Attribute zp_attr, + OpBuilder builder) { + double scale = mlir::cast(scale_attr).getValueAsDouble(); + int64_t zp = mlir::cast(zp_attr).getInt(); + + int quantized = static_cast(std::round(value / scale) + zp); + quantized = + std::min(quantized, static_cast(std::numeric_limits::max())); + quantized = + std::max(quantized, static_cast(std::numeric_limits::min())); + return builder.getI32IntegerAttr(quantized); +} + +// Decompose the TF ops with the registered composition library. +class DecomposeTFOpsPass + : public PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(DecomposeTFOpsPass) + + explicit DecomposeTFOpsPass(std::optional external_tfr_module) + : external_tfr_module_(external_tfr_module) {} + + StringRef getArgument() const final { return "tfr-decompose"; } + + StringRef getDescription() const final { + return "Decompose TF ops with the registered composition library."; + } + + void runOnOperation() override; + + private: + // Apply canonicalization, mainly constant folding, on the function. + void ApplyCanonicalization(); + + // Rewrite unregistered TF ops to TFR func call ops. Return failure if all the + // ops are registered or the compose function doesn't exist. + LogicalResult RewriteUnregisteredTFOps(); + + // Inline the TFR func call ops. + LogicalResult InlineTFRFuncCalls(); + + // Optional external symbol table to look up the TFR function. + std::optional external_tfr_module_; +}; + +#include "tensorflow/compiler/mlir/tfr/passes/generated_decompose.inc" + +void DecomposeTFOpsPass::ApplyCanonicalization() { + func::FuncOp func = getOperation(); + RewritePatternSet patterns(&getContext()); + + populateWithGenerated(patterns); + populateCanonicalizationPatterns(func, patterns); + + (void)applyPatternsGreedily(func, std::move(patterns)); +} + +LogicalResult DecomposeTFOpsPass::RewriteUnregisteredTFOps() { + func::FuncOp func = getOperation(); + SymbolTable table(external_tfr_module_.has_value() + ? *external_tfr_module_ + : func->getParentOfType()); + OpBuilder builder(func); + bool changed = false; + func.walk([&table, &builder, &changed](Operation* op) { + // Only the un-registered ops requires decomposition. The remaining ones + // either will be constant folded or lowered by the rules defined in the + // bridge. + if (op->isRegistered()) { + return WalkResult::advance(); + } + + // Find out the compose function + auto compose_func_name = GetComposeFuncName(op->getName().getStringRef()); + auto compose_func = table.lookup(compose_func_name); + if (!compose_func || compose_func.isExternal()) { + // There are no decomposition methods defined for this op, skip. + return WalkResult::advance(); + } + + // Make sure all the attributes are valid. An attribute is valid when it is + // in the signature or it is allowed explicitly. + auto compose_func_signature = + table.lookup(compose_func_name + "_"); + if (!compose_func_signature) compose_func_signature = compose_func; + auto defined_attrs = compose_func_signature.getDefinedAttributeNames(); + if (failed(ValidateAttrs(op, defined_attrs))) { + return WalkResult::interrupt(); + } + + tensorflow::IncreaseOpExpansionExecuteCounterByOne( + op->getName().getStringRef().str()); + + auto compose_func_type = compose_func.getFunctionType(); + builder.setInsertionPoint(op); + TFRTensorType unconstrainted_tensor_type = builder.getType(); + + // Create the new operands. This is mapping the operands from the target + // TF ops to the TFR function arguments. If the TFR function argument is + // a tensor_list, a "tfr.build_list" op is used to concat the available + // TF op operands. If the TFR function argument isn't a tensor/tensor_list, + // a constant is created by using the attribute stored in the TF op or the + // default value in the argument attribute. + llvm::SmallVector new_operands; + for (auto arg : llvm::enumerate(compose_func_type.getInputs())) { + if (auto tensor_type = mlir::dyn_cast(arg.value())) { + auto casted = CastOp::create(builder, op->getLoc(), tensor_type, + op->getOperand(arg.index())); + new_operands.push_back(casted); + } else if (auto list_type = + mlir::dyn_cast(arg.value())) { + llvm::SmallVector variadic_operands; + for (int i = arg.index(); i < op->getNumOperands(); i++) { + auto casted = + CastOp::create(builder, op->getLoc(), unconstrainted_tensor_type, + op->getOperand(i)); + variadic_operands.push_back(casted); + } + auto build_list_op = BuildListOp::create(builder, op->getLoc(), + list_type, variadic_operands); + new_operands.push_back(build_list_op.getOut()); + } else { + auto attr_name = compose_func.getArgAttrOfType( + arg.index(), kAttrArgumentNameAttr); + auto attribute = op->getAttr(attr_name.getValue()); + if (!attribute) { + attribute = + compose_func.getArgAttr(arg.index(), kAttrArgumentDefaultAttr); + } + if (!attribute && attr_name.getValue() == "out_type") { + auto type = op->getResult(0).getType(); + if (mlir::isa(type)) { + type = mlir::cast(type).getElementType(); + } + attribute = TypeAttr::get(type); + } + Value attr_cst; + // Wrap these special attributes as a special TFR constant, so the SSA + // value has a valid type to be used as TFR function argument. These + // attributes are not expected to be manipulated by the lowering passes. + if (mlir::isa(attribute) || mlir::isa(attribute) || + mlir::isa(attribute) || + mlir::isa(attribute)) { + TFRAttrType output_type = TFRAttrType::get(builder.getContext()); + attr_cst = + ConstOp::create(builder, op->getLoc(), output_type, attribute); + } else { + attr_cst = mlir::arith::ConstantOp::create( + builder, op->getLoc(), cast(attribute)); + } + new_operands.push_back(attr_cst); + } + } + + // Create the TFR call op + auto new_op = CallOp::create( + builder, op->getLoc(), compose_func_type.getResults(), + SymbolRefAttr::get(builder.getContext(), compose_func.getName()), + new_operands, /*args_attrs=*/nullptr, /*res_attrs=*/nullptr); + + // Replace the use of the old op. This is mapping the results from the + // target TF ops to the TFR function returns. If the TFR function return is + // a tensor_list, "tfr.get_element" op is used to extract the required TF + // op result. + llvm::SmallVector new_results; + for (auto res : llvm::enumerate(compose_func_type.getResults())) { + if (mlir::dyn_cast(res.value())) { + new_results.push_back(new_op.getResult(res.index())); + } else if (auto list_type = + mlir::dyn_cast(res.value())) { + for (int i = res.index(), j = 0; i < op->getNumResults(); i++, j++) { + auto index = mlir::arith::ConstantOp::create(builder, op->getLoc(), + builder.getIndexAttr(j)); + auto element_op = GetElementOp::create( + builder, op->getLoc(), unconstrainted_tensor_type, + new_op.getResult(res.index()), index.getResult()); + new_results.push_back(element_op.getOut()); + } + } + } + for (auto res : llvm::zip(op->getResults(), new_results)) { + auto casted = CastOp::create( + builder, op->getLoc(), std::get<0>(res).getType(), std::get<1>(res)); + std::get<0>(res).replaceAllUsesWith(casted.getOut()); + } + + // Copy all the unregisted attributes to the new op. + if (failed(CopyAllowedUnregisteredAttrs(op, new_op, defined_attrs))) { + return WalkResult::interrupt(); + } + + op->erase(); + changed |= true; + return WalkResult::advance(); + }); + + // If `changed` is false, it is considered as a failure, so the recursive + // rewrite will stop. + return success(changed); +} + +LogicalResult DecomposeTFOpsPass::InlineTFRFuncCalls() { + // The Inliner will automatically use the registered dialect inliner. + InlinerInterface inliner(&getContext()); + InlinerConfig config; + func::FuncOp func = getOperation(); + SymbolTable table(external_tfr_module_.has_value() + ? *external_tfr_module_ + : func->getParentOfType()); + + // The inliner only inlines the TFR call op. + bool changed = false; + auto walk_result = func.walk([&](CallOp call_op) { + auto callee = table.lookup(call_op.getCallee()); + if (!callee || callee.isExternal()) return WalkResult::advance(); + + // Record the boundary of the inlined operations. The inlined operation will + // be inserted between these two operations. + Operation* inlined_point = call_op.getOperation(); + Operation* after_inlined_point = + &*std::next(Block::iterator(call_op.getOperation())); + + // Use the inliner to replace all the uses of the call_op by its + // composition. + if (failed(inlineCall(inliner, config.getCloneCallback(), + cast(call_op.getOperation()), + cast(callee.getOperation()), + callee.getCallableRegion(), + /**shouldCloneInLinedRegion=*/true))) { + // This failure is usually because the decompose function is not defined. + // This call will be raised to TF ops. + return WalkResult::interrupt(); + } + + // Propagate all the attributes to the inlined operations, which are defined + // by the two boundary operations. + PropagateAttrsToOperations(call_op, Block::iterator(inlined_point), + Block::iterator(after_inlined_point)); + + // Remove the call_op to finish the op expansion. + call_op.erase(); + changed |= true; + return WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + signalPassFailure(); + return failure(); + } + + // If `changed` is false, it is considered as a failure, so the recursive + // rewrite will stop. + return success(changed); +} + +void DecomposeTFOpsPass::runOnOperation() { + // Set a maximum iteration threshold in case there are infinite loops in the + // call stack. + int max_iterators = 10; + do { + // canonicalization + ApplyCanonicalization(); + + // rewrite unregistered tf ops. Failed either because no ops can be + // decomposed or the compose function isn't defined. + auto rewrite_status = RewriteUnregisteredTFOps(); + // inline the tfr call op until there are no tfr.call op can be inlined. + auto inline_status = InlineTFRFuncCalls(); + + if (failed(rewrite_status) && failed(inline_status)) { + break; + } + } while (max_iterators-- >= 0); +} + +} // namespace + +// Creates an instance of the pass to decompose the TF ops. +std::unique_ptr> CreateDecomposeTFOpsPass( + std::optional tfr_module) { + return std::make_unique(tfr_module); +} + +static PassRegistration pass([] { + return CreateDecomposeTFOpsPass(); +}); + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose_patterns.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose_patterns.td new file mode 100644 index 0000000000000000000000000000000000000000..bd7f6c0286029a8169a1a2da4f4ee67eeb7420ea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/decompose_patterns.td @@ -0,0 +1,59 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +include "mlir/IR/OpBase.td" +include "mlir/IR/PatternBase.td" +include "mlir/Dialect/Arith/IR/ArithOps.td" +include "mlir/Dialect/Func/IR/FuncOps.td" +include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.td" + +class Quantize : NativeCodeCall<"TFR::Quantize(" # value # ", $0, $1, $_builder)">; + +class HasStringAttr : AttrConstraint< + CPred<"llvm::cast($_self).getValue() == \"" # value # "\"">>; + +def QuantActRangeNonePattern : + Pattern< + (TFR_TFRQuantActRangeOp + (TFR_ConstOp HasStringAttr<"NONE">:$act), $scale, $zp), + [(TFR_ConstantTensorOp (Arith_ConstantOp ConstantAttr)), + (TFR_ConstantTensorOp (Arith_ConstantOp ConstantAttr))]>; + +def QuantActRangeReluPattern : + Pattern< + (TFR_TFRQuantActRangeOp + (TFR_ConstOp HasStringAttr<"RELU">:$act), + (ConstantLikeMatcher F32Attr:$scale), + (ConstantLikeMatcher I64Attr:$zp)), + [(TFR_ConstantTensorOp (Arith_ConstantOp (Quantize<"0.0f"> $scale, $zp))), + (TFR_ConstantTensorOp (Arith_ConstantOp ConstantAttr))]>; + +def QuantActRangeRelu6Pattern : + Pattern< + (TFR_TFRQuantActRangeOp + (TFR_ConstOp HasStringAttr<"RELU6">:$act), + (ConstantLikeMatcher F32Attr:$scale), + (ConstantLikeMatcher I64Attr:$zp)), + [(TFR_ConstantTensorOp (Arith_ConstantOp (Quantize<"0.0f"> $scale, $zp))), + (TFR_ConstantTensorOp (Arith_ConstantOp (Quantize<"6.0f"> $scale, $zp)))]>; + + +def QuantActRangeReluN1To1Pattern : + Pattern< + (TFR_TFRQuantActRangeOp + (TFR_ConstOp HasStringAttr<"RELU_N1_TO_1">:$act), + (ConstantLikeMatcher F32Attr:$scale), + (ConstantLikeMatcher I64Attr:$zp)), + [(TFR_ConstantTensorOp (Arith_ConstantOp (Quantize<"-1.0f"> $scale, $zp))), + (TFR_ConstantTensorOp (Arith_ConstantOp (Quantize<"1.0f"> $scale, $zp)))]>; diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/passes.h new file mode 100644 index 0000000000000000000000000000000000000000..66637c4f9c48e955b0b2fd5e5006a776bcc2c3d7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/passes.h @@ -0,0 +1,52 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFR_PASSES_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFR_PASSES_PASSES_H_ + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace mlir { +namespace TFR { + +// Scans the func op and adds all the canonicalization patterns of the ops +// except the tf ops, inside the function. +void populateCanonicalizationPatterns(func::FuncOp func, + RewritePatternSet &patterns); + +// Decompose ops. +std::unique_ptr> CreateDecomposeTFOpsPass( + std::optional tfr_module = std::nullopt); + +// Rewrites quantized operands and results with their storage types. +// This pass should be run at module level after decomposition, if there are +// quantized operands or results. +std::unique_ptr> CreateRewriteQuantizedIOPass(); + +// Raise to TF ops. +std::unique_ptr> CreateRaiseToTFOpsPass( + std::optional tfr_module = std::nullopt, + bool materialize_derived_attrs = false); + +} // namespace TFR +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TFR_PASSES_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/raise_to_tf.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/raise_to_tf.cc new file mode 100644 index 0000000000000000000000000000000000000000..2c422d7f4ad726401b4c386ac889f48b2701082e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/raise_to_tf.cc @@ -0,0 +1,522 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_types.h" +#include "tensorflow/compiler/mlir/tfr/passes/passes.h" +#include "tensorflow/compiler/mlir/tfr/utils/utils.h" + +//===----------------------------------------------------------------------===// +// The pass to rewrite the TFR function call ops by TF ops. The callee of the +// TFR function call defines the signatures of the TF ops. +// +namespace mlir { +namespace TFR { + +namespace { + +// This pattern is to rewrite the "tfr.call" op and the "tfr.cast" ops on the +// operands by a TF op with "tfr.cast" ops on the results. The result type of +// the new TF op is an unranked tensor with element type derived. +class RewriteTFRCallOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + public: + explicit RewriteTFRCallOp(MLIRContext* context, const SymbolTable& table, + bool materialize_derived_attrs) + : OpRewritePattern(context), + symbol_table_(table), + materialize_derived_attrs_(materialize_derived_attrs) {} + + LogicalResult matchAndRewrite(CallOp call_op, + PatternRewriter& rewriter) const override; + + private: + // Derives the attribute values for the attributes attached to the + // `input_tfr_type`. These attributes are only for the element type of the + // inputs, and these type information has been collected in the `input_types`. + // The result is stored in `derived_attrs` as the named attributes. Returns + // failure if the attributes stored in the `input_tfr_type` violates the + // assumptions. + LogicalResult AddDerivedAttrs( + PatternRewriter& rewriter, Type input_tfr_type, + ArrayRef input_types, + llvm::StringMap* derived_attrs) const; + + // Collects the operands and attributes for the TF op. At the same time, it + // collects all the derived attribute values to derive the output types of the + // TF op. + LogicalResult CollectInputsAndAttributes( + PatternRewriter& rewriter, TFRFuncOp signature, CallOp call_op, + SmallVectorImpl* inputs, NamedAttrList* arg_attrs, + llvm::StringMap* derived_attrs) const; + + // Uses the collected attribute values to derive all the output types. + LogicalResult DeriveOutputTypes(Location loc, FunctionType signature, + const llvm::StringMap& attrs, + SmallVectorImpl* output_types) const; + + // Creates the TF op and also the necessary tfr.cast ops to replace the + // original TFR call op. + LogicalResult CreateAndReplaceOp( + PatternRewriter& rewriter, CallOp call_op, + const SmallVectorImpl& output_types, + const SmallVectorImpl& inputs, const NamedAttrList& attr_list, + const llvm::StringMap& derived_attrs) const; + + // Converts the attribute to the specific type. + Attribute ProcessAttributeValue(Attribute attr, StringAttr attr_type) const; + + Type GetFixedElementType(StringRef element_type, Builder& builder) const { + if (element_type == "i32_") return builder.getI32Type(); + if (element_type == "i64_") return builder.getI64Type(); + if (element_type == "f32_") return builder.getF32Type(); + if (element_type == "i1_") return builder.getI1Type(); + return {}; + } + + // Adds a tf.Cast op if the tfr.tensor attribute indicated a fixed element + // type. + // TODO(fengliuai): This method is required when the operand types are not set + // by the frontend correctly. + Value CastToNonDerivedType(PatternRewriter& rewriter, Location loc, + CastOp cast_op, Type input_tfr_type) const { + auto tensor_type = mlir::dyn_cast(input_tfr_type); + if (!tensor_type) return cast_op.getArg(); + + auto attr_names = tensor_type.getAttrKeys(); + if (attr_names.empty() || attr_names.size() > 1) return cast_op.getArg(); + StringRef tfr_type_attr = attr_names[0].getValue(); + if (!fixed_elt_type_attrs_.contains(tfr_type_attr)) return cast_op.getArg(); + + Type result_elt_type = GetFixedElementType(tfr_type_attr, rewriter); + if (!result_elt_type) { + return cast_op.getArg(); + } + + Type original_input_type = + mlir::cast(cast_op.getInputElementType()).getValue(); + if (result_elt_type != original_input_type) { + UnrankedTensorType result_type = UnrankedTensorType::get(result_elt_type); + return TF::CastOp::create(rewriter, loc, result_type, cast_op.getArg()); + } + return cast_op.getArg(); + } + + // For variadic operands, we have to enforce them to use the same types. + // TODO(fengliuai): This method is required when the operand types are not set + // by the frontend correctly. + void CastValuesToSameType(PatternRewriter& rewriter, Location loc, + const llvm::SmallVectorImpl& input_types, + llvm::SmallVectorImpl& input_values) const { + if (input_types.size() <= 1) return; + + Type target_input_type = mlir::cast(input_types[0]).getValue(); + auto result_type = UnrankedTensorType::get(target_input_type); + for (auto i = 1; i < input_types.size(); ++i) { + Type current_input_type = mlir::cast(input_types[i]).getValue(); + if (current_input_type != target_input_type) { + input_values[i] = + TF::CastOp::create(rewriter, loc, result_type, input_values[i]); + } + } + } + + const SymbolTable& symbol_table_; + const bool materialize_derived_attrs_; + const llvm::SmallDenseSet fixed_elt_type_attrs_{"i32_", "i64_", + "f32_", "i1_"}; +}; + +LogicalResult RewriteTFRCallOp::AddDerivedAttrs( + PatternRewriter& rewriter, Type input_tfr_type, + ArrayRef input_types, + llvm::StringMap* derived_attrs) const { + // If there is an attribute associated to the input in the signature, we + // store it as an derived attribute. + if (auto tensor_type = mlir::dyn_cast(input_tfr_type)) { + auto attr_names = tensor_type.getAttrKeys(); + if (attr_names.empty()) return success(); + + if (attr_names.size() == 1) { + derived_attrs->insert({attr_names[0].getValue(), input_types[0]}); + return success(); + } + } + + // If there is an attribute associated to the input in the signature, + // we store it as an derived attribute. + if (auto list_type = mlir::dyn_cast(input_tfr_type)) { + auto attr_names = list_type.getAttrKeys(); + if (attr_names.empty()) return success(); + + // N*T case + if (attr_names.size() == 2) { + derived_attrs->insert({attr_names[0].getValue(), + rewriter.getI32IntegerAttr(input_types.size())}); + // Note that this uses the first element of the list to infer the T value. + // A tf.Cast is required to cast the other inputs to the same type. + derived_attrs->insert({attr_names[1].getValue(), input_types[0]}); + return success(); + } + + // list(dtype) case + if (attr_names.size() == 1) { + derived_attrs->insert( + {attr_names[0].getValue(), rewriter.getArrayAttr(input_types)}); + return success(); + } + } + + return failure(); +} + +LogicalResult RewriteTFRCallOp::CollectInputsAndAttributes( + PatternRewriter& rewriter, TFRFuncOp signature, CallOp call_op, + SmallVectorImpl* inputs, NamedAttrList* arg_attrs, + llvm::StringMap* derived_attrs) const { + for (const auto& operand : + llvm::enumerate(signature.getFunctionType().getInputs())) { + // If the index is larger than the operand number of the call_op, the + // default value of the operand needs to be used. + if (operand.index() >= call_op.getNumOperands()) { + auto attr_name = signature.getArgAttrOfType( + operand.index(), kAttrArgumentNameAttr); + auto attr_value = + signature.getArgAttr(operand.index(), kAttrArgumentDefaultAttr); + arg_attrs->push_back( + rewriter.getNamedAttr(attr_name.getValue(), attr_value)); + continue; + } + + // The index is valid for the call_op. + Value input = call_op.getOperand(operand.index()); + Operation* input_op = input.getDefiningOp(); + auto input_tfr_type = + signature.getFunctionType().getInputs()[operand.index()]; + + // There are three cases for the preceding input_op: + + // 1. The preceding op can be a tfr.cast op, which will be fused to the + // current op, so the result op has input with tensor type. + if (auto cast_op = dyn_cast_or_null(input_op)) { + Value input_to_cast = CastToNonDerivedType(rewriter, call_op.getLoc(), + cast_op, input_tfr_type); + inputs->push_back(input_to_cast); + if (failed(AddDerivedAttrs(rewriter, input_tfr_type, + {cast_op.getInputElementType()}, + derived_attrs))) { + return failure(); + } + continue; + } + + // 2. The preceding op is a tfr.build_list op, which collects multiple + // values with tensor types via the tfr.cast ops. These ops will be fused + // to the current op as well, so all the tfr.cast op inputs will be inputs + // to the result op. + if (auto list_op = dyn_cast_or_null(input_op)) { + // Find out all the inputs to the build list op + // TODO(fengliuai): make build_list op only take tensor argument + llvm::SmallVector list_input_types; + llvm::SmallVector list_inputs; + for (auto list_input : list_op.getOperands()) { + auto cast_op = dyn_cast_or_null(list_input.getDefiningOp()); + if (!cast_op) return failure(); + list_inputs.push_back(cast_op.getArg()); + list_input_types.push_back(cast_op.getInputElementType()); + } + CastValuesToSameType(rewriter, call_op.getLoc(), list_input_types, + list_inputs); + inputs->append(list_inputs.begin(), list_inputs.end()); + if (failed(AddDerivedAttrs(rewriter, input_tfr_type, list_input_types, + derived_attrs))) { + return failure(); + } + continue; + } + + // 3. The preceding op is a constant, thus the value of this constant is + // used to create an attribute of the result op, according to the signature. + Attribute arg_value; + // A failure indicates the argument isn't a constant value, so we should + // not use it as an attribute. + if (!matchPattern(input, m_Constant(&arg_value))) { + return failure(); + } + auto attr_name = signature.getArgAttrOfType( + operand.index(), kAttrArgumentNameAttr); + auto attr_type = signature.getArgAttrOfType( + operand.index(), kAttrArgumentTypeAttr); + auto value = ProcessAttributeValue(arg_value, attr_type); + arg_attrs->push_back(rewriter.getNamedAttr(attr_name.getValue(), value)); + } + return success(); +} + +Attribute RewriteTFRCallOp::ProcessAttributeValue(Attribute attr, + StringAttr attr_type) const { + if (!attr_type) return attr; + + if (attr_type.getValue() == "tensor") { + if (auto f = mlir::dyn_cast(attr)) { + RankedTensorType type = RankedTensorType::get({}, f.getType()); + return DenseFPElementsAttr::get(type, attr); + } + // TODO(fengliuai): handles ArrayAttr. Note that it can be nested ArrayAttr. + } + + return attr; +} + +// For each output, uses the attribute name associated to the tfr types to find +// out the attribute value from the collected `attrs` and create the output type +// of the result op by using the attribute value as the element type. +LogicalResult RewriteTFRCallOp::DeriveOutputTypes( + Location loc, FunctionType signature, + const llvm::StringMap& attrs, + SmallVectorImpl* output_types) const { + for (auto res : llvm::enumerate(signature.getResults())) { + if (auto tensor_type = mlir::dyn_cast(res.value())) { + // tfr.tensor should only have one attribute attached. + auto attr_key = tensor_type.getAttrKeys().front(); + Builder builder(signature.getContext()); + if (auto attr = attrs.lookup(attr_key.getValue())) { + output_types->push_back( + UnrankedTensorType::get(mlir::cast(attr).getValue())); + } else if (Type element_type = + GetFixedElementType(attr_key.getValue(), builder)) { + output_types->push_back(UnrankedTensorType::get(element_type)); + } else { + emitError(loc) << "type " << attr_key.getValue() + << " can't be resolved for the signature of the op"; + return failure(); + } + continue; + } + + if (auto list_type = mlir::dyn_cast(res.value())) { + // There are two cases: N*T or list(dtype) + auto attr_keys = list_type.getAttrKeys(); + // N*T case + if (attr_keys.size() == 2) { + // The first one is N, and the second one is T + int list_size = + mlir::cast(attrs.lookup(attr_keys[0].getValue())) + .getInt(); + Type list_type = + mlir::cast(attrs.lookup(attr_keys[1].getValue())) + .getValue(); + for (int i = 0; i < list_size; ++i) { + output_types->push_back(UnrankedTensorType::get(list_type)); + } + continue; + } + // TODO(fengliuai): list(dtype) case + } + return failure(); + } + return success(); +} + +LogicalResult RewriteTFRCallOp::CreateAndReplaceOp( + PatternRewriter& rewriter, CallOp call_op, + const SmallVectorImpl& output_types, + const SmallVectorImpl& inputs, const NamedAttrList& attr_list, + const llvm::StringMap& derived_attrs) const { + // Create the new op + Location loc = call_op.getLoc(); + rewriter.setInsertionPointAfter(call_op); + std::string tf_op_name = GetTFOpName(call_op.getCallee()); + OperationState new_state(loc, tf_op_name, inputs, output_types, attr_list); + Operation* new_op = rewriter.create(new_state); + if (materialize_derived_attrs_) { + for (const auto& attr : derived_attrs) { + // Add or update the derived attribute with the value. Skip the fixed + // element type attributes, in case they are present in the NodeDef. + if (!fixed_elt_type_attrs_.contains(attr.first())) { + new_op->setAttr(attr.first(), attr.second); + } + } + } + // Create the tfr.cast ops on the results and replace the uses of the + // original call op. + TFRTensorType unconstrainted_type = rewriter.getType(); + SmallVector new_results; + for (auto res : llvm::enumerate(call_op.getResultTypes())) { + Type res_type = res.value(); + if (mlir::dyn_cast(res_type)) { + Value new_res = new_op->getResult(res.index()); + auto casted = CastOp::create(rewriter, loc, res_type, new_res); + new_results.push_back(casted.getOut()); + } else if (auto list_type = + mlir::dyn_cast(res.value())) { + SmallVector tensor_list; + for (int i = res.index(); i < new_op->getNumResults(); i++) { + Value new_res = new_op->getResult(i); + auto casted = + CastOp::create(rewriter, loc, unconstrainted_type, new_res); + tensor_list.push_back(casted.getOut()); + } + auto list_op = BuildListOp::create(rewriter, loc, res_type, tensor_list); + new_results.push_back(list_op.getOut()); + } + } + + // Copy all the allowed attributes to the new op. + if (failed(CopyNonSymbolRefAttrs(call_op, new_op))) return failure(); + + rewriter.replaceOp(call_op, new_results); + return success(); +} + +LogicalResult RewriteTFRCallOp::matchAndRewrite( + CallOp call_op, PatternRewriter& rewriter) const { + // Get the func op and verify that it is external. The type of this external + // func op is used as the signature of the corresponding TF ops. All the + // external func ops have the trailing underscore. + std::string external_callee_name = call_op.getCallee().str().append("_"); + TFRFuncOp func = symbol_table_.lookup(external_callee_name); + if (!func || !func.isExternal()) return failure(); + // Get the inputs and attributes. The attributes include these from the + // argument list and also these derived from the inputs. + SmallVector inputs; + NamedAttrList argument_attrs; + llvm::StringMap derived_attrs; + if (failed(CollectInputsAndAttributes(rewriter, func, call_op, &inputs, + &argument_attrs, &derived_attrs))) { + return failure(); + } + + // Derive the output types. The result type is derived by using the + // attributes attched to the result type of the signature. The attribute + // value should be either in the attribute argument list or the derived + // attribute from the input tensors. All the result type + // are unranked, and shape inference should be applied afterwards. + SmallVector output_types; + + // Merge the attributes from the argument list to the derived ones. + for (auto& attr : argument_attrs) { + derived_attrs.insert({attr.getName(), attr.getValue()}); + } + + // Derive the output types by using the attributes attached to the tfr + // types. + if (failed(DeriveOutputTypes(call_op->getLoc(), func.getFunctionType(), + derived_attrs, &output_types))) { + return failure(); + } + + // Create the new op and replace the old TFR call op. + return CreateAndReplaceOp(rewriter, call_op, output_types, inputs, + argument_attrs, derived_attrs); +} + +// Raise TFR call ops to the TF ops. +class RaiseToTFOpsPass + : public PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RaiseToTFOpsPass) + + void getDependentDialects(DialectRegistry& registry) const override { + registry.insert(); + } + + explicit RaiseToTFOpsPass(std::optional tfr_module, + bool materialize_derived_attrs) + : external_tfr_module_(tfr_module), + materialize_derived_attrs_(materialize_derived_attrs) {} + + StringRef getArgument() const final { return "tfr-raise-to-tf"; } + + StringRef getDescription() const final { + return "Raise all the TFR call ops to TF ops."; + } + + void runOnOperation() override; + + private: + std::optional external_tfr_module_; + const bool materialize_derived_attrs_; +}; + +void RaiseToTFOpsPass::runOnOperation() { + func::FuncOp func = getOperation(); + MLIRContext* ctx = &getContext(); + SymbolTable table(external_tfr_module_.has_value() + ? *external_tfr_module_ + : func->getParentOfType()); + + RewritePatternSet patterns(&getContext()); + patterns.add(ctx, table, materialize_derived_attrs_); + + populateCanonicalizationPatterns(func, patterns); + + (void)applyPatternsGreedily(func, std::move(patterns)); +} +} // namespace + +// Creates an instance of the pass to raise TFR call ops to the TF ops. +std::unique_ptr> CreateRaiseToTFOpsPass( + std::optional tfr_module, bool materialize_derived_attrs) { + return std::make_unique(tfr_module, + materialize_derived_attrs); +} + +static PassRegistration pass([] { + return CreateRaiseToTFOpsPass(); +}); + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/rewrite_quantized_io.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/rewrite_quantized_io.cc new file mode 100644 index 0000000000000000000000000000000000000000..01c936570fa15ffda253e0ebc71ebdf0bcf557bc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/rewrite_quantized_io.cc @@ -0,0 +1,120 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Quant/IR/QuantTypes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/compiler/mlir/tfr/passes/passes.h" + +namespace mlir { +namespace TFR { + +class RewriteQuantizedIOPass + : public PassWrapper> { + public: + StringRef getArgument() const final { return "tfr-rewrite-quantized-io"; } + + StringRef getDescription() const final { + return "Replaces operands and results that has quantized type with their " + "storage types."; + } + void runOnOperation() override; +}; + +void RewriteQuantizedIOPass::runOnOperation() { + ModuleOp module = getOperation(); + OpBuilder builder(module); + module.walk([&](func::FuncOp func) { + Block& block = func.front(); + Operation* terminator = block.getTerminator(); + + // Replace input_arg(tensor) -> tfr.cast + // with input_arg(tensor) -> tfr.cast + for (BlockArgument arg : block.getArguments()) { + Type arg_type = arg.getType(); + if (auto quant_type = llvm::dyn_cast( + llvm::cast(arg_type).getElementType())) { + if (arg.hasOneUse() && llvm::isa(*arg.user_begin())) { + arg.setType(llvm::cast(arg_type).clone( + quant_type.getStorageType())); + } else { + std::string error_message; + llvm::raw_string_ostream os{error_message}; + os << "The argument with type "; + arg.getType().print(os); + os << " should have one user, which should be tfr.cast."; + func->emitError(error_message); + return; + } + } + } + + builder.setInsertionPoint(terminator); + // Replace tfr.cast(tensor) -> output + // with tfr.cast(tensor) -> output + for (OpOperand& returned_value : terminator->getOpOperands()) { + auto returned_type = + llvm::dyn_cast(returned_value.get().getType()); + if (!returned_type || + !llvm::isa(returned_type.getElementType())) { + continue; + } + + if (auto returned_op = + returned_value.get().getDefiningOp()) { + auto new_type = returned_type.clone( + llvm::cast(returned_type.getElementType()) + .getStorageType()); + auto new_op = TFR::CastOp::create(builder, returned_op->getLoc(), + new_type, returned_op.getArg()); + returned_value.set(new_op.getResult()); + if (returned_op.use_empty()) { + returned_op.erase(); + } + } else { + returned_value.get().getDefiningOp()->emitError( + "The producer of quantized type result should be a tfr.cast op."); + return; + } + } + + auto new_func_type = builder.getFunctionType(block.getArgumentTypes(), + terminator->getOperandTypes()); + func.setType(new_func_type); + }); +} + +// Creates an instance of the pass to decompose the TF ops. +std::unique_ptr> CreateRewriteQuantizedIOPass() { + return std::make_unique(); +} + +static PassRegistration pass([] { + return CreateRewriteQuantizedIOPass(); +}); + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/tfr_opt.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/tfr_opt.cc new file mode 100644 index 0000000000000000000000000000000000000000..f7804c3c6672f33315bda5a06c0f96ba678277e7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/passes/tfr_opt.cc @@ -0,0 +1,44 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/Quant.h" // from @llvm-project +#include "mlir/Dialect/Quant/IR/QuantTypes.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/InitAllPasses.h" // from @llvm-project +#include "mlir/Tools/mlir-opt/MlirOptMain.h" // from @llvm-project +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/quantization/common/ir/QuantOps.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" + +int main(int argc, char **argv) { + tensorflow::InitMlir y(&argc, &argv); + + mlir::registerAllPasses(); + + mlir::DialectRegistry registry; + registry.insert(); + mlir::func::registerAllExtensions(registry); + return failed(mlir::MlirOptMain(argc, argv, "TFR Pass Driver\n", registry)); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/composite.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/composite.py new file mode 100644 index 0000000000000000000000000000000000000000..539c24f5afd1bd4b98f4269c6e2d90e0bfec61be --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/composite.py @@ -0,0 +1,52 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Op composition registration.""" + + +# TODO(fengliuai): add the tf_export decrator +class Composite(object): + """A decorator to register a function as a composition for an TF operator. + + The argument to the decorator must be the name of a TF raw operator the + function composites for. Decorated function must take positional arguments + which corresponds to the input and attributes in OpDef of the TF operation. + # TODO(fengliuai): more documents here. + + Example: + @composite.Composite('AddN') + def _compose_add_n(inputs, N): + if N == 1: + .... + """ + + # TODO(fengliuai): support input_binding and output_binding so the arguments + # are not positional. + def __init__(self, + op_name, + inputs=None, + attrs=None, + derived_attrs=None, + outputs=None): + self._op_name = op_name + self._inputs = inputs + self._attrs = attrs + self._derived_attrs = derived_attrs + self._outputs = outputs + + def __call__(self, compose_fn): + # TODO(fengliuai): more sanity check of the input function and make sure + # the bounded arguments of the function matches the 'inputs' and 'attrs'. + setattr(compose_fn, '_tfr_op_name', self._op_name) + return compose_fn diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..def9c63d23cafe8d657df0ba9f9a7eed1788b598 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen.py @@ -0,0 +1,145 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""op_reg_gen: Generate op registration code from composite op code.""" + +# pylint: disable=invalid-name +# pylint: disable=missing-function-docstring +# pylint: disable=g-direct-tensorflow-import + +import gast as ast + +from tensorflow.python.autograph.pyct import transformer +from tensorflow.python.autograph.pyct import transpiler +from tensorflow.python.framework import op_def_registry +from tensorflow.python.util import tf_inspect + +_COMPOSITE_ARG_LIST = ['op_name', 'inputs', 'attrs', 'derived_attrs', 'outputs'] + + +class OpRegGenImpl(transformer.CodeGenerator): + """Visit the AST and generate C++ op registration functions.""" + + def __init__(self, ctx): + super(OpRegGenImpl, self).__init__(ctx) + self.ctx = ctx + + def visit_Name(self, node): + return node.id + + def visit_Constant(self, node): + return node.value + + def visit_keyword(self, node): + return node.arg, self.visit(node.value) + + def visit_List(self, node): + return [self.visit(cst) for cst in node.elts] + + def visit_arguments(self, node): + return [self.visit(arg) for arg in node.args] + + def visit_FunctionDef(self, node): + # TODO(fengliuai): create one utility method to match different apis and + # shared it with the tfr_gen.py module. + compose_dec = [] + for dec in node.decorator_list: + if isinstance(dec, ast.Call): + if isinstance(dec.func, ast.Attribute) and dec.func.attr == 'Composite': + compose_dec.append(dec) + if isinstance(dec.func, ast.Name) and dec.func.id == 'Composite': + compose_dec.append(dec) + + if not compose_dec: + # skip a non-composition function + return + elif len(compose_dec) > 1: + raise KeyError('More than one TF ops decomposes for.') + + all_dec_args = {} + for arg_name, arg_value in zip(_COMPOSITE_ARG_LIST, compose_dec[0].args): + all_dec_args[arg_name] = self.visit(arg_value) + + kw_dec_args = dict([self.visit(kw) for kw in compose_dec[0].keywords]) + + if all_dec_args.keys() & kw_dec_args.keys(): + raise KeyError('More arguments than expected.') + + all_dec_args.update(kw_dec_args) + + op_name = all_dec_args['op_name'] + op_def = op_def_registry.get(op_name) + if op_def: + if len(all_dec_args) > 1: + # Op has been registered, so it is a user error to specify op def. + raise ValueError('Op has been registered: ' + op_name) + else: + # Op has been registered, then we don't need to generate register code. + return + + # Validates the function inputs match what are in the decorator. + inputs = all_dec_args.get('inputs', []) + attrs = all_dec_args.get('attrs', []) + expected_args = [arg.split(':')[0] for arg in inputs + attrs] + all_func_args = self.visit(node.args) + + if len(expected_args) != len(all_func_args): + raise KeyError( + 'Composition arguments for {} do not match the registration. {} vs {}' + .format(op_name, expected_args, all_func_args)) + + cxx_reg_code = ['\nREGISTER_OP("{}")'.format(op_name)] + for input_ in inputs: + cxx_reg_code.append('.Input("{}")'.format(input_)) + for attr in attrs: + py_str = attr.replace('"', "'") + cxx_reg_code.append('.Attr("{}")'.format(py_str)) + for attr in all_dec_args.get('derived_attrs', []): + py_str = attr.replace('"', "'") + cxx_reg_code.append('.Attr("{}")'.format(py_str)) + for output_ in all_dec_args.get('outputs', []): + cxx_reg_code.append('.Output("{}")'.format(output_)) + cxx_reg_code[-1] += ';\n' + self.emit('\n '.join(cxx_reg_code)) + + +class OpRegGen(transpiler.GenericTranspiler): + """Transforms Python objects into TFR MLIR source code.""" + + def transform_ast(self, node, ctx): + gen = OpRegGenImpl(ctx) + gen.visit(node) + return gen.code_buffer + + +def op_reg_gen(func): + """Parse a function and emit the TFR functions.""" + op_reg_code, _ = OpRegGen().transform(func, None) + return op_reg_code + + +def gen_register_op(source, method_prefix=None): + """Parse a python code and emit the TFR functions from a target class.""" + mlir_funcs = [ + op_reg_gen(func) + for name, func in tf_inspect.getmembers(source, tf_inspect.isfunction) + if not method_prefix or name.startswith(method_prefix) + ] + headers = r""" +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + """ + code = '\n'.join(mlir_funcs) + return headers + code + '} // namespace tensorflow\n' diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen_test.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen_test.py new file mode 100644 index 0000000000000000000000000000000000000000..38ce29ed3ea6b0544539218fb89db4448fcb6138 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/op_reg_gen_test.py @@ -0,0 +1,77 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `op_reg_gen` module.""" + +# pylint: disable=missing-function-docstring +# pylint: disable=invalid-name +# pylint: disable=g-direct-tensorflow-import + +import sys + +from tensorflow.compiler.mlir.python.mlir_wrapper import filecheck_wrapper as fw +from tensorflow.compiler.mlir.tfr.python import composite +from tensorflow.compiler.mlir.tfr.python.op_reg_gen import gen_register_op +from tensorflow.python.platform import test + + +Composite = composite.Composite + + +@composite.Composite( + 'TestNoOp', derived_attrs=['T: numbertype'], outputs=['o1: T']) +def _composite_no_op(): + pass + + +@Composite( + 'TestCompositeOp', + inputs=['x: T', 'y: T'], + attrs=['act: {"", "relu"}', 'trans: bool = true'], + derived_attrs=['T: numbertype'], + outputs=['o1: T', 'o2: T']) +def _composite_op(x, y, act, trans): + return x + act, y + trans + + +class TFRGenTensorTest(test.TestCase): + """MLIR Generation Tests for MLIR TFR Program.""" + + def test_op_reg_gen(self): + cxx_code = gen_register_op(sys.modules[__name__]) + cxx_code_exp = r""" + CHECK: #include "tensorflow/core/framework/op.h" + CHECK-EMPTY + CHECK: namespace tensorflow { + CHECK-EMPTY + CHECK-LABEL: REGISTER_OP("TestNoOp") + CHECK-NEXT: .Attr("T: numbertype") + CHECK-NEXT: .Output("o1: T"); + CHECK-EMPTY + CHECK-LABEL: REGISTER_OP("TestCompositeOp") + CHECK-NEXT: .Input("x: T") + CHECK-NEXT: .Input("y: T") + CHECK-NEXT: .Attr("act: {'', 'relu'}") + CHECK-NEXT: .Attr("trans: bool = true") + CHECK-NEXT: .Attr("T: numbertype") + CHECK-NEXT: .Output("o1: T") + CHECK-NEXT: .Output("o2: T"); + CHECK-EMPTY + CHECK: } // namespace tensorflow + """ + self.assertTrue(fw.check(str(cxx_code), cxx_code_exp), str(cxx_code)) + + +if __name__ == '__main__': + test.main() diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/test_utils.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..957f9693d23f5dddb39cf3e9e7fb7cdca454309c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/test_utils.py @@ -0,0 +1,47 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test utils for composite op definition.""" +from tensorflow.python.eager import backprop +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +class OpsDefsTest(test.TestCase): + """Test utils.""" + + def _assertOpAndComposite(self, vars_, compute_op, compute_composite, kwargs, + op_kwargs=None): + if op_kwargs is None: + op_kwargs = kwargs + if test_util.IsMklEnabled(): + self.skipTest("Not compatible with oneDNN custom ops.") + + # compute with op. + with backprop.GradientTape() as gt: + for var_ in vars_: + gt.watch(var_) + y = compute_op(**op_kwargs) # uses op and decomposites by the graph pass. + grads = gt.gradient(y, vars_) # uses registered gradient function. + + # compute with composition + with backprop.GradientTape() as gt: + for var_ in vars_: + gt.watch(var_) + re_y = compute_composite(**kwargs) # uses composite function. + re_grads = gt.gradient(re_y, vars_) # uses gradients compposite function. + + for v, re_v in zip(y, re_y): + self.assertAllClose(v, re_v) + for g, re_g in zip(grads, re_grads): + self.assertAllClose(g, re_g) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..454af942c9466864f48839ac24ef9a887007d476 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen.py @@ -0,0 +1,1573 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""tfr_gen: Generate mlir tfr decomposition function from python code.""" + +# pylint: disable=invalid-name +# pylint: disable=missing-function-docstring +# pylint: disable=g-direct-tensorflow-import + +import enum +import os +import re +import types +import gast as ast + +from tensorflow.compiler.mlir.tfr import tfr_wrapper as tfr +from tensorflow.core.framework import types_pb2 +from tensorflow.python.autograph.converters import control_flow +from tensorflow.python.autograph.converters import return_statements +from tensorflow.python.autograph.impl import api +from tensorflow.python.autograph.pyct import anno +from tensorflow.python.autograph.pyct import cfg +from tensorflow.python.autograph.pyct import qual_names +from tensorflow.python.autograph.pyct import transformer +from tensorflow.python.autograph.pyct import transpiler +from tensorflow.python.autograph.pyct.static_analysis import activity +from tensorflow.python.autograph.pyct.static_analysis import reaching_definitions +from tensorflow.python.autograph.pyct.static_analysis import reaching_fndefs +from tensorflow.python.autograph.pyct.static_analysis import type_inference +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import load_library +from tensorflow.python.framework import op_def_registry +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import tf_inspect + +# TODO(mdan): Use class definitions so that we can mix these with Python types. + + +class TFRTypes(enum.Enum): + """All the supported types. + + 1-3: tfr types + 4-99: mlir built-in types + 100-199: TF related translator internal types + 200- : Python related translator internal types + """ + TENSOR = 1 + TENSOR_LIST = 2 + ATTR = 3 + NONE = 4 + SHAPE = 5 # shape -> !shape.shape + I1 = 21 + I8 = 22 + I16 = 23 + I32 = 24 + I64 = 25 + F32 = 26 + INDEX = 27 + AG_UNDEFINED_VAL = 100 + AG_BUILTIN_FUNC = 101 + TF_RAW_OP = 102 + TF_REGION = 103 + TF_TENSOR_SHAPE_FUNC = 104 # shape.as_list + TF_TENSOR_SHAPE_LIST = 105 # shape.as_list() + PY_BUILTIN_FUNC = 200 + TFR_BUILTIN_FUNC = 201 + + # As these are not real types, __getattribute__ helps them appear more like + # actual types (i.e. class definitions). + def __getattribute__(self, name): + if name == 'shape' and object.__getattribute__(self, 'value') == 1: + return TFRTypes.SHAPE + if name == 'as_list' and object.__getattribute__(self, 'value') == 5: + return TFRTypes.TF_TENSOR_SHAPE_FUNC + return object.__getattribute__(self, name) + + def __str__(self): + if self.value < 4: # pylint: disable=comparison-with-callable + return '!tfr.' + self.name.lower() + elif self.value < 10: # pylint: disable=comparison-with-callable + return '!shape.' + self.name.lower() + else: + return self.name.lower() + + +_ATTRIBUTE_TYPES = ( + TFRTypes.I1, TFRTypes.I32, TFRTypes.I64, TFRTypes.F32, TFRTypes.INDEX, + TFRTypes.ATTR + ) + +# TODO(b/203493652): implement the "rename_to" for the customization in +# tensorflow/core/api_def/base_api/* +# {op_name: {API's attribute name: OpDef's attribute name}} +_ATTRIBUTE_RENAMES = { + 'Mean': {'axis': 'reduction_indices'}, + 'Split': {'axis': 'split_dim'}, + 'SplitV': {'axis': 'split_dim'}, +} + + +def _get_type_from_proto(arg_def=None, attr_def=None): + if not arg_def: + if attr_def.type == 'bool': + return TFRTypes.I1 + elif attr_def.type == 'int32': + return TFRTypes.I32 + elif attr_def.type == 'int' or attr_def.type == 'int64': + return TFRTypes.I64 + elif attr_def.type == 'float': + return TFRTypes.F32 + else: + return TFRTypes.ATTR + + if arg_def.number_attr or arg_def.type_list_attr: + return TFRTypes.TENSOR_LIST + else: + return TFRTypes.TENSOR + + +def _get_type_info_from_proto(arg_def=None, attr_def=None): + attr_type = _get_type_from_proto(arg_def, attr_def) + if not arg_def: + return '{}{{tfr.name="{}",tfr.type="{}"}}'.format( + attr_type, attr_def.name, attr_def.type) + else: + attr_names = [] + if arg_def.number_attr: + attr_names.append(arg_def.number_attr) + if arg_def.type_attr: + attr_names.append(arg_def.type_attr) + if arg_def.type_list_attr: + attr_names.append(arg_def.type_list_attr) + + # TODO(fengliuai): currently we don't support backward type inference, so we + # have to store these non-derivable type in the signatures, and then they + # can be used to cast the values when raising to tf ops. + if arg_def.type == types_pb2.DT_FLOAT: + attr_names.append('f32_') + elif arg_def.type == types_pb2.DT_INT32: + attr_names.append('i32_') + elif arg_def.type == types_pb2.DT_INT64: + attr_names.append('i64_') + elif arg_def.type == types_pb2.DT_BOOL: + attr_names.append('i1_') + + if not attr_names: + return str(attr_type) + else: + return '{}<{}>'.format(attr_type, ','.join(attr_names)) + + +def _get_val_from_proto(attr_type, attr_val): + if attr_type == TFRTypes.I1: + return 'true' if attr_val.b else 'false' + elif attr_type == TFRTypes.I32 or attr_type == TFRTypes.I64: + return attr_val.i + elif attr_type == TFRTypes.F32: + return attr_val.f + elif attr_type == TFRTypes.ATTR: + # string + if attr_val.HasField('s'): + return '"{}"'.format(attr_val.s.decode()) + # type + if attr_val.HasField('type'): + if attr_val.type == types_pb2.DT_FLOAT: + return 'f32' + elif attr_val.type == types_pb2.DT_INT32: + return 'i32' + elif attr_val.type == types_pb2.DT_INT64: + return 'i64' + elif attr_val.type == types_pb2.DT_BOOL: + return 'i1' + # list + if attr_val.HasField('list'): + if attr_val.list.f: + elt_ty = TFRTypes.F32 + values = attr_val.list.f + elif attr_val.list.i: + elt_ty = TFRTypes.I64 + values = attr_val.list.i + else: + elt_ty = TFRTypes.NONE + values = [] + array_attr_elts = ['{}:{}'.format(val, elt_ty) for val in values] + return '[{}]'.format(','.join(array_attr_elts)) + raise NotImplementedError( + 'Proto AttrValue not recognized. type: {}, value: {}'.format( + attr_type, attr_val)) + + +def _collect_derived_attrs_from_proto(op_def): + derived_attrs = set() + for arg in op_def.input_arg: + if arg.type_attr: + derived_attrs.add(arg.type_attr) + if arg.number_attr: + derived_attrs.add(arg.number_attr) + if arg.type_list_attr: + derived_attrs.add(arg.type_list_attr) + + # TODO(fengliuai): currently we don't support backward type inference, so we + # have to store these non-derivable type in the signatures, and then they + # can be used to cast the values when raising to tf ops. + if arg.type == types_pb2.DT_FLOAT: + derived_attrs.add('f32_') + elif arg.type == types_pb2.DT_INT32: + derived_attrs.add('i32_') + elif arg.type == types_pb2.DT_INT64: + derived_attrs.add('i64_') + elif arg.type == types_pb2.DT_BOOL: + derived_attrs.add('i1_') + return derived_attrs + + +def _require_tensor_list(arg_def): + return arg_def.type_list_attr or arg_def.number_attr + + +def _camel_to_snake(name): + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + +class OpDefCache(object): + """A Dict to cache the OpDef for the Python function name.""" + + def __init__(self): + self._op_defs = {} + + def lookup(self, f_name, func_def=None, optional=False): + if f_name in self._op_defs: + return self._op_defs[f_name] + + if isinstance(func_def, types.FunctionType): + if not hasattr(func_def, '_tfr_op_name'): + # skip a non-composition function + if optional: + return (None, None) + else: + raise KeyError('OpDef does not exist: ' + f_name) + op_name = getattr(func_def, '_tfr_op_name') + elif not func_def: + op_name = f_name + else: + # TODO(fengliuai): create one utility method to match different APIs. + compose_dec = [] + for dec in func_def.decorator_list: + if isinstance(dec, ast.Call): + if isinstance(dec.func, + ast.Attribute) and dec.func.attr == 'Composite': + compose_dec.append(dec) + if isinstance(dec.func, ast.Name) and dec.func.id == 'Composite': + compose_dec.append(dec) + + if not compose_dec: + # skip a non-composition function + if optional: + return (None, None) + else: + raise KeyError('OpDef does not exist: ' + f_name) + elif len(compose_dec) > 1: + raise KeyError('More than one TF ops decomposes for.') + else: + op_name = compose_dec[0].args[0].value + + op_def = op_def_registry.get(op_name) + if not op_def: + raise ValueError('Not a registered op: ' + op_name) + derived_attrs = _collect_derived_attrs_from_proto(op_def) + self._op_defs[f_name] = (op_def, derived_attrs) + return (op_def, derived_attrs) + + def mlir_external_funcs(self): + tfr_funcs = set() + for _, (op_def, derived_attrs) in sorted(self._op_defs.items()): + tfr_func = '\ntfr.func @tf__{}_('.format(_camel_to_snake(op_def.name)) + + # tensor inputs + inputs = [ + _get_type_info_from_proto(arg_def) for arg_def in op_def.input_arg + ] + + # attribute inputs. The attribute with default values are moved backwards. + non_derived_attrs = [ + attr for attr in op_def.attr if attr.name not in derived_attrs + ] + attrs_no_default = [ + attr for attr in non_derived_attrs + if not attr.HasField('default_value') + ] + attrs_with_default = [ + attr for attr in non_derived_attrs if attr.HasField('default_value') + ] + attr_names = {'f32_', 'i32_', 'i64_', 'i1_'} # reserved + for attr_def in attrs_no_default + attrs_with_default: + inputs.append(_get_type_info_from_proto(None, attr_def)) + attr_names.add(attr_def.name) + + # tensor outputs + outputs = [ + _get_type_info_from_proto(arg_def) for arg_def in op_def.output_arg + ] + + inputs = ','.join(inputs) + outputs = ','.join(outputs) + attrs = ','.join(sorted(derived_attrs.union(attr_names))) + tfr_funcs.add('{}{}) -> ({}) attributes {{{}}}'.format( + tfr_func, inputs, outputs, attrs)) + return sorted(list(tfr_funcs)) + + +_PY_TYPE_TO_TFR = { + bool: TFRTypes.I1, + int: TFRTypes.I64, + float: TFRTypes.F32, +} + +_TF_DTYPE_TO_TFR = { + 'bool': TFRTypes.I1, + 'int64': TFRTypes.I64, + 'int32': TFRTypes.I32, + 'int16': TFRTypes.I16, + 'int8': TFRTypes.I8, + 'float32': TFRTypes.F32, +} + +_AG_FIXED_RETURN_TYPE = { + 'for_stmt': type(None), + 'if_stmt': type(None), + 'Undefined': TFRTypes.AG_UNDEFINED_VAL, +} + +QN = qual_names.QN + +# TODO(mdan): Fix this with an importable module. +AG_MODULE = api._TRANSPILER.get_extra_locals()['ag__'] # pylint:disable=protected-access + +# When an item is callable, the signature is (*operand_types) -> result_type(s) +TFR_BUILTINS = { + '_tfr_quant_act_range': (TFRTypes.TENSOR, TFRTypes.TENSOR), + '_tfr_quant_rescale': TFRTypes.TENSOR, + '_tfr_quant_raw_data': lambda input_type: input_type, + '_tfr_quant_qparam': (TFRTypes.TENSOR, TFRTypes.TENSOR), + '_tfr_quant_scale_factor': TFRTypes.TENSOR, +} + + +class TFRTypeResolver(type_inference.Resolver): + """Resolve types for the external names, calls and arguments.""" + + def __init__(self, op_defs): + super(TFRTypeResolver, self).__init__() + self._op_defs = op_defs + + # This pattern matching mechanism works with the functional form generated + # by autograph: + # + # for i in data: + # print(i) + # + # generates: + # + # def loop_body(itr): + # i = itr + # print(i) + # ag__.for_stmt(target) + # + # The mechanism lets us infer the type of the itr argument based on that of + # target. + self._for_loop_target_types = {} # Maps body function name to iterated. + self._for_loop_body_fns = {} # Used only to avoid collisions. + + def res_name(self, ns, types_ns, name): + name_str = str(name) + if name_str in TFR_BUILTINS: + return {TFRTypes.TFR_BUILTIN_FUNC}, name_str + if name_str in ns: + ns_val = ns[name_str] + return {type(ns_val)}, ns_val + if name_str in __builtins__: + return {TFRTypes.PY_BUILTIN_FUNC}, __builtins__[name_str] + # This name is not in the namespace because the autograph transformation + # is not backloaded into Python. + if name_str == 'ag__': + return {type(AG_MODULE)}, AG_MODULE + + return None, None + + def res_value(self, ns, value): + # resolves the type of the symbol by the metadata in 'value' + if value is None: + return {TFRTypes.NONE} + if value in (TFRTypes.SHAPE, TFRTypes.TF_TENSOR_SHAPE_FUNC): + # See TFRTypes.__getattribute__. + # TODO(mdan): Replacing the enum with classes would avoid this overlap. + return {value} + # TODO(mdan): Index more efficiently. Could do a name check instead. + if any(v is value for v in AG_MODULE.__dict__.values()): + return {TFRTypes.AG_BUILTIN_FUNC} + if getattr(value, '__name__', None) == 'tensorflow.raw_ops': + return {types.ModuleType} + if hasattr(value, '__module__'): + if isinstance(value, dtypes.DType): + return {TFRTypes.ATTR} + + # All the imported operations, which are not autograph built-ins, are + # considered to be TF raw ops. + # TODO(fengliuai): refine the condition that we only match TensorFlow + # ops here. + return {TFRTypes.TF_RAW_OP} + # TODO(mdan): Is ATTR equivalent to string? + return {_PY_TYPE_TO_TFR.get(type(value), TFRTypes.ATTR)} + + def res_call(self, ns, types_ns, node, f_type, args, keywords): + # resolves the return type of the function call. + name = anno.Basic.QN.of(node.func) + if f_type == (TFRTypes.AG_BUILTIN_FUNC,): + + if name == QN(QN('ag__'), attr='if_stmt'): + nouts = node.args[6].value + # TODO(mdan): Look at the actual types out of if_body. + side_effects = { + qual_names.QN(n.value): {TFRTypes.TENSOR} + for n in node.args[5].elts[:nouts] + } + return {type(None)}, side_effects + + if name == QN(QN('ag__'), attr='for_stmt'): + assert isinstance(node.args[2], ast.Name) + body_fn_name = str(anno.Basic.QN.of(node.args[2])) + assert body_fn_name not in self._for_loop_body_fns, ( + 'Previously used here: {}. Are you reusing the Resolver across ' + 'transformations?').format(self._for_loop_body_fns[body_fn_name]) + self._for_loop_body_fns[body_fn_name] = anno.Basic.ORIGIN.of(node) + + iterated_type = args[0] + assert iterated_type & { + TFRTypes.TENSOR_LIST, TFRTypes.TENSOR, TFRTypes.ATTR + }, ( + iterated_type) + self._for_loop_target_types[body_fn_name] = iterated_type + + return {type(None)}, None + + # TODO(mdan): Actually resolve the type here instead. + ret_type = _AG_FIXED_RETURN_TYPE.get(name.qn[1], None) + if ret_type is not None: + return {ret_type}, None + raise NotImplementedError('return type of {}'.format(name)) + + elif f_type == (TFRTypes.TF_RAW_OP,): + # This is a TF operation, so it should be found in the op_defs. + op_name = name.qn[1] + op_def, _ = self._op_defs.lookup(op_name) + if len(op_def.output_arg) == 1: + return {_get_type_from_proto(op_def.output_arg[0])}, None + return ({tuple(_get_type_from_proto(arg) for arg in op_def.output_arg)}, + None) + + elif f_type == (TFRTypes.PY_BUILTIN_FUNC,): + assert name.is_simple() + if name == QN('range'): + return {TFRTypes.ATTR}, None + + if name == QN('len'): + return {TFRTypes.INDEX}, None + + elif f_type == (TFRTypes.TFR_BUILTIN_FUNC,): + op_name = name.qn[0] + if callable(TFR_BUILTINS[op_name]): + return {TFR_BUILTINS[op_name](*[list(arg)[0] for arg in args])}, None + return {TFR_BUILTINS[op_name]}, None + + elif f_type == (TFRTypes.TF_TENSOR_SHAPE_FUNC,): + return {TFRTypes.TF_TENSOR_SHAPE_LIST}, None + + elif f_type == (types.FunctionType,): + # This is a function call which isn't using tf.raw_op.. + op_name = name.qn[0] + + # 'new TF operation' produces outputs defined by the composition function. + op_def, _ = self._op_defs.lookup(op_name) + if len(op_def.output_arg) == 1: + return {_get_type_from_proto(op_def.output_arg[0])}, None + return ({tuple(_get_type_from_proto(arg) for arg in op_def.output_arg)}, + None) + + raise NotImplementedError('Function:', name, f_type) + + def res_arg(self, ns, types_ns, f_name, name, type_anno, f_is_local): + if f_is_local: + f_name_str = str(f_name) + if f_name_str in self._for_loop_target_types: + # See autograph/converters/control_flow.py - the function has a single + # argument, the iterate before any expansion. + assert self._for_loop_target_types[f_name_str] & {TFRTypes.ATTR} + # Assume all loops are TF loops. Then the iterates are autoboxed into + # Tensors. + return {TFRTypes.INDEX} + else: + return None + + func = ns[f_name] + + op_def, derived_attrs = self._op_defs.lookup(f_name, func) + if op_def is None: + return None + pos = tf_inspect.getfullargspec(func).args.index(str(name)) + + if pos < len(op_def.input_arg): + arg_def = op_def.input_arg[pos] + return {_get_type_from_proto(arg_def)} + elif pos < len(op_def.input_arg) + len(op_def.attr) - len(derived_attrs): + non_derived_attr_pos = pos - len(op_def.input_arg) + for attr_def in op_def.attr: + # derived attribute, skip this one and continue to the next one. + if attr_def.name in derived_attrs: + continue + if non_derived_attr_pos == 0: + return {_get_type_from_proto(None, attr_def)} + non_derived_attr_pos -= 1 + + raise ValueError('Argument is not defined in OpDef: ' + str(name)) + + def res_slice(self, ns, types_ns, node_or_slice, value, slice_): + if not value: + return value + + if isinstance(value, set): + type_tuple = value.pop() + if isinstance(type_tuple, tuple): + value = {type_tuple[node_or_slice]} + else: + value = {type_tuple} + + assert len(value) == 1 + value, = tuple(value) + if value == TFRTypes.TF_TENSOR_SHAPE_LIST: + # TODO(mdan): This is not entirely correct for multi-element slices. + return {int} + elif value in (TFRTypes.TENSOR_LIST, TFRTypes.TENSOR): + # TODO(mdan): This is not entirely correct for multi-element slices. + return {TFRTypes.TENSOR} + else: + return {value} + + def res_compare(self, ns, types_ns, node, left, right): + # TODO(fengliuai): make sure left and right are compatible + return {TFRTypes.I1} + + def res_unop(self, ns, types_ns, node, opnd): + return opnd + + def res_binop(self, ns, types_ns, node, left, right): + # TODO(fengliuai): make sure left and right are compatible + return left + + def _coerce_to_more_specific_type(self, elt_types): + # TODO(mdan): This needs some type theory study. + if TFRTypes.INDEX in elt_types: + # Constants collapse to indices. + elt_types.discard(TFRTypes.I64) + if TFRTypes.TENSOR in elt_types: + # Constants collapse to tensors. + elt_types.discard(TFRTypes.I64) + # Indices collapse to tensors. + elt_types.discard(TFRTypes.INDEX) + return elt_types + + def res_list_literal(self, ns, elt_types): + all_elt_types = set() + for t in elt_types: + all_elt_types |= t + + if len(all_elt_types) != 1: + all_elt_types = self._coerce_to_more_specific_type(all_elt_types) + + if len(all_elt_types) != 1: + raise ValueError('ambiguous list element types: {}'.format(elt_types)) + + if TFRTypes.TENSOR in all_elt_types: + return {TFRTypes.TENSOR_LIST} + return {TFRTypes.ATTR} + + +class SymbolTable(object): + """Symbol Table for python code.""" + + def __init__(self): + self.symbols = [] + self.enter_scope() + self.scf_scope = 0 + # reserved key words + self.insert_symbol('len', 'len', TFRTypes.PY_BUILTIN_FUNC) + + def enter_scope(self, scf_scope=False): + """Enter a new scope - at function level.""" + self.symbols.append({'types': {}, 'symbols': {}}) + self.curr_table = self.symbols[len(self.symbols) - 1] + if scf_scope: + self.scf_scope += 1 + + def insert_symbol(self, name, value, type_): + self.curr_table['symbols'][name] = (value, type_) + # TODO(mdan): Use the inferred type rather than tracking it here. + # The following field is deprecated. + self.curr_table['types'][name] = type_ + return value + + def exit_scope(self): + self.symbols.pop() + self.curr_table = self.symbols[len(self.symbols) - 1] + if self.scf_scope > 0: + self.scf_scope -= 1 + + def in_scf_scope(self): + return self.scf_scope > 0 + + def lookup(self, name): + curr_idx = len(self.symbols) - 1 + while curr_idx >= 0 and (name not in self.symbols[curr_idx]['symbols']): + curr_idx -= 1 + if curr_idx < 0: + return None + return self.symbols[curr_idx]['symbols'][name] + + +class TFRGen(transformer.CodeGenerator): + """Visit the AST and generate MLIR TFR functions.""" + + def __init__(self, ctx, op_defs): + super(TFRGen, self).__init__(ctx) + self.ctx = ctx + self.symbol_table = SymbolTable() + self._op_defs = op_defs + + def _create_mlir_loc(self, loc): + """Creates mlir location from autograph ORIGIN value. + + Args: + loc: OriginInfo + + Returns: + A serialized mlir location string. + """ + if loc is not None and loc.loc.filename: + file_name = os.path.basename(loc.loc.filename) + return 'loc("{}":{}:{})'.format(file_name, loc.loc.lineno, + loc.loc.col_offset) + else: + return 'loc(unknown)' + + def _emit_with_loc(self, op_str, node=None): + """Emit the mlir operation with the location associated with the node. + + Args: + op_str: The mlir operation string to be emitted. + node: The node of the AST tree, the mlir operation translated from. + """ + loc = '' + if node: + loc = self._create_mlir_loc( + anno.getanno(node, anno.Basic.ORIGIN, default=None)) + self.emit(op_str + ' ' + loc) + + def _get_inferred_type(self, node, default=None): + """Return single type or a tuple of types if more than one type.""" + types_ = anno.getanno(node, anno.Static.TYPES, None) + if not types_: + print('WARN: no Static.TYPES annotation. Fix the type inference pass: ') + self.debug_print(node) + return default + + if len(types_) == 1: + type_, = types_ + else: + type_ = types_ + + if default is not None and type_ != default: + print('WARN: type annotation {}({}) does not match {}({})'.format( + type_, type(type_), default, type(default))) + self.debug_print(node) + + return type_ + + def _pack_tensor_list(self, value): + # This is packing a list of tensors, then the axis is 0. + axis = self._ssa_name('zero') + self._emit_with_loc('\n{} = arith.constant 0 : i64'.format(axis)) + casted = self._ssa_name('pack') + self.emit('\n{} = tfr.call @tf__pack({}, {})'.format(casted, value, axis)) + self._emit_with_loc(' : (!tfr.tensor_list, i64) -> !tfr.tensor') + # load the op def of tf.Pack + self._op_defs.lookup('Pack') + return casted, TFRTypes.TENSOR + + def _index_to_I64(self, value, ty): + if ty == TFRTypes.INDEX: + casted = self._ssa_name('casted') + self._emit_with_loc('\n{} = arith.index_cast {} : index to i64'.format( + casted, value)) + return casted, TFRTypes.I64 + else: + return value, ty + + def _i64_to_index(self, value, ty): + if ty == TFRTypes.I64: + casted = self._ssa_name('casted') + self._emit_with_loc('\n{} = arith.index_cast {} : i64 to index'.format( + casted, value)) + return casted, TFRTypes.INDEX + else: + return value, ty + + def _value_to_tensor(self, value, ty, node): + value, ty = self._index_to_I64(value, ty) + cst_tensor = self._ssa_name('cst') + self.emit('\n{} = "tfr.constant_tensor"({})'.format(cst_tensor, value)) + self._emit_with_loc(' : ({}) -> !tfr.tensor'.format(ty), node) + return cst_tensor, TFRTypes.TENSOR + + def _ssa_name(self, prefix): + if isinstance(prefix, qual_names.QN): + assert prefix.is_simple(), 'ANF transform should have cleaned this up' + prefix = prefix.ssf() + return '%' + self.ctx.namer.new_symbol(prefix, set()) + + def _op_def(self, op_name): + return op_def_registry.get(op_name) + + def visit_block(self, block): + return [self.visit(item) for item in block] + + def visit_Pass(self, node): + if self.symbol_table.in_scf_scope(): + self._emit_with_loc('\nscf.yield', node) + else: + self._emit_with_loc('\ntfr.return', node) + + def visit_Attribute(self, node): + node_type = self._get_inferred_type(node, None) + if isinstance(node.value, ast.Name): + if node.value.id == 'ag__': + # some variables are assigned with 'ag__.xxx' method, we should handle + # them following the autograph convensions. + return (node.attr, TFRTypes.AG_BUILTIN_FUNC) + + if node_type == TFRTypes.TF_RAW_OP: + # This branch is used when it is inside tensorflow + return (node.attr, TFRTypes.TF_RAW_OP) + + if node_type == TFRTypes.ATTR: + attr = self._ssa_name('attr') + tfr_type = _TF_DTYPE_TO_TFR.get(node.attr) + self._emit_with_loc( + '\n{} = tfr.constant {} -> !tfr.attr'.format(attr, tfr_type), node) + return (attr, TFRTypes.ATTR) + + value, _ = self.visit(node.value) + tensor_type = self._get_inferred_type(node.value, None) + # TODO(fengliuai): use node_type once it + if node_type == TFRTypes.SHAPE: + print('TODO: use "node_type"') + if node.attr == 'shape' and tensor_type == TFRTypes.TENSOR: + ssa_value = self._ssa_name('shape') + self._emit_with_loc( + '\n{} = tfr.get_shape {} -> !shape.shape'.format(ssa_value, value), + node) + return (ssa_value, TFRTypes.SHAPE) + + if isinstance(node.value, ast.Attribute): + if isinstance(node.value.value, ast.Name): + if node.value.value.id == 'tf' and node.value.attr == 'raw_ops': + return (node.attr, TFRTypes.TF_RAW_OP) + + value, ty = self.visit(node.value) + # TODO(fengliuai): use node_type once it + if node_type == TFRTypes.TF_TENSOR_SHAPE_FUNC: + print('TODO: use "node_type"') + if ty == TFRTypes.SHAPE and node.attr == 'as_list': + return (value, TFRTypes.TF_TENSOR_SHAPE_FUNC) + + raise NotImplementedError('Attribute kind not recognized.') + + def visit_Assign(self, node): + values = self.visit(node.value) + if isinstance(node.targets[0], ast.Tuple): + targets = [elt.id for elt in node.targets[0].elts] + elif isinstance(node.targets[0], ast.Name): + targets = [node.targets[0].id] + else: + raise NotImplementedError('Assignment target type not recognized.') + + if isinstance(values, list): + if isinstance(node.value, ast.Call): + expected = tuple(t for n, t in values) + if len(values) == 1: + expected = expected[0] + elif isinstance(node.value, ast.Tuple): + expected = tuple(t for n, t in values) + else: + raise ValueError('unknown assignment target node', node.value) + ty = self._get_inferred_type(node.value, expected) + + if len(targets) == len(values): + # TODO(mdan): This should already be a tuple. + ty_ = (ty,) if len(values) == 1 else ty + for key, value, t in zip(targets, values, ty_): + ssa_value, _ = value + self.symbol_table.insert_symbol(key, ssa_value, t) + elif len(values) == 1: + name, tys = values[0] + if ty == TFRTypes.TENSOR_LIST: + # assign single tensor_list to multiple variables + for idx, key in enumerate(targets): + idx_name = self._ssa_name('idx') + self._emit_with_loc( + '\n{} = arith.constant {} : index'.format(idx_name, idx), node) + elt_name = self._ssa_name('elt') + self.emit('\n{} = tfr.get_element {}[{}]'.format( + elt_name, name, idx_name)) + self._emit_with_loc(' : (!tfr.tensor_list, index) -> !tfr.tensor', + node) + self.symbol_table.insert_symbol(key, elt_name, TFRTypes.TENSOR) + else: + # assign single value to multiple targets. This single value is + # usually a function return. The return type should be in the tuple of + # the value. + for idx, key in enumerate(targets): + ssa_name = '{}#{}'.format(name, idx) + ssa_type = tys[idx] + self.symbol_table.insert_symbol(key, ssa_name, ssa_type) + elif len(targets) == 1: + ssa_names = [n for n, _ in values] + self.symbol_table.insert_symbol(targets[0], ssa_names, ty) + return + + ty = self._get_inferred_type(node.value, values[1]) + self.symbol_table.insert_symbol(targets[0], values[0], ty) + + def _emit_binary_op(self, op, lhs, lhs_ty, rhs, rhs_ty): + assert lhs_ty, rhs_ty + if isinstance(op, ast.Sub): + code = 'arith.sub' + elif isinstance(op, ast.Add): + code = 'arith.add' + elif isinstance(op, ast.Mult): + code = 'arith.mul' + elif isinstance(op, ast.Div): + code = 'arith.div' + else: + raise NotImplementedError('BinOp operator not recognized' + op) + + if lhs_ty == TFRTypes.I64 or lhs_ty == TFRTypes.I32: + suffix = 'i' + elif lhs_ty == TFRTypes.F32: + suffix = 'f' + else: + raise NotImplementedError('BinOp operand type not recognized' + op) + + ret = self._ssa_name(code) + self._emit_with_loc( + '\n{} = {}{} {}, {} : {}'.format(ret, code, suffix, lhs, rhs, lhs_ty), + op) + return ret, lhs_ty + + def visit_AugAssign(self, node): + lhs, lhs_ty = self.visit(node.target) + rhs, rhs_ty = self.visit(node.value) + ret, ret_ty = self._emit_binary_op(node.op, lhs, lhs_ty, rhs, rhs_ty) + self.symbol_table.insert_symbol(node.target.id, ret, ret_ty) + + def visit_BinOp(self, node): + lhs, lhs_ty = self.visit(node.left) + rhs, rhs_ty = self.visit(node.right) + return self._emit_binary_op(node.op, lhs, lhs_ty, rhs, rhs_ty) + + def visit_BoolOp(self, node): + values = [self.visit(value) for value in node.values] + # TODO(fengliuai): Handle more ast node types. + if isinstance(node.op, ast.Or): + raise NotImplementedError('Or operator not recognized') + elif isinstance(node.op, ast.And): + raise NotImplementedError('And operator not recognized') + + def visit_Call(self, node): + func_name, func_type = self.visit(node.func) + func_type = self._get_inferred_type(node.func, func_type) + if func_type == TFRTypes.AG_BUILTIN_FUNC: + if func_name == 'if_stmt': + cond, _ = self.visit(node.args[0]) + body, _ = self.visit(node.args[1]) + orelse, _ = self.visit(node.args[2]) + get_state, _ = self.visit(node.args[3]) + nouts = int(node.args[6].value) + out_symbols = [] + # The out symbols are just a Tuple of names + for out in node.args[5].elts[:nouts]: + val, ty = self.symbol_table.lookup(out.value) + out_symbols.append(out.value) + return self._visit_if_stmt(cond, body, orelse, get_state, out_symbols, + node) + elif func_name == 'for_stmt': + range_ = self._visit_iter(node.args[0]) + body, _ = self.visit(node.args[2]) + get_state, _ = self.visit(node.args[3]) + loop_carried = [out.value for out in node.args[5].elts] + # TODO(fengliuai): opt is not used here. + return self._visit_for_stmt(range_, body, get_state, loop_carried, node) + elif func_name == 'Undefined': + val = self._ssa_name(node.args[0].value) + return (val, TFRTypes.AG_UNDEFINED_VAL) + elif func_name == 'UndefinedReturnValue': + val = self._ssa_name('return_val') + return (val, TFRTypes.AG_UNDEFINED_VAL) + + if func_type == TFRTypes.TF_RAW_OP: + return self._visit_tf_op(func_name, node.args, node.keywords, node) + + if func_type == TFRTypes.TFR_BUILTIN_FUNC: + return self._visit_tfr_builtins(func_name, node.args, node) + + if func_type == types.FunctionType: + return self._visit_tf_op(func_name, node.args, node.keywords, node) + + if func_type == TFRTypes.TF_TENSOR_SHAPE_FUNC: + return (func_name, TFRTypes.TF_TENSOR_SHAPE_LIST) + + if func_type == TFRTypes.PY_BUILTIN_FUNC: + if func_name == 'len': + arg, ty = self.visit(node.args[0]) + ty = self._get_inferred_type(node.args[0], ty) + if ty == TFRTypes.TF_TENSOR_SHAPE_LIST: + len_value = self._ssa_name('len') + self._emit_with_loc( + '\n{} = shape.rank {} : !shape.shape -> !shape.size'.format( + len_value, arg), node) + size_value = self._ssa_name('len_size') + self._emit_with_loc( + '\n{} = shape.size_to_index {} : !shape.size'.format( + size_value, len_value), node) + elif ty == TFRTypes.TENSOR_LIST: + size_value = self._ssa_name('len') + self._emit_with_loc( + '\n{} = tfr.get_length {} -> index'.format(size_value, arg), node) + return (size_value, TFRTypes.INDEX) + + raise NotImplementedError('call operator not recognized: {} {}'.format( + func_name, func_type)) + + def visit_Compare(self, node): + lhs, lhs_ty = self.visit(node.left) + for op, right in zip(node.ops, node.comparators): + rhs, rhs_ty = self.visit(right) + if isinstance(op, ast.Eq): + pred = 'eq' + elif isinstance(op, ast.Lt): + pred = 'ult' + elif isinstance(op, ast.LtE): + pred = 'ule' + elif isinstance(op, ast.Gt): + pred = 'ugt' + elif isinstance(op, ast.GtE): + pred = 'uge' + elif isinstance(op, ast.NotEq): + pred = 'ne' + else: + raise NotImplementedError('Compare operator not recognized') + + ret = self._ssa_name(pred) + if lhs_ty == TFRTypes.ATTR: + self._emit_with_loc( + '\n{} = tfr.equal {}, {} -> i1'.format(ret, lhs, rhs), node) + else: + if lhs_ty == TFRTypes.I64: + code = 'arith.cmpi' + elif lhs_ty == TFRTypes.F32: + code = 'arith.cmpf' + elif lhs_ty == TFRTypes.INDEX: + code = 'arith.cmpi' + # TODO(fengliuai): the reverse type inference should solve the issue. + rhs, _ = self._i64_to_index(rhs, rhs_ty) + else: + raise NotImplementedError('Compare operand type not recognized') + self._emit_with_loc( + '\n{} = {} "{}", {}, {} : {}'.format(ret, code, pred, lhs, rhs, + lhs_ty), node) + + return ret, TFRTypes.I1 + + def visit_Constant(self, node): + cst_name = self._ssa_name('cst') + if node.value is None: + cst_ty = TFRTypes.NONE + elif isinstance(node.value, bool): + cst_ty = self._get_inferred_type(node) + cst_val = str(node.value).lower() + self._emit_with_loc('\n{} = arith.constant {}'.format(cst_name, cst_val), + node) + else: + cst_ty = self._get_inferred_type(node) + cst_val = node.value + if cst_ty == TFRTypes.ATTR: + self._emit_with_loc( + '\n{} = tfr.constant "{}" -> {}'.format(cst_name, cst_val, cst_ty), + node) + else: + self._emit_with_loc( + '\n{} = arith.constant {} : {}'.format(cst_name, cst_val, cst_ty), + node) + return cst_name, cst_ty + + def visit_FunctionDef(self, node): + op_def, derived_attrs = self._op_defs.lookup(node.name, node, True) + if op_def is None: + # Nested function. Insert it to symbol table for looking up later. + self.symbol_table.insert_symbol(node.name, node, None) + return + op_name = op_def.name + if self.symbol_table.lookup(op_name): + raise LookupError('Composition has not been registered for op: ' + + op_name) + else: + self.symbol_table.insert_symbol(node.name, None, None) + + self.symbol_table.enter_scope() + self.emit('\ntfr.func @tf__{0}('.format(_camel_to_snake(op_name))) + + arg_list = [] + idx = 0 + max_idx = len(op_def.input_arg) + len(op_def.attr) + for arg in node.args.args: + arg_name = self._ssa_name(anno.getanno(arg, anno.Basic.QN)) + arg_type = anno.getanno(arg, anno.Static.TYPES)[0] + + arg_attr = '' + if idx >= len(op_def.input_arg): + attr_def = op_def.attr[idx - len(op_def.input_arg)] + # skip the derived attributes + while attr_def.name in derived_attrs and (idx + 1) < max_idx: + idx += 1 + attr_def = op_def.attr[idx - len(op_def.input_arg)] + if idx >= max_idx: + raise ValueError('Argument is not defined in OpDef: ' + arg_name) + + arg_attr += '{{tfr.name="{}"'.format(attr_def.name) + if attr_def.HasField('default_value'): + default_val = _get_val_from_proto(arg_type, attr_def.default_value) + arg_attr += ',tfr.default={}'.format(default_val) + arg_attr += '}' + + idx += 1 + arg_str = '{}: {}{}'.format(arg_name, arg_type, arg_attr) + arg_list.append(arg_str) + self.symbol_table.insert_symbol(arg.id, arg_name, arg_type) + + ret_type_list = [] + for ret_def in op_def.output_arg: + if ret_def.number_attr or ret_def.type_list_attr: + ret_type_list.append(str(TFRTypes.TENSOR_LIST)) + else: + ret_type_list.append(str(TFRTypes.TENSOR)) + + self.emit('{}) -> ({}) {{'.format(', '.join(arg_list), + ', '.join(ret_type_list))) + self.visit_block(node.body) + self._emit_with_loc('\n}', node) + self.symbol_table.exit_scope() + + def visit_arguments(self, node): + # TODO(fengliuai): return ordered the types and names. + # We need to order the arguments to match the assumption in the TFR dialect. + raise NotImplementedError('arguments not supported.') + + def visit_Lambda(self, node): + raise NotImplementedError('Lambda not supported.') + + def _get_mlir_ssa_values(self, name_prefix, out_types): + """Create MLIR convention SSA values.""" + out_ssa_values = [] + if not out_types: + return '', out_ssa_values + + out_name = self._ssa_name(name_prefix) + if len(out_types) == 1: + out_name_suffix = '' + out_ssa_values.append(out_name) + else: + # For multiple returns, MLIR uses '%s:i' when they are defined and + # '%s#i' when they are used. + out_name_suffix = ':{}'.format(len(out_types)) + for idx, _ in enumerate(out_types): + out_ssa_values.append('{}#{}'.format(out_name, idx)) + + return '{}{}'.format(out_name, out_name_suffix), out_ssa_values + + def _visit_if_stmt(self, cond, body_def, orelse_def, get_state, out_symbols, + node): + self.emit('\n') + ret_str, ret_ssa_values = self._get_mlir_ssa_values( + 'if_stmt', [TFRTypes.TENSOR] * len(out_symbols)) + if ret_ssa_values: + self.emit(ret_str + ' = ') + + out_types = [] + for symbol, ssa_value in zip(out_symbols, ret_ssa_values): + out_types.append(str(TFRTypes.TENSOR)) + + self.emit('scf.if {} -> ({}) {{'.format(cond, ', '.join(out_types))) + # Create a new scope in case the local variables are leaked. + self.symbol_table.enter_scope(scf_scope=True) + self.visit_block(body_def.body) + self.visit_block(get_state.body) + self.symbol_table.exit_scope() + + self.emit('\n} else {') + + # Create a new scope in case the local variables are leaked. + self.symbol_table.enter_scope(scf_scope=True) + self.visit_block(orelse_def.body) + self.visit_block(get_state.body) + self.symbol_table.exit_scope() + + # add ssa values to the symbol table + for symbol, ssa_value in zip(out_symbols, ret_ssa_values): + self.symbol_table.insert_symbol(symbol, ssa_value, TFRTypes.TENSOR) + + self._emit_with_loc('\n}', node) + return list(zip(ret_ssa_values, out_types)) + + def _visit_iter(self, node): + if isinstance(node, ast.Call): + f_name = anno.getanno(node.func, anno.Basic.QN) + if f_name == QN('range'): + args = [self.visit(arg) for arg in node.args] + begin = None + step = None + end = None + if len(args) == 1: + end, end_ty = args[0] + elif len(args) == 2: + begin, begin_ty = args[0] + end, end_ty = args[1] + elif len(args) == 3: + begin, begin_ty = args[0] + end, end_ty = args[1] + step, step_ty = args[2] + + if begin is None: + begin = self._ssa_name('begin') + self._emit_with_loc('\n{} = arith.constant 0 : index'.format(begin), + node) + elif begin_ty != TFRTypes.INDEX: + begin_ = self._ssa_name('begin') + self._emit_with_loc( + '\n{} = arith.index_cast {} : {} to index'.format( + begin_, begin, begin_ty), node) + begin = begin_ + + if end_ty != TFRTypes.INDEX: + end_ = self._ssa_name('end') + self._emit_with_loc( + '\n{} = arith.index_cast {} : {} to index'.format( + end_, end, end_ty), node) + end = end_ + + if step is None: + step = self._ssa_name('step') + self._emit_with_loc('\n{} = arith.constant 1 : index'.format(step), + node) + elif step_ty != TFRTypes.INDEX: + step_ = self._ssa_name('step') + self._emit_with_loc( + '\n{} = arith.index_cast {} : {} to index'.format( + step_, step, step_ty), node) + step = step_ + + return begin, end, step + + raise NotImplementedError('Iterator entity not supported.' + node) + + def _visit_for_stmt(self, range_, body_def, get_state, loop_carried, node): + self.emit('\n') + ret_str, ret_ssa_values = self._get_mlir_ssa_values( + 'for_stmt', [TFRTypes.TENSOR] * len(loop_carried)) + if ret_ssa_values: + self.emit(ret_str + ' = ') + + # Before enter the loop, we use the original ssa values as the initial + # values to the loop iteration arguments. We also create new ssa values as + # the returns of the scf for statements. The symbol table needs to be + # updated to these new ssa values before it enters the scope of the loop. + out_types = [] + init_values = [] + for symbol, ssa_value in zip(loop_carried, ret_ssa_values): + init, ty = self.symbol_table.lookup(symbol) + self.symbol_table.insert_symbol(symbol, ssa_value, ty) + out_types.append(str(ty)) + init_values.append((init, ty)) + + # Create a new scope in case the local variables are leaked. + self.symbol_table.enter_scope(scf_scope=True) + + # Create the iteration variable with index type + assert len(body_def.args.args) == 1 + it_name = body_def.args.args[0].id + it = self._ssa_name(it_name) + self.symbol_table.insert_symbol(it_name, it, TFRTypes.INDEX) + + self.emit('scf.for {} = {} to {} step {} '.format(it, range_[0], range_[1], + range_[2])) + if loop_carried: + iter_args = [] + for symbol, init in zip(loop_carried, init_values): + # create new ssa values for the loop carried variables + it_arg = self._ssa_name('it_arg') + self.symbol_table.insert_symbol(symbol, it_arg, init[1]) + iter_args.append('{} = {}'.format(it_arg, init[0])) + self.emit('iter_args({}) '.format(', '.join(iter_args))) + self.emit('-> ({}) {{'.format(', '.join(out_types))) + else: + self.emit(' {') + self.visit_block(body_def.body) + self.visit_block(get_state.body) + self.symbol_table.exit_scope() + self._emit_with_loc('\n}', node) + return list(zip(ret_ssa_values, out_types)) + + def _emit_default_constant_from_proto(self, attr_def): + """emit mlir constant statement from default value of the ArgDef proto.""" + name = self._ssa_name('cst') + cst_ty = _get_type_from_proto(None, attr_def) + try: + cst_val = _get_val_from_proto(cst_ty, attr_def.default_value) + except AttributeError: + raise AttributeError( + f'attribute "{attr_def.name}" does not have default_value. If the ' + "attribute names from the API and OpDef don't match, please add it " + 'to _ATTRIBUTE_RENAMES.') + if cst_ty == TFRTypes.ATTR: + self._emit_with_loc('\n{} = tfr.constant {} -> {}'.format( + name, cst_val, cst_ty)) + elif cst_ty == TFRTypes.I1: + self._emit_with_loc('\n{} = arith.constant {}'.format(name, cst_val)) + else: + self._emit_with_loc('\n{} = arith.constant {} : {}'.format( + name, cst_val, cst_ty)) + return name, cst_ty + + def visit_keyword(self, node): + return node.arg, self.visit(node.value) + + def _visit_tfr_builtins(self, op_name, args, node): + arg_strs = [] + arg_tys = [] + for arg in args: + value, ty = self.visit(arg) + arg_strs.append(value) + arg_tys.append(ty) + tfr_op_name = 'tfr.' + op_name[5:] + ret_tys = ( + TFR_BUILTINS[op_name](*arg_tys) + if callable(TFR_BUILTINS[op_name]) else TFR_BUILTINS[op_name]) + # Convert the tfr builtin returns to a list. + if isinstance(ret_tys, tuple): + ret_tys = list(ret_tys) + else: + ret_tys = [ret_tys] + + ret_str, ret_ssa_values = self._get_mlir_ssa_values(op_name, ret_tys) + + arg_str = ', '.join(arg_strs) + arg_ty_str = ', '.join(str(ty) for ty in arg_tys) + ret_ty_str = ', '.join(str(ty) for ty in ret_tys) + self._emit_with_loc('\n{} = {}({}) : ({}) -> ({})'.format( + ret_str, tfr_op_name, arg_str, arg_ty_str, ret_ty_str), node) + return list(zip(ret_ssa_values, ret_tys)) + + def _visit_tf_op(self, op_name, args, keywords, node): + op_def, derived_attrs = self._op_defs.lookup(op_name) + ret_tys = [_get_type_from_proto(arg) for arg in op_def.output_arg] + + ret_str, ret_ssa_values = self._get_mlir_ssa_values(op_name, ret_tys) + + arg_strs = [] + ty_strs = [] + for arg in args: + value, ty = self.visit(arg) + arg_strs.append(value) + ty_strs.append(str(ty)) + + input_args = [arg for arg in op_def.input_arg] + attrs_no_default = [ + attr for attr in op_def.attr + if not attr.HasField('default_value') and attr.name not in derived_attrs + ] + attrs_with_default = [ + attr for attr in op_def.attr + if attr.HasField('default_value') and attr.name not in derived_attrs + ] + + kw_args = {} + for arg in keywords: + value, (ssa_name, ty) = self.visit(arg) + ty = self._get_inferred_type(arg.value, ty) + + # TODO(b/203493652): see comment on _ATTRIBUTE_RENAMES + if op_name in _ATTRIBUTE_RENAMES and value in _ATTRIBUTE_RENAMES[op_name]: + value = _ATTRIBUTE_RENAMES[op_name][value] + + kw_args[value] = (ssa_name, ty) + + # tensor arguments and attribute arguments + ordered_args = input_args + attrs_no_default + attrs_with_default + for attr_def in ordered_args[len(args):]: + if attr_def.name in kw_args: + value, ty = kw_args[attr_def.name] + if attr_def in input_args: + if ty in _ATTRIBUTE_TYPES: + # the argument shouldn't be used as tf op calls directly. + value, ty = self._value_to_tensor(value, ty, node) + if ty is TFRTypes.TENSOR_LIST and not _require_tensor_list(attr_def): + value, ty = self._pack_tensor_list(value) + else: + value, ty = self._emit_default_constant_from_proto(attr_def) + arg_strs.append(value) + ty_strs.append(str(ty)) + + if ret_ssa_values: + self.emit('\n{} = '.format(ret_str)) + + self.emit('tfr.call @tf__{}('.format(_camel_to_snake(op_name))) + arg_str = ', '.join(arg_strs) + arg_ty_str = ', '.join(ty_strs) + ret_ty_str = ', '.join([str(ty) for ty in ret_tys]) + self._emit_with_loc( + '{}) : ({}) -> ({})'.format(arg_str, arg_ty_str, ret_ty_str), node) + return list(zip(ret_ssa_values, ret_tys)) + + def visit_If(self, node): + raise NotImplementedError('If not supported.') + + def visit_Name(self, node): + val_and_lookup_type = self.symbol_table.lookup(node.id) + if val_and_lookup_type: + (val, lookup_type) = val_and_lookup_type + elif node.id in TFR_BUILTINS: + val = node.id + lookup_type = anno.getanno(node, anno.Static.TYPES, types.FunctionType) + else: + op_def, _ = self._op_defs.lookup(node.id) + val = op_def.name + lookup_type = anno.getanno(node, anno.Static.TYPES, types.FunctionType) + type_ = self._get_inferred_type(node, lookup_type) + return val, type_ + + def visit_Return(self, node): + values = self.visit(node.value) + if self.symbol_table.in_scf_scope(): + self.emit('\nscf.yield ') + else: + self.emit('\ntfr.return ') + if not values: + return + + if isinstance(values, list): + vals, tys = zip(*values) + else: + vals = values[0] + tys = values[1] + + if isinstance(tys, list) or isinstance(tys, tuple): + tys = [str(t) for t in tys] + self._emit_with_loc('{} : {}'.format(', '.join(vals), ', '.join(tys)), + node) + elif tys != TFRTypes.NONE: + # TODO(fengliuai): scf region yield uses this branch. Fix it. + self._emit_with_loc('{} : {}'.format(vals, tys), node) + + def visit_Subscript(self, node): + val, ty = self.visit(node.value) + type_ = self._get_inferred_type(node.value, ty) + + # TODO(fengliuai): Here we hardcode the node.slice here to get the index + # type. Use the visit method once the type inference is done. + # slice_val, slice_ty = self.visit(node.slice) + s = node.slice + if not isinstance(s, (ast.Tuple, ast.Slice)): + if isinstance(s, ast.Constant): + # TODO(fengliuai): promote to an assignment + idx_val = self._ssa_name('cst') + self._emit_with_loc( + '\n{} = arith.constant {} : index'.format(idx_val, s.value), node) + else: + idx_val, _ = self.visit(s) + else: + raise NotImplementedError('non-index slice not supported.') + + elt = self._ssa_name('elt') + if type_ == TFRTypes.TENSOR_LIST: + self.emit('\n{} = tfr.get_element {}[{}] '.format(elt, val, idx_val)) + self._emit_with_loc(': (!tfr.tensor_list, index) -> !tfr.tensor', node) + return (elt, TFRTypes.TENSOR) + elif type_ == TFRTypes.TF_TENSOR_SHAPE_LIST: + size_ = self._ssa_name('size') + self.emit('\n{} = shape.get_extent {}, {}'.format(size_, val, idx_val)) + self._emit_with_loc(': !shape.shape, index -> !shape.size', node) + self._emit_with_loc( + '\n{} = shape.size_to_index {} : !shape.size'.format(elt, size_), + node) + return (elt, TFRTypes.INDEX) + + def visit_List(self, node): + out_type = self._get_inferred_type(node) + vals = [] + tys = [] + for elt in node.elts: + val, ty = self.visit(elt) + ty = self._get_inferred_type(elt, ty) + if ty in _ATTRIBUTE_TYPES and out_type == TFRTypes.TENSOR_LIST: + # This list is a tensor list, then cast all the input values to tensors. + val, ty = self._value_to_tensor(val, ty, node) + else: + # We shouldn't use index type to build the list because list will be use + # as attribute. + val, ty = self._index_to_I64(val, ty) + vals.append(val) + tys.append(str(ty)) + + list_val = self._ssa_name('list') + self.emit('\n{} = "tfr.build_list"({})'.format(list_val, ', '.join(vals))) + self._emit_with_loc(' : ({}) -> {}'.format(', '.join(tys), out_type), node) + return (list_val, out_type) + + def visit_Tuple(self, node): + return [self.visit(elt) for elt in node.elts] + + def visit_UnaryOp(self, node): + value, ty = self.visit(node.operand) + if isinstance(node.op, ast.USub): + zero_value = self._ssa_name('zero') + ssa_value = self._ssa_name('cst') + if ty == TFRTypes.I32 or ty == TFRTypes.I64: + self._emit_with_loc( + '\n{} = arith.constant 0 : {}'.format(zero_value, ty), node) + self._emit_with_loc( + '\n{} = arith.subi {}, {} : {}'.format(ssa_value, zero_value, value, + ty), node) + elif ty == TFRTypes.F32: + self._emit_with_loc( + '\n{} = arith.constant 0.0 : {}'.format(zero_value, ty), node) + self._emit_with_loc( + '\n{} = arith.subf {}, {} : {}'.format(ssa_value, zero_value, value, + ty), node) + else: + raise NotImplementedError('USub type not recognized: ' + str(ty)) + return ssa_value, ty + raise NotImplementedError('USub operator not recognized') + + def visit_For(self, node): + raise NotImplementedError('For operator not recognized') + + def visit_While(self, node): + raise NotImplementedError('While operator not recognized') + + def visit_Try(self, node): + # Only handles the body of the try statement. + self.visit_block(node.body) + + +def _apply_py_to_tf_passes(node, ctx): + """Apply transformations from PyToTF to match tf.function tracing.""" + # TODO(fengliuai): we don't know which passes are required, thus we evaluate + # each one when the corresponding node is handled. + # copied from PyToTF.transform_ast + node = return_statements.transform(node, ctx, False) + node = control_flow.transform(node, ctx) + return node + + +class TfrGen(transpiler.GenericTranspiler): + """Transforms Python objects into TFR MLIR source code.""" + + def __init__(self, op_defs): + self._op_defs = op_defs + + def transform_ast(self, node, ctx): + node = _apply_py_to_tf_passes(node, ctx) + # TODO(mdan): Enable this. + # node = anf.transform(node, ctx) + + graphs = cfg.build(node) + node = qual_names.resolve(node) + node = activity.resolve(node, ctx) + node = reaching_definitions.resolve(node, ctx, graphs) + node = reaching_fndefs.resolve(node, ctx, graphs) + node = type_inference.resolve(node, ctx, graphs, + TFRTypeResolver(self._op_defs)) + + mlir_generator = TFRGen(ctx, self._op_defs) + mlir_generator.visit(node) + return mlir_generator.code_buffer + + +def tfr_gen(func, op_defs): + """Parse a function and emit the TFR functions.""" + mlir_code, _ = TfrGen(op_defs).transform(func, None) + assert tfr.verify(mlir_code), 'mlir code not verified: {}'.format(mlir_code) + return mlir_code + + +def tfr_funcs_gen_from_module(source, op_defs, method_prefix=None, + op_libraries=None): + """Parse the input source module and emit the TFR functions.""" + + # Load the op library so the op is added to the op registry. This is + # required when the op cc_library couldn't be statically linked in open + # source. + # This is a no op if the op shared library couldn't be found in the same + # directory of the op Python API. + # TODO(fengliuai): make the .so file path configurable. + if op_libraries: + prefix_len = len('gen_') + for m in op_libraries: + lib_dir = os.path.dirname(m.__file__) + lib_name = os.path.basename(m.__file__)[prefix_len:].replace('.py', '.so') + lib_path = os.path.join(lib_dir, lib_name) + if os.path.exists(lib_path): + logging.info('load file: ' + lib_path) + load_library.load_op_library(lib_path) + else: + # The op library is generated from the source module, then we load all the + # .so file in the directory + lib_dir = os.path.dirname(source.__file__) + for lib_name in os.listdir(lib_dir): + if lib_name.endswith('.so'): + lib_path = os.path.join(lib_dir, lib_name) + logging.info('load file: ' + lib_path) + load_library.load_op_library(lib_path) + + py_funcs = [ + func + for name, func in tf_inspect.getmembers(source, tf_inspect.isfunction) + if not method_prefix or name.startswith(method_prefix) + ] + # Sort the methods by the line number, to make sure the definitions are + # processed before the usages. + # TODO(fengliuai): Use type inference resolver to recursively process any + # functions called. + py_funcs = sorted(py_funcs, key=lambda x: x.__code__.co_firstlineno) + mlir_funcs = [tfr_gen(func, op_defs) for func in py_funcs] + + return mlir_funcs + + +def tfr_gen_from_module(source, method_prefix=None, op_libraries=None, + op_defs=OpDefCache()): + """Parse the input source module and emit the TFR and external functions.""" + mlir_funcs = tfr_funcs_gen_from_module( + source, op_defs, method_prefix, op_libraries) + + return '\n'.join(mlir_funcs + op_defs.mlir_external_funcs()) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen_test.py b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9481a3b901b21026ab988c926957541f4cc93d50 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_gen_test.py @@ -0,0 +1,645 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for `tfr_gen` module.""" + +# pylint: disable=missing-function-docstring + +import sys + +from tensorflow.compiler.mlir.python.mlir_wrapper import filecheck_wrapper as fw +from tensorflow.compiler.mlir.tfr.python import composite +from tensorflow.compiler.mlir.tfr.python.tfr_gen import tfr_gen_from_module as tfr_gen +from tensorflow.compiler.mlir.tfr.resources import gen_test_ops as test_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import gen_array_ops as array_ops +from tensorflow.python.ops import gen_math_ops as math_ops +from tensorflow.python.platform import test + + +Composite = composite.Composite + +#--- test fn for mlir location --- + + +@Composite('TestInputNOp') +def _tfr_loc_test(x): + n = 10 + x_sum = x[0] + for i in range(1, n): + x_sum = math_ops.Add(x_sum, x[i]) + return x_sum + + +#--- test fn for tfr tensors --- + + +@composite.Composite('TestNoOp') +def _tfr_tensor_empty_arg(): + pass + + +@composite.Composite('TestIdentityOp') +def _tfr_tensor_tensor(x): + return x + + +@composite.Composite('TestIdentityNOp') +def _tfr_tensor_tensor_list(x): + return x + + +@composite.Composite('TestInputNOp') +def _tfr_tensor_tensor_list_get_elt(x): + return x[1] + + +@composite.Composite('TestOutputNOp') +def _tfr_tensor_tensor_list_output(x): + return [x, x] + + +@composite.Composite('TestTwoInputsOp') +def _tfr_tensor_tensor_list_split(x, y, pred): + z, _ = array_ops.Split(axis=0, value=x, num_split=2) + (y, pred) # pylint: disable=pointless-statement + return z + + +@composite.Composite('TestTwoOutputsOp') +def _tfr_tensor_two_output(x): + z = array_ops.Split(axis=0, value=x, num_split=2) + return z[0], z[1] + + +@composite.Composite('TestNumAttrsOp') +def _tfr_tensor_tensor_with_cst(x1, y1, x2, y2): + x = array_ops.OneHot( + indices=[0, 2, -1, x1], depth=y1, on_value=True, off_value=False) + (x, x2, y2) # pylint: disable=pointless-statement + return + +#--- test fn for scf control flow --- + + +@composite.Composite('TestTwoInputsOp') +def _tfr_control_flow_if(x, y, pred): + if pred: + return x + else: + return y + + +@composite.Composite('TestThreeInputsOp') +def _tfr_control_flow_nested_if(x, y, z, select): + if select == 'x': + return x + elif select == 'y': + return y + else: + return z + + +@composite.Composite('TestInputNOp') +def _tfr_control_flow_range_for(x): + # TODO(fengliuai): use len(x) instead + n = 10 + x_sum = x[0] + for i in range(1, n): + x_sum = math_ops.Add(x_sum, x[i]) + return x_sum + + +@composite.Composite('TestInputNOp') +def _tfr_control_flow_tensor_list_size(ins): + n = len(ins) + if n == 0: + return array_ops.Const(value=[[0, 1], [2, 3]], dtype=dtypes.int64) + else: + return math_ops.AddN(ins) + + +#--- test fn for tf ops --- + + +@composite.Composite('TestComplexTFOp') +def _tfr_tf_ops_complex(lhs, rhs): + left_padding, _ = array_ops.SplitV( + value=lhs, size_splits=[rhs, -1], axis=0, num_split=2) + _, right_padding = array_ops.SplitV( + value=lhs, size_splits=[rhs, rhs], axis=1, num_split=2) + return [left_padding, right_padding] + + +@composite.Composite('TestIdentityOp') +def _tfr_tf_ops_tensor(x): + return array_ops.Identity(x) + + +@composite.Composite('TestTwoInputsOp') +def _tfr_tf_ops_tensors(x, y, pred): + if pred: + return math_ops.Add(x, y) + else: + return array_ops.Concat(0, [x, y]) + + +@composite.Composite('TestInputNOp') +def _tfr_tf_ops_with_defaults(ins): + return test_ops.TestTwoInputsOp(ins[0], ins[1]) + + +#--- test fn for tfr attributes --- + + +@composite.Composite('TestNumAttrsOp') +def _tfr_attrs_num_type(x, y, x1, y1): + # int + z0 = [x, y] + z1 = x == y + z2 = x < y + z3 = x <= y + z4 = x > y + z5 = x >= y + z6 = x != y + z7 = x + y + z8 = x - y + z8 += x + z8 += 1 + (z0, z1, z2, z3, z4, z5, z6, z7, z8) # pylint: disable=pointless-statement + + # float + z9 = x1 > y1 + z10 = x1 + y1 + z11 = [x1, y1] + (z9, z10, z11) # pylint: disable=pointless-statement + return + + +@composite.Composite('TestNonNumAttrsOp') +def _tfr_attrs_tfr_type(x, y, z): + z1 = x == y + z2 = x == 'test' + z3 = y == z + (z1, z2, z3) # pylint: disable=pointless-statement + return + + +#--- test fn for shapes --- + + +@composite.Composite('TestIdentityOp') +def _tfr_shapes(x): + s1 = x.shape + s3 = x.shape.as_list() + + for i in range(len(s3)): + s3[i] # pylint: disable=pointless-statement + + for i in range(1, len(s3), 2): + s3[i] # pylint: disable=pointless-statement + + s5 = array_ops.Shape(x) + (s1, s3, s5) # pylint: disable=pointless-statement + return x + + +#--- test fn for nested functions --- + + +@composite.Composite('TestIdentityNOp') +def _tfr_temp_op(x): + return x + + +@composite.Composite('TestIdentityOp') +def _tfr_temp_use_op(x): + y = _tfr_temp_op([x]) + return y[0] + +#--- test fn for quant built-ins --- + + +# pylint: disable=undefined-variable +@composite.Composite('TestIdentityOp') +def _tfr_quant_test(x): + y = _tfr_quant_raw_data(x) + s, z = _tfr_quant_qparam(x) + s = _tfr_quant_scale_factor(1.0, [s, s]) + s = _tfr_quant_scale_factor(1.0, [s]) + y = math_ops.Sub(y, z) + qmin, qmax = _tfr_quant_act_range('RELU', 1.0, 0) + (qmin, qmax) # pylint: disable=pointless-statement + d = _tfr_quant_rescale(y, s, 0) + e = math_ops.Cast(x=d, DstT=dtypes.int16) + f = math_ops.Cast(x=e, DstT=dtypes.int8) + return f + + +@composite.Composite('TestIdentityNOp') +def _tfr_quant_test_n(x): + y = _tfr_quant_raw_data(x) + return y + + +class TFRGenTestBase(test.TestCase): + + def _check_code(self, tfr_code, exp_tfr_code): + return self.assertTrue(fw.check(str(tfr_code), exp_tfr_code), str(tfr_code)) + + +class TFRGenTensorTest(TFRGenTestBase): + """MLIR Generation Tests for MLIR TFR Program.""" + + def test_tfr_loc(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_loc', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_input_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor) { + CHECK-NEXT: %[[n:.*]] = arith.constant 10 : i64 + CHECK-SAME loc("tfr_gen_test.py":%{{.*}}:6) + CHECK-NEXT: %[[cst:.*]] = arith.constant 0 : index + CHECK-SAME loc("tfr_gen_test.py":%[[sum_line:.*]]:10) + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %x[%[[cst]]] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-SAME loc("tfr_gen_test.py":%[[sum_line]]:10) + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 1 : i64 + CHECK-SAME loc("tfr_gen_test.py":%[[for_line:.*]]:2) + CHECK-NEXT: %[[begin:.*]] = arith.index_cast %[[cst_1]] : i64 to index + CHECK-SAME loc("tfr_gen_test.py":%[[for_line]]:2) + CHECK-NEXT: %[[end:.*]] = arith.index_cast %[[n]] : i64 to index + CHECK-SAME loc("tfr_gen_test.py":%[[for_line]]:2) + CHECK-NEXT: %[[step:.*]] = arith.constant 1 : index + CHECK-SAME loc("tfr_gen_test.py":%[[for_line]]:2) + CHECK-NEXT: %[[for_stmt:.*]] = scf.for %[[itr_1:.*]] = %[[begin]] to %[[end]] step %[[step]] + CHECK-SAME: iter_args(%[[it_arg:.*]] = %[[elt]]) -> (!tfr.tensor) { + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %x[%itr_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-SAME loc("tfr_gen_test.py":%[[add_line:.*]]:34) + CHECK-NEXT: %[[Add:.*]] = tfr.call @tf__add(%[[it_arg]], %[[elt_1]]) : (!tfr.tensor, !tfr.tensor) -> (!tfr.tensor) + CHECK-SAME loc("tfr_gen_test.py":%[[add_line]]:12) + CHECK-NEXT: scf.yield %[[Add]] : !tfr.tensor + CHECK-SAME loc(unknown) + CHECK-NEXT: } + CHECK-SAME loc("tfr_gen_test.py":%[[for_line]]:2) + CHECK-NEXT: %{{.*}} = arith.constant true + CHECK-SAME loc(unknown) + CHECK-NEXT: tfr.return %[[for_stmt]] : !tfr.tensor + CHECK-SAME loc(unknown) + CHECK-NEXT: } + CHECK-SAME loc("tfr_gen_test.py":%{{def_line:.*}}:0) + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_tfr_tensors(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_tensor', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_no_op() -> () { + CHECK-NEXT: tfr.return + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_identity_op(%x: !tfr.tensor) -> (!tfr.tensor) { + CHECK-NEXT: constant true + CHECK-NEXT: tfr.return %x : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_identity_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor_list) { + CHECK-NEXT: constant true + CHECK-NEXT: tfr.return %x : !tfr.tensor_list + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_input_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor) { + CHECK-NEXT: constant true + CHECK-NEXT: %[[index:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[sub:.*]] = tfr.get_element %x[%cst_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: tfr.return %[[sub]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_output_n_op(%x: !tfr.tensor) -> (!tfr.tensor_list) { + CHECK-NEXT: constant true + CHECK-NEXT: %[[list:.*]] = "tfr.build_list"(%x, %x) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: tfr.return %[[list]] : !tfr.tensor_list + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_two_inputs_op(%x: !tfr.tensor, %y: !tfr.tensor, %pred: i1{tfr.name="pred",tfr.default=false}) -> (!tfr.tensor) { + CHECK-NEXT: %[[cst:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[cst_2:.*]] = "tfr.constant_tensor"(%[[cst]]) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[Split:.*]] = tfr.call @tf__split(%[[cst_2]], %x, %[[cst_1]]) : (!tfr.tensor, !tfr.tensor, i64) -> (!tfr.tensor_list) + CHECK-NEXT: %[[cst_4:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %[[Split]][%idx] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_5:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %[[Split]][%idx_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: constant true + CHECK-NEXT: tfr.return %[[elt]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_two_outputs_op(%x: !tfr.tensor) -> (!tfr.tensor, !tfr.tensor) { + CHECK-NEXT: %[[cst:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[cst_2:.*]] = "tfr.constant_tensor"(%[[cst]]) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[Split:.*]] = tfr.call @tf__split(%[[cst_2]], %x, %[[cst_1]]) : (!tfr.tensor, !tfr.tensor, i64) -> (!tfr.tensor_list) + CHECK-NEXT: constant true + CHECK-NEXT: %[[cst_4:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %[[Split]][%cst_4] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_5:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %[[Split]][%cst_5] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: tfr.return %[[elt]], %[[elt_1]] : !tfr.tensor, !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_num_attrs_op(%x1: i64{tfr.name="x1",tfr.default=-10}, %y1: i64{tfr.name="y1",tfr.default=1}, %x2: f32{tfr.name="x2",tfr.default=0.0}, %y2: f32{tfr.name="y2",tfr.default=-3.0}) -> () { + CHECK-NEXT: %[[cst:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[cst_2:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %[[zero:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_3:.*]] = arith.subi %zero, %cst_2 : i64 + CHECK-NEXT: %[[list:.*]] = "tfr.build_list"(%[[cst]], %[[cst_1]], %[[cst_3]], %x1) : (i64, i64, i64, i64) -> !tfr.attr + CHECK-NEXT: %[[cst_4:.*]] = arith.constant true + CHECK-NEXT: %[[cst_5:.*]] = arith.constant false + CHECK-NEXT: %[[cst_6:.*]] = "tfr.constant_tensor"(%[[list]]) : (!tfr.attr) -> !tfr.tensor + CHECK-NEXT: %[[cst_7:.*]] = "tfr.constant_tensor"(%y1) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[cst_8:.*]] = "tfr.constant_tensor"(%[[cst_4]]) : (i1) -> !tfr.tensor + CHECK-NEXT: %[[cst_9:.*]] = "tfr.constant_tensor"(%[[cst_5]]) : (i1) -> !tfr.tensor + CHECK-NEXT: %[[cst_10:.*]] = arith.constant -1 : i64 + CHECK-NEXT: %[[OneHot:.*]] = tfr.call @tf__one_hot(%[[cst_6]], %[[cst_7]], %[[cst_8]], %[[cst_9]], %[[cst_10]]) + CHECK-SAME: (!tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor, i64) -> (!tfr.tensor) + CHECK-NEXT: constant true + CHECK-NEXT: tfr.return + CHECK-NEXT: } + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_tfr_control_flow(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_control_flow', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_two_inputs_op(%x: !tfr.tensor, %y: !tfr.tensor, + CHECK-SAME: %pred: i1{tfr.name="pred",tfr.default=false}) -> (!tfr.tensor) { + CHECK-NEXT: %[[if:.*]] = scf.if %pred -> (!tfr.tensor) { + CHECK-NEXT: arith.constant true + CHECK-NEXT: scf.yield %x : !tfr.tensor + CHECK-NEXT: } else { + CHECK-NEXT: arith.constant true + CHECK-NEXT: scf.yield %y : !tfr.tensor + CHECK-NEXT: } + CHECK-NEXT: tfr.return %if_stmt : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_three_inputs_op(%x: !tfr.tensor, %y: !tfr.tensor, %z: !tfr.tensor, + CHECK-SAME: %select: !tfr.attr{tfr.name="act",tfr.default="z"}) -> (!tfr.tensor) { + CHECK-NEXT: %[[cst:.*]] = tfr.constant "x" -> !tfr.attr + CHECK-NEXT: %[[eq:.*]] = tfr.equal %select, %[[cst]] -> i1 + CHECK-NEXT: %[[if_stmt:.*]] = scf.if %[[eq]] -> (!tfr.tensor) { + CHECK-NEXT: %[[cst_1:.*]] = arith.constant true + CHECK-NEXT: scf.yield %x : !tfr.tensor + CHECK-NEXT: } else { + CHECK-NEXT: %[[cst_2:.*]] = tfr.constant "y" -> !tfr.attr + CHECK-NEXT: %[[eq_1:.*]] = tfr.equal %select, %[[cst_2]] -> i1 + CHECK-NEXT: %[[if_stmt1:.*]] = scf.if %[[eq_1]] -> (!tfr.tensor) { + CHECK-NEXT: %[[cst_3:.*]] = arith.constant true + CHECK-NEXT: scf.yield %y : !tfr.tensor + CHECK-NEXT: } else { + CHECK-NEXT: %[[cst_4:.*]] = arith.constant true + CHECK-NEXT: scf.yield %z : !tfr.tensor + CHECK-NEXT: } + CHECK-NEXT: scf.yield %[[if_stmt1]] : !tfr.tensor + CHECK-NEXT: } + CHECK-NEXT: tfr.return %[[if_stmt]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_input_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor) { + CHECK-NEXT: %[[n:.*]] = arith.constant 10 : i64 + CHECK-NEXT: %[[cst:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %x[%[[cst]]] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %[[begin:.*]] = arith.index_cast %[[cst_1]] : i64 to index + CHECK-NEXT: %[[end:.*]] = arith.index_cast %[[n]] : i64 to index + CHECK-NEXT: %[[step:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[for_stmt:.*]] = scf.for %[[itr_1:.*]] = %[[begin]] to %[[end]] step %[[step]] + CHECK-SAME: iter_args(%[[it_arg:.*]] = %[[elt]]) -> (!tfr.tensor) { + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %x[%itr_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[Add:.*]] = tfr.call @tf__add(%[[it_arg]], %[[elt_1]]) : (!tfr.tensor, !tfr.tensor) -> (!tfr.tensor) + CHECK-NEXT: scf.yield %[[Add]] : !tfr.tensor + CHECK-NEXT: } + CHECK-NEXT: %{{.*}} = arith.constant true + CHECK-NEXT: tfr.return %[[for_stmt]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_input_n_op(%ins: !tfr.tensor_list) -> (!tfr.tensor) { + CHECK: %[[attr:.*]] = tfr.constant i64 -> !tfr.attr + CHECK: %Const = tfr.call @tf__const(%{{.*}}, %[[attr]]) : (!tfr.attr, !tfr.attr) -> (!tfr.tensor) + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_tfr_tf_ops(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_tf_ops', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_complex_tf_op(%lhs: !tfr.tensor, %rhs: !tfr.tensor) -> (!tfr.tensor_list) { + CHECK-NEXT: %[[cst:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %[[zero:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_1:.*]] = arith.subi %[[zero]], %cst : i64 + CHECK-NEXT: %[[cst_2:.*]] = "tfr.constant_tensor"(%[[cst_1]]) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[list:.*]] = "tfr.build_list"(%rhs, %[[cst_2]]) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: %[[cst_3:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[cst_4:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[zero_1:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[pack:.*]] = tfr.call @tf__pack(%[[list]], %[[zero_1]]) : (!tfr.tensor_list, i64) -> !tfr.tensor + CHECK-NEXT: %[[cst_5:.*]] = "tfr.constant_tensor"(%[[cst_3]]) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[SplitV:.*]] = tfr.call @tf__split_v(%lhs, %[[pack]], %[[cst_5]], %[[cst_4]]) + CHECK-NEXT: %[[idx:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %SplitV[%idx] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[idx_1:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %SplitV[%idx_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[list_1:.*]] = "tfr.build_list"(%rhs, %rhs) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: %[[cst_6:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %[[cst_7:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[zero_2:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[pack_1:.*]] = tfr.call @tf__pack(%[[list_1]], %[[zero_2]]) : (!tfr.tensor_list, i64) -> !tfr.tensor + CHECK-NEXT: %[[cst_8:.*]] = "tfr.constant_tensor"(%[[cst_6]]) : (i64) -> !tfr.tensor + CHECK-NEXT: %[[SplitV_1:.*]] = tfr.call @tf__split_v(%lhs, %[[pack_1]], %[[cst_8]], %[[cst_7]]) + CHECK-NEXT: %[[idx_2:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt_2:.*]] = tfr.get_element %SplitV_1[%idx_2] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[idx_3:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[elt_3:.*]] = tfr.get_element %SplitV_1[%idx_3] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_9:.*]] = arith.constant true + CHECK-NEXT: %[[list_2:.*]] = "tfr.build_list"(%[[elt]], %[[elt_3]]) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: tfr.return %[[list_2]] : !tfr.tensor_list + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_identity_op(%x: !tfr.tensor) -> (!tfr.tensor) { + CHECK-NEXT: %cst = arith.constant true + CHECK-NEXT: %[[Id:.*]] = tfr.call @tf__identity(%x) : (!tfr.tensor) -> (!tfr.tensor) + CHECK-NEXT: tfr.return %[[Id]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_two_inputs_op(%x: !tfr.tensor, %y: !tfr.tensor, + CHECK-SAME: %pred: i1{tfr.name="pred",tfr.default=false}) -> (!tfr.tensor) { + CHECK-NEXT: %[[if_stmt:.*]] = scf.if %pred -> (!tfr.tensor) { + CHECK-NEXT: %cst = arith.constant true + CHECK-NEXT: %[[Add:.*]] = tfr.call @tf__add(%x, %y) : (!tfr.tensor, !tfr.tensor) -> (!tfr.tensor) + CHECK-NEXT: scf.yield %[[Add]] : !tfr.tensor + CHECK-NEXT: } else { + CHECK-NEXT: %cst_1 = arith.constant true + CHECK-NEXT: %[[cst_2:.*]] = arith.constant 0 : i64 + CHECK-NEXT: %[[list:.*]] = "tfr.build_list"(%x, %y) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: %[[Concat:.*]] = tfr.call @tf__concat(%[[cst_2]], %[[list]]) : (i64, !tfr.tensor_list) -> (!tfr.tensor) + CHECK-NEXT: scf.yield %[[Concat]] : !tfr.tensor + CHECK-NEXT: } + CHECK-NEXT: tfr.return %[[if_stmt]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_input_n_op(%ins: !tfr.tensor_list) -> (!tfr.tensor) { + CHECK-NEXT: %cst = arith.constant true + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[elt:.*]] = tfr.get_element %ins[%cst_1] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_2:.*]] = arith.constant 1 : index + CHECK-NEXT: %[[elt_1:.*]] = tfr.get_element %ins[%cst_2] : (!tfr.tensor_list, index) -> !tfr.tensor + CHECK-NEXT: %[[cst_3:.*]] = arith.constant false + CHECK-NEXT: %[[call:.*]] = tfr.call @tf__test_two_inputs_op( + CHECK-SAME: %[[elt]], %[[elt_1]], %[[cst_3]]) : (!tfr.tensor, !tfr.tensor, i1) -> (!tfr.tensor) + CHECK-NEXT: tfr.return %[[call]] : !tfr.tensor + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__add_(!tfr.tensor,!tfr.tensor) -> (!tfr.tensor) attributes {T,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__concat_(!tfr.tensor,!tfr.tensor_list) -> (!tfr.tensor) attributes {N,T,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__identity_(!tfr.tensor) -> (!tfr.tensor) attributes {T,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__pack_(!tfr.tensor_list,i64{tfr.name="axis",tfr.type="int"}) -> (!tfr.tensor) attributes {N,T,axis,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__split_v_(!tfr.tensor,!tfr.tensor,!tfr.tensor,i64{tfr.name="num_split",tfr.type="int"}) -> (!tfr.tensor_list) attributes {T,Tlen,f32_,i1_,i32_,i64_,num_split} + + CHECK-LABEL: tfr.func @tf__test_complex_tf_op_(!tfr.tensor,!tfr.tensor,i64{tfr.name="N",tfr.type="int"}) -> (!tfr.tensor_list) attributes {N,T,Tlen,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__test_identity_op_(!tfr.tensor) -> (!tfr.tensor) attributes {T,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__test_input_n_op_(!tfr.tensor_list) -> (!tfr.tensor) attributes {N,T,f32_,i1_,i32_,i64_} + + CHECK-LABEL: tfr.func @tf__test_two_inputs_op_(!tfr.tensor,!tfr.tensor,i1{tfr.name="pred",tfr.type="bool"}) -> (!tfr.tensor) attributes {T,f32_,i1_,i32_,i64_,pred} + + CHECK-LABEL: tfr.func @tf__test_two_outputs_op_(!tfr.tensor) -> (!tfr.tensor,!tfr.tensor) attributes {T,f32_,i1_,i32_,i64_} + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_tfr_attrs(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_attrs', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_num_attrs_op( + CHECK-SAME: %x: i64{tfr.name="x1",tfr.default=-10}, + CHECK-SAME: %y: i64{tfr.name="y1",tfr.default=1}, + CHECK-SAME: %x1: f32{tfr.name="x2",tfr.default=0.0}, + CHECK-SAME: %y1: f32{tfr.name="y2",tfr.default=-3.0}) -> () { + CHECK-NEXT: %{{.*}} = "tfr.build_list"(%x, %y) : (i64, i64) -> !tfr.attr + CHECK-NEXT: %{{.*}} = arith.cmpi "eq", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.cmpi "ult", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.cmpi "ule", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.cmpi "ugt", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.cmpi "uge", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.cmpi "ne", %x, %y : i64 + CHECK-NEXT: %{{.*}} = arith.addi %x, %y : i64 + CHECK-NEXT: %[[sub_1:.*]] = arith.subi %x, %y : i64 + CHECK-NEXT: %[[add_1:.*]] = arith.addi %[[sub_1]], %x : i64 + CHECK-NEXT: %[[cst:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %{{.*}} = arith.addi %[[add_1]], %[[cst]] : i64 + CHECK-NEXT: %{{.*}} = arith.cmpf "ugt", %x1, %y1 : f32 + CHECK-NEXT: %{{.*}} = arith.addf %x1, %y1 : f32 + CHECK-NEXT: %{{.*}} = "tfr.build_list"(%x1, %y1) : (f32, f32) -> !tfr.attr + CHECK-NEXT: %{{.*}} = arith.constant true + CHECK-NEXT: tfr.return + CHECK-NEXT: } + + CHECK-LABEL: tfr.func @tf__test_non_num_attrs_op( + CHECK-SAME: %x: !tfr.attr{tfr.name="z"}, + CHECK-SAME: %y: !tfr.attr{tfr.name="x",tfr.default="hello"}, + CHECK-SAME: %z: !tfr.attr{tfr.name="y",tfr.default=f32}) -> () { + CHECK-NEXT: %{{.*}} = tfr.equal %x, %y -> i1 + CHECK-NEXT: %[[cst:.*]] = tfr.constant "test" -> !tfr.attr + CHECK-NEXT: %{{.*}} = tfr.equal %x, %[[cst]] -> i1 + CHECK-NEXT: %{{.*}} = tfr.equal %y, %z -> i1 + CHECK-NEXT: %{{.*}} = arith.constant true + CHECK-NEXT: tfr.return + CHECK-NEXT: } + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_tf_tensor_shape(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_shapes', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_identity_op(%x: !tfr.tensor) -> (!tfr.tensor) { + CHECK-NEXT: %[[shape:.*]] = tfr.get_shape %x -> !shape.shape + + CHECK-NEXT: %[[shape_1:.*]] = tfr.get_shape %x -> !shape.shape + CHECK-NEXT: %[[len:.*]] = shape.rank %[[shape_1]] : !shape.shape -> !shape.size + CHECK-NEXT: %[[index:.*]] = shape.size_to_index %[[len]] : !shape.size + CHECK-NEXT: %[[begin:.*]] = arith.constant 0 : index + CHECK-NEXT: %[[step:.*]] = arith.constant 1 : index + CHECK-NEXT: scf.for %[[itr_1:.*]] = %[[begin]] to %[[index]] step %[[step]] { + CHECK-NEXT: %[[size:.*]] = shape.get_extent %[[shape_1]], %[[itr_1]]: !shape.shape, index -> !shape.size + CHECK-NEXT: %[[elt:.*]] = shape.size_to_index %[[size]] : !shape.size + CHECK-NEXT: scf.yield + CHECK-NEXT: } + + CHECK-NEXT: %[[cst:.*]] = arith.constant 1 : i64 + CHECK-NEXT: %[[len_1:.*]] = shape.rank %shape_1 : !shape.shape -> !shape.size + CHECK-NEXT: %[[len_size_1:.*]] = shape.size_to_index %[[len_1]] : !shape.size + CHECK-NEXT: %[[cst_1:.*]] = arith.constant 2 : i64 + CHECK-NEXT: %[[begin_1:.*]] = arith.index_cast %[[cst]] : i64 to index + CHECK-NEXT: %[[step_1:.*]] = arith.index_cast %[[cst_1]] : i64 to index + CHECK-NEXT: scf.for %[[itr_3:.*]] = %[[begin_1]] to %[[len_size_1]] step %[[step_1]] + + CHECK: %[[cst:.*]] = tfr.constant i32 -> !tfr.attr + CHECK-NEXT: %[[Shape:.*]] = tfr.call @tf__shape(%x, %[[cst]]) : (!tfr.tensor, !tfr.attr) -> (!tfr.tensor) + CHECK-NEXT: %{{.*}} = arith.constant true + CHECK-NEXT: tfr.return %x : !tfr.tensor + CHECK-NEXT: } + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_temp_function(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_temp', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_identity_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor_list) + + CHECK-LABEL: tfr.func @tf__test_identity_op(%x: !tfr.tensor) -> (!tfr.tensor) { + CHECK-NEXT: %[[list:.*]] = "tfr.build_list"(%x) : (!tfr.tensor) -> !tfr.tensor_list + CHECK-NEXT: %[[call:.*]] = tfr.call @tf__test_identity_n_op(%[[list]]) : (!tfr.tensor_list) + """ + self._check_code(mlir_code, mlir_code_exp) + + def test_quant_builtins(self): + mlir_code = tfr_gen(sys.modules[__name__], '_tfr_quant', [test_ops]) + mlir_code_exp = r""" + CHECK-LABEL: tfr.func @tf__test_identity_op(%x: !tfr.tensor) -> (!tfr.tensor) { + CHECK-NEXT: %[[raw_data:.*]] = tfr.quant_raw_data(%x) : (!tfr.tensor) -> (!tfr.tensor) + CHECK-NEXT: %[[qparam:.*]]:2 = tfr.quant_qparam(%x) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + CHECK: %[[list:.*]] = "tfr.build_list"(%[[qparam]]#0, %[[qparam]]#0) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + CHECK: %[[factor:.*]] = tfr.quant_scale_factor(%{{.*}}, %[[list]]) : (f32, !tfr.tensor_list) -> (!tfr.tensor) + CHECK: %[[list1:.*]] = "tfr.build_list"(%[[factor]]) : (!tfr.tensor) -> !tfr.tensor_list + CHECK: %[[factor1:.*]] = tfr.quant_scale_factor(%{{.*}}, %[[list1]]) : (f32, !tfr.tensor_list) -> (!tfr.tensor) + CHECK-NEXT: %[[Sub:.*]] = tfr.call @tf__sub(%[[raw_data]], %[[qparam]]#1) : (!tfr.tensor, !tfr.tensor) -> (!tfr.tensor) + CHECK: %[[act_range:.*]]:2 = tfr.quant_act_range(%{{.*}}, %{{.*}}, %{{.*}}) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + CHECK: %[[rescale:.*]] = tfr.quant_rescale(%[[Sub]], %[[factor1]], %{{.*}}) : (!tfr.tensor, !tfr.tensor, i64) -> (!tfr.tensor) + CHECK: %[[attr:.*]] = tfr.constant i16 -> !tfr.attr + CHECK: %[[Cast:.*]] = tfr.call @tf__cast(%[[rescale]], %[[attr]], %{{.*}}) : (!tfr.tensor, !tfr.attr, i1) -> (!tfr.tensor) + CHECK: %[[attr_1:.*]] = tfr.constant i8 -> !tfr.attr + CHECK: tfr.call @tf__cast(%[[Cast]], %[[attr_1]], %{{.*}}) : (!tfr.tensor, !tfr.attr, i1) -> (!tfr.tensor) + CHECK: } + + CHECK-LABEL: tfr.func @tf__test_identity_n_op(%x: !tfr.tensor_list) -> (!tfr.tensor_list) { + CHECK-NEXT: %[[raw_data:.*]] = tfr.quant_raw_data(%x) : (!tfr.tensor_list) -> (!tfr.tensor_list) + CHECK: tfr.return %[[raw_data:.*]] : !tfr.tensor_list + CHECK: } + """ + self._check_code(mlir_code, mlir_code_exp) + + +if __name__ == '__main__': + test.main() diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_wrapper.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_wrapper.cc new file mode 100644 index 0000000000000000000000000000000000000000..223649858758e3676720048a0961b0ee928ea9fd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/python/tfr_wrapper.cc @@ -0,0 +1,62 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/Support/LogicalResult.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include "mlir/Dialect/Arith/IR/Arith.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/SCF/IR/SCF.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/IR/AsmState.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "pybind11/pybind11.h" // from @pybind11 +#include "pybind11/stl.h" // from @pybind11 +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" +#include "tensorflow/python/lib/core/pybind11_lib.h" + +PYBIND11_MODULE(tfr_wrapper, m) { + m.def("verify", [](std::string input) { + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext ctx(registry); + ctx.loadAllAvailableDialects(); + + llvm::SourceMgr source_mgr = llvm::SourceMgr(); + source_mgr.AddNewSourceBuffer(llvm::MemoryBuffer::getMemBuffer(input), + llvm::SMLoc()); + auto module = mlir::parseSourceFile(source_mgr, &ctx); + if (!module) { + return false; + } + + mlir::SourceMgrDiagnosticHandler sourceMgrHandler(source_mgr, &ctx); + if (failed(mlir::verify(*module))) { + module->emitError("Invalid MLIR module: failed verification."); + return false; + } + return true; + }); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..aacb9f2d4866a2399cfa2b01ef373a2cbb11e90c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/BUILD @@ -0,0 +1,25 @@ +load("//tensorflow/compiler/mlir/tfr:build_defs.bzl", "gen_op_bindings") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + ":friends", + ], + licenses = ["notice"], +) + +package_group( + name = "friends", + packages = [ + "//tensorflow/compiler/mlir/...", + ], +) + +filegroup( + name = "decomposition_lib", + srcs = ["decomposition_lib.mlir"], +) + +gen_op_bindings(name = "composite") + +gen_op_bindings(name = "test") diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/composite_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/composite_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..904759ae04672ca86d5fd977a7cfff674ed3d235 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/composite_ops.cc @@ -0,0 +1,36 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + +REGISTER_OP("MyAddN") + .Input("inputs: N * T") + .Output("sum: T") + .Attr("N: int >= 1") + .Attr("T: {numbertype, variant}") + .SetIsCommutative() + .SetIsAggregate(); + +REGISTER_OP("MyBiasedDense") + .Input("input: T") + .Input("weight: T") + .Input("bias: T") + .Output("out: T") + .Attr("T: {float, int8}") + .Attr("act: {'', 'relu', 'relu6'} = ''"); + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/decomposition_lib.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/decomposition_lib.mlir new file mode 100644 index 0000000000000000000000000000000000000000..631defb1cb93aeabae0da538e6fd52632e0c51ab --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/decomposition_lib.mlir @@ -0,0 +1,114 @@ +// A test resource file which contains some pre-defined internal tfr.functions +// for decomposition and external tfr.functions for raising the decomposition +// result to the ops in the TF dialect. +// +// All the tfr.func functions are supposed to be translated from the Python +// function with tf.composite annotation. +// All the external tfr.func functions modeles the op signature defined by +// OpDefs. + +tfr.func @tf__my_add_n(%values: !tfr.tensor_list, + %n: i64 {tfr.name="N"}) -> !tfr.tensor { + %index = arith.constant 0 : index + %cst = arith.constant 1 : i64 + %eq = arith.cmpi eq, %n, %cst : i64 + %v1 = tfr.get_element %values[%index] : (!tfr.tensor_list, index) -> !tfr.tensor + %res = scf.if %eq -> !tfr.tensor { + scf.yield %v1 : !tfr.tensor + } else { + %step = arith.index_cast %cst : i64 to index + %end = arith.index_cast %n : i64 to index + %reduce = scf.for %i = %step to %end step %step iter_args(%reduce_iter=%v1) -> !tfr.tensor { + %v = tfr.get_element %values[%i] : (!tfr.tensor_list, index) -> !tfr.tensor + %reduce_next = tfr.call @tf__add(%reduce_iter, %v) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + scf.yield %reduce_next : !tfr.tensor + } + scf.yield %reduce : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +tfr.func @tf__my_add_n_(!tfr.tensor_list, i64 {tfr.name="N"}) -> !tfr.tensor attributes {N,T} + +// Translated from tf.compose Python function. +tfr.func @tf__my_biased_dense(%input: !tfr.tensor, %weight: !tfr.tensor, + %bias: !tfr.tensor, + %act: !tfr.attr{tfr.name="act", tfr.default=""}) -> !tfr.tensor { + %dot = tfr.call @tf__mat_mul(%input, %weight) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + %add = tfr.call @tf__add(%dot, %bias) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + + %relu = tfr.constant "relu" -> !tfr.attr + %relu6 = tfr.constant "relu6" -> !tfr.attr + + %is_relu = tfr.equal %act, %relu -> i1 + %res = scf.if %is_relu -> !tfr.tensor { + %applied_relu = tfr.call @tf__relu(%add) : (!tfr.tensor) -> !tfr.tensor + scf.yield %applied_relu : !tfr.tensor + } else { + %is_relu6 = tfr.equal %act, %relu6 -> i1 + %res1 = scf.if %is_relu6 -> !tfr.tensor { + %applied_relu6 = tfr.call @tf__relu6(%add) : (!tfr.tensor) -> !tfr.tensor + scf.yield %applied_relu6 : !tfr.tensor + } else { + scf.yield %add : !tfr.tensor + } + scf.yield %res1 : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +tfr.func @tf__my_biased_dense_(!tfr.tensor, !tfr.tensor, !tfr.tensor, + !tfr.attr{tfr.name="act", tfr.default=""}) -> !tfr.tensor attributes {T} + +// This is a wong decomposition and used to verify that tf.Elu isn't decomposed +// since its kernel has been registered. +tfr.func @tf__elu_(%input: !tfr.tensor) -> !tfr.tensor { + tfr.return %input : !tfr.tensor +} + +// Translated from: +// +// REGISTER_OP("Add") +// .Input("x: T") +// .Input("y: T") +// .Output("z: T") +// .Attr( +// "T: {bfloat16, half, float, double, uint8, int8, int16, int32, int64, " +// "complex64, complex128, string}") +tfr.func @tf__add_(!tfr.tensor, !tfr.tensor) + -> !tfr.tensor attributes{T} + +// Translated from: +// +// REGISTER_OP("MatMul") +// .Input("a: T") +// .Input("b: T") +// .Output("product: T") +// .Attr("transpose_a: bool = false") +// .Attr("transpose_b: bool = false") +// .Attr("T: {bfloat16, half, float, double, int32, int64, complex64, complex128}") +// T is a derived attribute. +// transpose_a and transpose_b is materialized attributes. +tfr.func @tf__mat_mul_(!tfr.tensor, !tfr.tensor, + i1 {tfr.name="transpose_a", tfr.default=false}, + i1 {tfr.name="transpose_b", tfr.default=false}) + -> !tfr.tensor attributes{T} + +// Translated from: +// +// REGISTER_OP("Relu") +// .Input("features: T") +// .Output("activations: T") +// .Attr("T: {realnumbertype, qint8}") +// T is a derived attribute. +tfr.func @tf__relu_(!tfr.tensor) -> !tfr.tensor attributes{T} + + +// Translated from: +// +// REGISTER_OP("Relu6") +// .Input("features: T") +// .Output("activations: T") +// .Attr("T: {realnumbertype}") +// T is a derived attribute. +tfr.func @tf__relu6_(!tfr.tensor) -> !tfr.tensor attributes{T} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/test_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/test_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..86e06d2c71b451426ae3bd769b4ed217174faa07 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/resources/test_ops.cc @@ -0,0 +1,85 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + +REGISTER_OP("TestNoOp"); + +REGISTER_OP("TestIdentityOp") + .Input("input: T") + .Output("output: T") + .Attr("T: numbertype"); + +REGISTER_OP("TestIdentityNOp") + .Input("input: N * T") + .Output("output: N * T") + .Attr("N: int >= 1") + .Attr("T: numbertype"); + +REGISTER_OP("TestInputNOp") + .Input("input: N * T") + .Output("output: T") + .Attr("N: int >= 1") + .Attr("T: numbertype"); + +REGISTER_OP("TestOutputNOp") + .Input("input: T") + .Output("output: N * T") + .Attr("N: int >= 1") + .Attr("T: numbertype"); + +REGISTER_OP("TestTwoInputsOp") + .Input("lhs: T") + .Input("rhs: T") + .Output("output: T") + .Attr("T: numbertype") + .Attr("pred: bool = false"); + +REGISTER_OP("TestComplexTFOp") + .Input("lhs: T") + .Input("rhs: Tlen") + .Output("output: N * T") + .Attr("N: int >= 1") + .Attr("T: numbertype") + .Attr("Tlen: {int32, int64} = DT_INT64"); + +REGISTER_OP("TestNumAttrsOp") + .Attr("x1: int = -10") + .Attr("y1: int = 1") + .Attr("x2: float = 0.0") + .Attr("y2: float = -3.0"); + +REGISTER_OP("TestNonNumAttrsOp") + .Attr("z: shape") + .Attr("x: string = 'hello'") + .Attr("y: type = DT_FLOAT"); + +REGISTER_OP("TestThreeInputsOp") + .Input("x: T") + .Input("y: T") + .Input("z: T") + .Output("output: T") + .Attr("T: numbertype") + .Attr("act: {'x', 'y', 'z'} = 'z'"); + +REGISTER_OP("TestTwoOutputsOp") + .Input("input: T") + .Output("output1: T") + .Output("output2: T") + .Attr("T: numbertype"); + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/canonicalize.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/canonicalize.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c9fb1f3ddc74b6fcf7cb7b85c348ebcffbf00f8a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/canonicalize.mlir @@ -0,0 +1,230 @@ +// RUN: tfr-opt %s -canonicalize="test-convergence" -verify-diagnostics -split-input-file | FileCheck %s + +// Tests for ops with canonicalization patterns. + +// CHECK-LABEL: get_real_shape +func.func @get_real_shape(%arg0: tensor<1x2xf32>) -> tensor<2xindex> { + %0 = "tfr.cast"(%arg0) : (tensor<1x2xf32>) -> !tfr.tensor + %1 = tfr.get_shape %0 -> !shape.shape + %2 = shape.to_extent_tensor %1 : !shape.shape -> tensor<2xindex> + func.return %2 : tensor<2xindex> + +// CHECK-NEXT: %[[e:.*]] = shape.const_shape [1, 2] : tensor<2xindex> +// CHECK-NEXT: return %[[e]] : tensor<2xindex> +} + +// CHECK-LABEL: equal +func.func @equal() -> (i1, i1, i1, i1) { + %0 = tfr.constant f32 -> !tfr.attr + %1 = tfr.constant f32 -> !tfr.attr + %2 = tfr.constant i32 -> !tfr.attr + %same_type = tfr.equal %0,%1 -> i1 + %diff_type = tfr.equal %0,%2 -> i1 + + %3 = tfr.constant "hello" -> !tfr.attr + %4 = tfr.constant "hello" -> !tfr.attr + %5 = tfr.constant "how are you" -> !tfr.attr + %same_str = tfr.equal %3,%4 -> i1 + %diff_str = tfr.equal %3,%5 -> i1 + func.return %same_type, %diff_type, %same_str, %diff_str : i1, i1, i1, i1 + +// CHECK-DAG: %true = arith.constant true +// CHECK-DAG: %false = arith.constant false +// CHECK-NEXT: return %true, %false, %true, %false : i1, i1, i1, i1 +} + +// ----- + +// CHECK-LABEL: constant_tensor_array +func.func @constant_tensor_array() -> !tfr.tensor { + %0 = tfr.constant [1, -1, 3] -> !tfr.attr + %1 = "tfr.constant_tensor"(%0) : (!tfr.attr) -> !tfr.tensor + func.return %1 : !tfr.tensor + +// CHECK-NEXT: %[[RES:.*]] = "tf.Const"() <{value = dense<[1, -1, 3]> : tensor<3xi64>}> : () -> tensor<3xi64> +// CHECK-NEXT: "tfr.cast"(%[[RES]]) : (tensor<3xi64>) -> !tfr.tensor +// CHECK-NEXT: return +} + +// ----- + +// CHECK-LABEL: constant_tensor_scalar +func.func @constant_tensor_scalar() -> !tfr.tensor { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "tfr.constant_tensor"(%0) : (i32) -> !tfr.tensor + func.return %1 : !tfr.tensor + +// CHECK-NEXT: %[[RES:.*]] = "tf.Const"() <{value = dense<42> : tensor}> : () -> tensor +// CHECK-NEXT: "tfr.cast"(%[[RES]]) : (tensor) -> !tfr.tensor +// CHECK-NEXT: return +} + +// ----- + +// CHECK-LABEL: quant_raw_data +func.func @quant_raw_data(%arg0: tensor<1x10x!quant.uniform>) -> tensor<1x10x!quant.uniform> { + %0 = "tfr.cast"(%arg0) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor + %1 = tfr.quant_raw_data(%0) : (!tfr.tensor) -> !tfr.tensor + %2 = tfr.call @tf__risc(%1) : (!tfr.tensor) -> !tfr.tensor + %3 = "tfr.cast"(%2) : (!tfr.tensor) -> tensor<1x10x!quant.uniform> + %4 = "tfr.cast"(%3) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor + %5 = tfr.quant_raw_data(%4) : (!tfr.tensor) -> !tfr.tensor + %6 = tfr.call @tf__risc(%5) : (!tfr.tensor) -> !tfr.tensor + %7 = "tfr.cast"(%6) : (!tfr.tensor) -> tensor<1x10x!quant.uniform> + func.return %7 : tensor<1x10x!quant.uniform> +// CHECK: %[[c0:.*]] = "tfr.cast"(%[[arg0:.*]]) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor +// CHECK: %[[c1:.*]] = tfr.call @tf__risc(%[[c0]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK: %[[c2:.*]] = tfr.call @tf__risc(%[[c1]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK: %[[c3:.*]] = "tfr.cast"(%[[c2]]) : (!tfr.tensor) -> tensor<1x10x!quant.uniform> +// CHECK: return %[[c3]] +} + +// ----- + +// CHECK-LABEL: quant_raw_data_with_list +func.func @quant_raw_data_with_list(%arg0: !tfr.tensor, %arg1: !tfr.tensor) -> !tfr.tensor { + %cst_1 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %1 = "tfr.cast"(%arg0) : (!tfr.tensor) -> tensor<1x4x4x3x!quant.uniform> + %2 = "tfr.cast"(%arg1) : (!tfr.tensor) -> tensor<1x3x4x3x!quant.uniform> + %3 = "tfr.cast"(%2) : (tensor<1x3x4x3x!quant.uniform>) -> !tfr.tensor + %4 = "tfr.cast"(%1) : (tensor<1x4x4x3x!quant.uniform>) -> !tfr.tensor + %5 = "tfr.build_list"(%3, %4) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %6 = tfr.quant_raw_data(%5) : (!tfr.tensor_list) -> !tfr.tensor_list + %7 = "tfr.cast"(%cst_1) : (tensor) -> !tfr.tensor + %8 = tfr.call @tf__concat(%7, %6) : (!tfr.tensor, !tfr.tensor_list) -> !tfr.tensor + func.return %8 : !tfr.tensor + +// CHECK: %[[CONST_0:.*]] = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor +// CHECK: %[[BUILD_LIST_0:.*]] = "tfr.build_list"(%arg1, %arg0) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list +// CHECK: %[[CAST_0:.*]] = "tfr.cast"(%[[CONST_0]]) : (tensor) -> !tfr.tensor +// CHECK: %[[CONCAT_O:.*]] = tfr.call @tf__concat(%[[CAST_0]], %[[BUILD_LIST_0]]) : (!tfr.tensor, !tfr.tensor_list) -> !tfr.tensor +// CHECK: return %[[CONCAT_O]] : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: cast_with_unranked_quant +func.func @cast_with_unranked_quant(%arg0: tensor<*xi8>, %arg1: tensor<*xi8>) -> tensor<*xf32> { + %0 = "tf.MaximumFloat"(%arg0, %arg1) : (tensor<*xi8>, tensor<*xi8>) -> tensor<*xi8> + %1 = "tfr.cast"(%0) : (tensor<*xi8>) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<*x!quant.uniform> + %3 = "tf.DequantizeFloat"(%2) : (tensor<*x!quant.uniform>) -> tensor<*xf32> + func.return %3 : tensor<*xf32> + // The cast ops should not be removed in this case or it will result in an + // invalid DequantizeFloat op as following: + // %0 = "tf.MaximumFloat"(%arg0, %arg1) : (tensor<*xi8>, tensor<*xi8>) -> tensor<*xi8> + // %1 = "tf.DequantizeFloat"(%0) : (tensor<*xi8>) -> tensor<*xf32> +// CHECK: %[[MAXIMUMFLOAT_0:.*]] = "tf.MaximumFloat"(%arg0, %arg1) : (tensor<*xi8>, tensor<*xi8>) -> tensor<*xi8> +// CHECK: %[[CAST_0:.*]] = "tfr.cast"(%[[MAXIMUMFLOAT_0]]) : (tensor<*xi8>) -> !tfr.tensor +// CHECK: %[[CAST_1:.*]] = "tfr.cast"(%[[CAST_0]]) : (!tfr.tensor) -> tensor<*x!quant.uniform> +// CHECK: %[[DEQUANTIZEFLOAT_0:.*]] = "tf.DequantizeFloat"(%[[CAST_1]]) : (tensor<*x!quant.uniform>) -> tensor<*xf32> +// CHECK: return %[[DEQUANTIZEFLOAT_0]] : tensor<*xf32> +} + +// ----- + +// CHECK-LABEL: quant_qparam +func.func @quant_qparam(%arg0: tensor<1x10x!quant.uniform>) -> (tensor, tensor) { + %0 = "tfr.cast"(%arg0) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor + %scale, %zp = tfr.quant_qparam(%0) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + %1 = "tfr.cast"(%scale) : (!tfr.tensor) -> tensor + %2 = "tfr.cast"(%zp) : (!tfr.tensor) -> tensor + func.return %1, %2 : tensor, tensor + +// CHECK-DAG: %[[scale:.*]] = "tf.Const"() <{value = dense<1.000000e-01> : tensor}> +// CHECK-DAG: %[[zp:.*]] = "tf.Const"() <{value = dense<42> : tensor}> : () -> tensor +// CHECK: return %[[scale]], %[[zp]] +} + +// CHECK-LABEL: quant_qparam_per_channel +func.func @quant_qparam_per_channel(%arg0: tensor<1x3x!quant.uniform>) -> (tensor<3xf32>, tensor<3xi32>) { + %0 = "tfr.cast"(%arg0) : (tensor<1x3x!quant.uniform>) -> !tfr.tensor + %scale, %zp = tfr.quant_qparam(%0) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + %1 = "tfr.cast"(%scale) : (!tfr.tensor) -> tensor<3xf32> + %2 = "tfr.cast"(%zp) : (!tfr.tensor) -> tensor<3xi32> + func.return %1, %2 : tensor<3xf32>, tensor<3xi32> + +// CHECK-DAG: %[[scale:.*]] = "tf.Const"() <{value = dense<[1.000000e-01, 2.000000e-01, 3.000000e-01]> : tensor<3xf32>}> +// CHECK-DAG: %[[zp:.*]] = "tf.Const"() <{value = dense<[1, 2, 3]> : tensor<3xi32>}> : () -> tensor<3xi32> +// CHECK: return %[[scale]], %[[zp]] +} + +// CHECK-LABEL: quant_qparam_invalid +func.func @quant_qparam_invalid(%arg0: tensor<1x3x!quant.calibrated>>) -> (!tfr.tensor, !tfr.tensor) { + %0 = "tfr.cast"(%arg0) : (tensor<1x3x!quant.calibrated>>) -> !tfr.tensor + %scale, %zp = tfr.quant_qparam(%0) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + func.return %scale, %zp: !tfr.tensor, !tfr.tensor + +// CHECK: %[[scale:.*]], %[[zp:.*]] = tfr.quant_qparam(%[[input:.*]]) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) +// CHECK: return %[[scale]], %[[zp]] +} + +// ----- + +// CHECK-LABEL: redundant_cast_with_different_element_type +func.func @redundant_cast_with_different_element_type(%arg0: tensor<*xf32>) -> (tensor<*xi32>, tensor<2xi32>) { + %0 = "tfr.cast"(%arg0) : (tensor<*xf32>) -> !tfr.tensor + %1 = "tfr.cast"(%0) : (!tfr.tensor) -> tensor<*xi32> + %2 = "tfr.cast"(%0) : (!tfr.tensor) -> tensor<2xi32> + func.return %1, %2 : tensor<*xi32>, tensor<2xi32> + +// CHECK: %[[tf_cast_unranked:.*]] = "tf.Cast"(%arg0) <{Truncate = false}> : (tensor<*xf32>) -> tensor<*xi32> +// CHECK: %[[ensure_shape:.*]] = "tf.EnsureShape"(%arg0) <{shape = #tf_type.shape<2>}> : (tensor<*xf32>) -> tensor<2xf32> +// CHECK: %[[tf_cast_ranked:.*]] = "tf.Cast"(%[[ensure_shape]]) <{Truncate = false}> : (tensor<2xf32>) -> tensor<2xi32> +// CHECK: return %[[tf_cast_unranked]], %[[tf_cast_ranked]] : tensor<*xi32>, tensor<2xi32> +} + +// ----- + +// CHECK-LABEL: redundant_cast_with_quant_type +func.func @redundant_cast_with_quant_type(%arg0: tensor<10x!quant.uniform>) -> (tensor<10xi32>) { + %0 = "tfr.cast"(%arg0) : (tensor<10x!quant.uniform>) -> !tfr.tensor + %1 = tfr.quant_raw_data(%0) : (!tfr.tensor) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<10xi8> + %3 = "tf.Cast"(%2) {Truncate = false} : (tensor<10xi8>) -> tensor<10xi32> + func.return %3 : tensor<10xi32> +// CHECK: %[[CAST_0:.*]] = "tfr.cast"(%arg0) : (tensor<10x!quant.uniform>) -> !tfr.tensor +// CHECK: %[[CAST_1:.*]] = "tfr.cast"(%[[CAST_0]]) : (!tfr.tensor) -> tensor<10xi8> +// CHECK: %[[CAST_2:.*]] = "tf.Cast"(%[[CAST_1]]) <{Truncate = false}> : (tensor<10xi8>) -> tensor<10xi32> +// CHECK: return %[[CAST_2]] : tensor<10xi32> +} + +// ----- + +// CHECK-LABEL: build_const_list +func.func @build_const_list() -> !tfr.attr { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "arith.constant"() {value = 41 : i32} : () -> i32 + %2 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + func.return %2 : !tfr.attr + +// CHECK-NEXT: %[[c:.*]] = tfr.constant [42 : i32, 41 : i32] -> !tfr.attr +// CHECK-NEXT: return %[[c]] : !tfr.attr +} + +// ----- + +// CHECK-LABEL: build_high_dim_const_list +func.func @build_high_dim_const_list() -> !tfr.attr { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "arith.constant"() {value = 41 : i32} : () -> i32 + %2 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + %3 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + %4 = "tfr.build_list"(%2, %3) : (!tfr.attr, !tfr.attr) -> !tfr.attr + func.return %4 : !tfr.attr + +// CHECK-NEXT: %[[c:.*]] = tfr.constant {{\[}}[42 : i32, 41 : i32], [42 : i32, 41 : i32]] -> !tfr.attr +// CHECK-NEXT: return %[[c]] : !tfr.attr +} + +// ----- + +// CHECK-LABEL: get_length +func.func @get_length(%arg0: !tfr.tensor, %arg1: !tfr.tensor) -> index { + %0 = "tfr.build_list"(%arg0, %arg1) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %1 = "tfr.get_length"(%0) : (!tfr.tensor_list) -> index + func.return %1 : index + +// CHECK-NEXT: %[[c:.*]] = arith.constant 2 : index +// CHECK-NEXT: return %[[c]] : index +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/control_flow.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/control_flow.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6a3ac68435caefd84190be33693380736aaf3550 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/control_flow.mlir @@ -0,0 +1,57 @@ +// RUN: tfr-opt %s -tfr-decompose -verify-diagnostics -split-input-file | FileCheck %s + +tfr.func @tf__my_pack(%values: !tfr.tensor_list, + %n: i32 {tfr.name="N"}, + %axis: i32 {tfr.name="axis"}) -> !tfr.tensor { + %index = arith.constant 0 : index + %cst = arith.constant 1 : i32 + %eq = arith.cmpi eq, %n, %cst : i32 + %v1 = tfr.get_element %values[%index] : (!tfr.tensor_list, index) -> !tfr.tensor + %temp = tfr.call @tf__expand_dims(%v1, %axis) : (!tfr.tensor, i32) -> !tfr.tensor + %res = scf.if %eq -> !tfr.tensor { + scf.yield %temp : !tfr.tensor + } else { + %step = arith.index_cast %cst : i32 to index + %end = arith.index_cast %n : i32 to index + %reduce = scf.for %i = %step to %end step %step iter_args(%reduce_iter=%temp) -> !tfr.tensor { + %v = tfr.get_element %values[%i] : (!tfr.tensor_list, index) -> !tfr.tensor + %temp1 = tfr.call @tf__expand_dims(%v, %axis) : (!tfr.tensor, i32) -> !tfr.tensor + %reduce_next = tfr.call @tf__risc_concat(%reduce_iter, %temp1, %axis) : (!tfr.tensor, !tfr.tensor, i32) -> !tfr.tensor + scf.yield %reduce_next : !tfr.tensor + } + scf.yield %reduce : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +// CHECK-LABEL: pack_one +func.func @pack_one(%arg0: tensor<2x3xf32>) -> tensor<1x2x3xf32> { + %0 = "tf.MyPack"(%arg0) {N=1:i32, axis=0:i32} : (tensor<2x3xf32>) -> tensor<1x2x3xf32> + func.return %0 : tensor<1x2x3xf32> + +// CHECK-NEXT: %[[AXIS:.*]] = arith.constant 0 : i32 +// CHECK-NEXT: %[[CAST:.*]] = "tfr.cast"(%arg0) : (tensor<2x3xf32>) -> !tfr.tensor +// CHECK-NEXT: %[[ED:.*]] = tfr.call @tf__expand_dims(%[[CAST]], %[[AXIS]]) : (!tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[BACK:.*]] = "tfr.cast"(%[[ED]]) : (!tfr.tensor) -> tensor<1x2x3xf32> +// CHECK-NEXT: return %[[BACK]] : tensor<1x2x3xf32> +} + +// CHECK-LABEL: pack_multiple +func.func @pack_multiple(%arg0: tensor<2x3xf32>, + %arg1: tensor<2x3xf32>, + %arg2: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyPack"(%arg0, %arg1, %arg2) {N=3:i32, axis=0:i32} : (tensor<2x3xf32>, tensor<2x3xf32>, tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[AXIS:.*]] = arith.constant 0 : i32 +// CHECK-NEXT: %[[CAST0:.*]] = "tfr.cast"(%arg0) : (tensor<2x3xf32>) -> !tfr.tensor +// CHECK-NEXT: %[[CAST1:.*]] = "tfr.cast"(%arg1) : (tensor<2x3xf32>) -> !tfr.tensor +// CHECK-NEXT: %[[CAST2:.*]] = "tfr.cast"(%arg2) : (tensor<2x3xf32>) -> !tfr.tensor +// CHECK-NEXT: %[[EX0:.*]] = tfr.call @tf__expand_dims(%[[CAST0]], %[[AXIS]]) : (!tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[EX1:.*]] = tfr.call @tf__expand_dims(%[[CAST1]], %[[AXIS]]) : (!tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[CONCAT1:.*]] = tfr.call @tf__risc_concat(%[[EX0]], %[[EX1]], %c0_i32) : (!tfr.tensor, !tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[EX2:.*]] = tfr.call @tf__expand_dims(%[[CAST2]], %[[AXIS]]) : (!tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[CONCAT2:.*]] = tfr.call @tf__risc_concat(%[[CONCAT1]], %[[EX2]], %[[AXIS]]) : (!tfr.tensor, !tfr.tensor, i32) -> !tfr.tensor +// CHECK-NEXT: %[[BACK:.*]] = "tfr.cast"(%[[CONCAT2]]) : (!tfr.tensor) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[BACK]] : tensor<3x2x3xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/decompose.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/decompose.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ea44f7f4770bf1feca1a9423be28bd8be62ff323 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/decompose.mlir @@ -0,0 +1,289 @@ +// RUN: tfr-opt %s -tfr-decompose -verify-diagnostics | FileCheck %s + +// Definitions for ops that are being used in the tests. +// ex) tf.MyOp refers to tfr.func @tf__my_op + +// CHECK-LABEL: @tf__fake_no_op +tfr.func @tf__fake_no_op(%arg0: !tfr.tensor) -> !tfr.tensor { + tfr.return %arg0 : !tfr.tensor + +// CHECK-NEXT: tfr.return %arg0 : !tfr.tensor +} + +// CHECK-LABEL: @tf__intermediate +tfr.func @tf__intermediate(%arg0: !tfr.tensor) -> !tfr.tensor { + %0 = tfr.call @tf__risc(%arg0) : (!tfr.tensor) -> !tfr.tensor + tfr.return %0 : !tfr.tensor + +// CHECK-NEXT: %[[id:.*]] = tfr.call @tf__risc(%arg0) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: tfr.return %[[id]] : !tfr.tensor +} + +// CHECK-LABEL: @tf__fused_n +tfr.func @tf__fused_n( + %arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: index {tfr.name="A",tfr.default=1:index}) + -> !tfr.tensor_list { + %0 = tfr.call @tf__intermediate(%arg0) : (!tfr.tensor) -> !tfr.tensor + %1 = tfr.get_element %arg1[%arg2] : (!tfr.tensor_list, index) -> !tfr.tensor + %2 = tfr.call @tf__intermediate(%1) : (!tfr.tensor) -> !tfr.tensor + %3 = "tfr.build_list"(%0, %2) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + tfr.return %3 : !tfr.tensor_list + +// CHECK-NEXT: %[[id1:.*]] = tfr.call @tf__intermediate(%arg0) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[ge:.*]] = tfr.get_element %arg1[%arg2] : (!tfr.tensor_list, index) -> !tfr.tensor +// CHECK-NEXT: %[[id2:.*]] = tfr.call @tf__intermediate(%[[ge]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[bl:.*]] = "tfr.build_list"(%[[id1]], %[[id2]]) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list +// CHECK-NEXT: tfr.return %[[bl]] : !tfr.tensor_list +} + +// CHECK-LABEL: @tf__my_max_pool +tfr.func @tf__my_max_pool(%input_: !tfr.tensor, %stride_w: i64{tfr.name="stride_w"}, %stride_h: i64{tfr.name="stride_h"}) -> (!tfr.tensor) { + %cst_1 = arith.constant 1 : i64 + %stride = "tfr.build_list"(%cst_1, %stride_w, %stride_h, %cst_1) : (i64, i64, i64, i64) -> !tfr.attr + %filter = tfr.constant [1, 2, 2, 1] -> !tfr.attr + %padding = tfr.constant "VALID" -> !tfr.attr + %explicit_paddings = tfr.constant [] -> !tfr.attr + %data_format = tfr.constant "NHWC" -> !tfr.attr + %MaxPool = tfr.call @tf__max_pool(%input_, %stride, %filter, %padding, %explicit_paddings, %data_format) : (!tfr.tensor, !tfr.attr, !tfr.attr, !tfr.attr, !tfr.attr, !tfr.attr) -> (!tfr.tensor) + tfr.return %MaxPool : !tfr.tensor +// CHECK: tf__max_pool +} + +// CHECK-LABEL: @tf__cast_float +tfr.func @tf__cast_float(%input_: !tfr.tensor, %out_type: !tfr.attr{tfr.name="out_type"}) -> (!tfr.tensor) { + %false = arith.constant false + %cast = tfr.call @tf__cast(%input_, %out_type, %false) : (!tfr.tensor, !tfr.attr, i1) -> (!tfr.tensor) + tfr.return %cast : !tfr.tensor +} + +// end op definitions + +// CHECK-LABEL: decompose_tf_no_op +func.func @decompose_tf_no_op(%arg0: tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> { + %0 = "tf.FakeNoOp"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> + func.return %0 : tensor<1x2x3x4x!tf_type.string> + +// CHECK-NEXT: return %arg0 +} + +// CHECK-LABEL: decompose_tf_intermediate +func.func @decompose_tf_intermediate(%arg0: tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> { + %0 = "tf.Intermediate"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> + func.return %0 : tensor<1x2x3x4x!tf_type.string> + +// CHECK-NEXT: %[[casted:.*]] = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor +// CHECK-NEXT: %[[id:.*]] = tfr.call @tf__risc(%[[casted]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[back:.*]] = "tfr.cast"(%[[id]]) : (!tfr.tensor) -> tensor<1x2x3x4x!tf_type.string> +// CHECK-NEXT: return %[[back]] +} + +// CHECK-LABEL: decompose_fused_n_default +func.func @decompose_fused_n_default(%arg0: tensor<1x2x3x4x!tf_type.string>, %arg1: tensor, %arg2: tensor) -> tensor { + %0:2 = "tf.FusedN"(%arg0, %arg1, %arg2) : (tensor<1x2x3x4x!tf_type.string>, tensor, tensor) -> (tensor<1x2x3x4x!tf_type.string>, tensor) + func.return %0#1 : tensor + +// CHECK-NEXT: %[[in0:.*]] = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor +// CHECK-NEXT: %[[in2:.*]] = "tfr.cast"(%arg2) : (tensor) -> !tfr.tensor +// CHECK-NEXT: %[[id0:.*]] = tfr.call @tf__risc(%[[in0]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[id2:.*]] = tfr.call @tf__risc(%[[in2]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[back:.*]] = "tfr.cast"(%[[id2]]) : (!tfr.tensor) -> tensor +// CHECK-NEXT: return %[[back]] : tensor +} + +// CHECK-LABEL: decompose_fused_n +func.func @decompose_fused_n(%arg0: tensor<1x2x3x4x!tf_type.string>, %arg1: tensor, %arg2: tensor) -> tensor { + %0:2 = "tf.FusedN"(%arg0, %arg1, %arg2) {A=0:index} : (tensor<1x2x3x4x!tf_type.string>, tensor, tensor) -> (tensor<1x2x3x4x!tf_type.string>, tensor) + func.return %0#1 : tensor + +// CHECK-NEXT: %[[in0:.*]] = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor +// CHECK-NEXT: %[[in1:.*]] = "tfr.cast"(%arg1) : (tensor) -> !tfr.tensor +// CHECK-NEXT: %[[id0:.*]] = tfr.call @tf__risc(%[[in0]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[id1:.*]] = tfr.call @tf__risc(%[[in1]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK-NEXT: %[[back:.*]] = "tfr.cast"(%[[id1]]) : (!tfr.tensor) -> tensor +// CHECK-NEXT: return %[[back]] : tensor +} + +// CHECK-LABEL: attribute_propagate_direct +func.func @attribute_propagate_direct(%arg0: tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> { + %0 = "tf.Intermediate"(%arg0) {_tpu_replicate, device="hello"} : (tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> + func.return %0 : tensor<1x2x3x4x!tf_type.string> + +// CHECK-NEXT: %[[casted:.*]] = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor +// CHECK-NEXT: %[[id:.*]] = tfr.call @tf__risc(%[[casted]]) {_tpu_replicate, device = "hello"} +// CHECK-NEXT: %[[back:.*]] = "tfr.cast"(%[[id]]) : (!tfr.tensor) -> tensor<1x2x3x4x!tf_type.string> +// CHECK-NEXT: return %[[back]] +} + +// CHECK-LABEL: attribute_propagate +func.func @attribute_propagate(%arg0: tensor<1x2x3x4x!tf_type.string>, %arg1: tensor, %arg2: tensor) -> tensor { + %0:2 = "tf.FusedN"(%arg0, %arg1, %arg2) {A=0:index, _tpu_replicate, device="hello"} : (tensor<1x2x3x4x!tf_type.string>, tensor, tensor) -> (tensor<1x2x3x4x!tf_type.string>, tensor) + func.return %0#1 : tensor + +// CHECK-NEXT: %[[in0:.*]] = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor +// CHECK-NEXT: %[[in1:.*]] = "tfr.cast"(%arg1) : (tensor) -> !tfr.tensor +// CHECK-NEXT: %[[id0:.*]] = tfr.call @tf__risc(%[[in0]]) {_tpu_replicate, device = "hello"} +// CHECK-NEXT: %[[id1:.*]] = tfr.call @tf__risc(%[[in1]]) {_tpu_replicate, device = "hello"} +// CHECK-NEXT: %[[back:.*]] = "tfr.cast"(%[[id1]]) : (!tfr.tensor) -> tensor +// CHECK-NEXT: return %[[back]] : tensor +} + +// CHECK: attribute_cast +func.func @attribute_cast(%arg0: tensor<1x4x4x1xf32>) -> tensor<1x2x2x1xf32> { + %0 = "tfr.cast"(%arg0) : (tensor<1x4x4x1xf32>) -> !tfr.tensor + %stride_i32 = arith.constant 2 : i32 + %1 = tfr.call @tf__my_max_pool(%0, %stride_i32, %stride_i32) : (!tfr.tensor, i32, i32) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<1x2x2x1xf32> + func.return %2 : tensor<1x2x2x1xf32> +// CHECK: tf__max_pool +} + +// CHECK-LABEL: no_tf_canonicalization +func.func @no_tf_canonicalization(%arg0: tensor<8xi1>, %arg1: tensor<8x3xf32>, %arg2: tensor<8x3xf32>) -> tensor<8x3xf32> { + %0 = "tf.Select"(%arg0, %arg1, %arg2) : (tensor<8xi1>, tensor<8x3xf32>, tensor<8x3xf32>) -> tensor<8x3xf32> + func.return %0: tensor<8x3xf32> + +// CHECK: "tf.Select" +} + +// CHECK-LABEL: denied_attribute +func.func @denied_attribute(%arg0: tensor<1x2x3x4x!tf_type.string>, %arg1: tensor, %arg2: tensor) -> tensor { + // expected-error@+1 {{Denied unregistered attribute was found: denied_attr}} + %0:2 = "tf.FusedN"(%arg0, %arg1, %arg2) {A=0:index, denied_attr} : (tensor<1x2x3x4x!tf_type.string>, tensor, tensor) -> (tensor<1x2x3x4x!tf_type.string>, tensor) + func.return %0#1 : tensor + +// CHECK-NEXT: "tf.FusedN"(%arg0, %arg1, %arg2) {A = 0 : index, denied_attr} +} + +// CHECK-LABEL: quantized_tensor +func.func @quantized_tensor(%arg0: tensor<1x10x!quant.uniform>) -> tensor<1x10x!quant.uniform> { + %0 = "tf.Intermediate"(%arg0) : (tensor<1x10x!quant.uniform>) -> tensor<1x10x!quant.uniform> + func.return %0 : tensor<1x10x!quant.uniform> + +// CHECK: "tfr.cast"(%[[arg0:.*]]) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor +// CHECK: "tfr.cast"(%[[result:.*]]) : (!tfr.tensor) -> tensor<1x10x!quant.uniform> +} +// CHECK-LABEL: decompose_quant_act_range +func.func @decompose_quant_act_range() -> !tfr.tensor_list { + %scale = arith.constant 0.1 : f32 + %zp = arith.constant 42 : i64 + %none_attr = tfr.constant "NONE" -> !tfr.attr + %relu_attr = tfr.constant "RELU" -> !tfr.attr + %relu6_attr = tfr.constant "RELU6" -> !tfr.attr + %reluN1_1_attr = tfr.constant "RELU_N1_TO_1" -> !tfr.attr + %none:2 = "tfr.quant_act_range"(%none_attr, %scale, %zp) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + %relu:2 = "tfr.quant_act_range"(%relu_attr, %scale, %zp) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + %relu6:2 = "tfr.quant_act_range"(%relu6_attr, %scale, %zp) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + %reluN1_1:2 = "tfr.quant_act_range"(%reluN1_1_attr, %scale, %zp) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + %result = "tfr.build_list"( + %none#0, %none#1, %relu#0, %relu#1, + %relu6#0, %relu6#1, %reluN1_1#0, %reluN1_1#1) : ( + !tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor, + !tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + func.return %result : !tfr.tensor_list +// CHECK-DAG: %[[N_128:.*]] = arith.constant -128 : i32 +// CHECK-DAG: %[[N32:.*]] = arith.constant 32 : i32 +// CHECK-DAG: %[[N42:.*]] = arith.constant 42 : i32 +// CHECK-DAG: %[[N52:.*]] = arith.constant 52 : i32 +// CHECK-DAG: %[[N102:.*]] = arith.constant 102 : i32 +// CHECK-DAG: %[[N127:.*]] = arith.constant 127 : i32 +// CHECK-NEXT: %[[none_min:.*]] = "tfr.constant_tensor"(%[[N_128]]) +// CHECK-NEXT: %[[none_max:.*]] = "tfr.constant_tensor"(%[[N127]]) +// CHECK-NEXT: %[[relu_min:.*]] = "tfr.constant_tensor"(%[[N42]]) +// CHECK-NEXT: %[[relu_max:.*]] = "tfr.constant_tensor"(%[[N127]]) +// CHECK-NEXT: %[[relu6_min:.*]] = "tfr.constant_tensor"(%[[N42]]) +// CHECK-NEXT: %[[relu6_max:.*]] = "tfr.constant_tensor"(%[[N102]]) +// CHECK-NEXT: %[[reluN1_1_min:.*]] = "tfr.constant_tensor"(%[[N32]]) +// CHECK-NEXT: %[[reluN1_1_max:.*]] = "tfr.constant_tensor"(%[[N52]]) +// CHECK-NEXT: %[[result:.*]] = "tfr.build_list"(%[[none_min]], %[[none_max]], %[[relu_min]], %[[relu_max]], +// CHECK-SAME: %[[relu6_min]], %[[relu6_max]], %[[reluN1_1_min]], %[[reluN1_1_max]] +// CHECK-NEXT: return %[[result]] +} + +// CHECK-LABEL: decompose_quant_act_range_invalid +func.func @decompose_quant_act_range_invalid() -> (!tfr.tensor, !tfr.tensor) { + %scale = arith.constant 0.1 : f32 + %zp = arith.constant 42 : i64 + %elu_attr = tfr.constant "ELU" -> !tfr.attr + %min, %max = "tfr.quant_act_range"(%elu_attr, %scale, %zp) : (!tfr.attr, f32, i64) -> (!tfr.tensor, !tfr.tensor) + func.return %min, %max : !tfr.tensor, !tfr.tensor +// CHECK: %[[elu_attr:.*]] = tfr.constant "ELU" -> !tfr.attr +// CHECK: %[[min:.*]], %[[max:.*]] = tfr.quant_act_range(%[[elu_attr]] +// CHECK: return %[[min]], %[[max]] +} + +// CHECK-LABEL: decompose_quant_scale_factor +func.func @decompose_quant_scale_factor() -> (!tfr.tensor, !tfr.tensor) { + %output_scale = arith.constant 0.1 : f32 + %input_scale = arith.constant 0.25 : f32 + %filter_scale = arith.constant 0.4 : f32 + %input_scale_tensor = "tfr.constant_tensor"(%input_scale) : (f32) -> !tfr.tensor + %filter_scale_tensor = "tfr.constant_tensor"(%filter_scale) : (f32) -> !tfr.tensor + %list = "tfr.build_list"(%input_scale_tensor, %filter_scale_tensor) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %out = "tfr.quant_scale_factor"(%output_scale, %list) : (f32, !tfr.tensor_list) -> !tfr.tensor + %perchannel_scale = arith.constant dense<[0.4, 4.0]> : tensor<2xf32> + %perchannel_scale_tensor = "tfr.cast"(%perchannel_scale) : (tensor<2xf32>) -> !tfr.tensor + %list2 = "tfr.build_list"(%input_scale_tensor, %perchannel_scale_tensor) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %perchannel = "tfr.quant_scale_factor"(%output_scale, %list2) : (f32, !tfr.tensor_list) -> !tfr.tensor + func.return %out, %perchannel : !tfr.tensor, !tfr.tensor +// CHECK-DAG: %[[scale_factors:.*]] = "tf.Const"() <{value = dense<[1.000000e+00, 1.000000e+01]> : tensor<2xf32>}> : () -> tensor<2xf32> +// CHECK-DAG: %[[scale_factor:.*]] = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor +// CHECK: %[[cast:.*]] = "tfr.cast"(%[[scale_factor]]) : (tensor) -> !tfr.tensor +// CHECK: %[[cast_perchannel:.*]] = "tfr.cast"(%[[scale_factors]]) : (tensor<2xf32>) -> !tfr.tensor +// CHECK: return %[[cast]], %[[cast_perchannel]] : !tfr.tensor, !tfr.tensor +} + +// CHECK-LABEL: decompose_quant_scale_factor_invalid +func.func @decompose_quant_scale_factor_invalid() -> !tfr.tensor { + %output_scale = arith.constant 0.1 : f32 + %input_scale = arith.constant 0.25 : f32 + %filter_scale = arith.constant 0.4 : f32 + %input_scale_tensor = "tfr.constant_tensor"(%input_scale) : (f32) -> !tfr.tensor + %filter_scale_tensor = "tfr.constant_tensor"(%filter_scale) : (f32) -> !tfr.tensor + %list = "tfr.build_list"(%input_scale_tensor, %filter_scale_tensor, %input_scale_tensor) : (!tfr.tensor, !tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %out = "tfr.quant_scale_factor"(%output_scale, %list) : (f32, !tfr.tensor_list) -> !tfr.tensor + func.return %out : !tfr.tensor +// CHECK-DAG: %[[cst_0:.*]] = arith.constant 1.000000e-01 : f32 +// CHECK-DAG: %[[cst_1:.*]] = "tf.Const"() <{value = dense<2.500000e-01> : tensor}> : () -> tensor +// CHECK-DAG: %[[cst_2:.*]] = "tf.Const"() <{value = dense<4.000000e-01> : tensor}> : () -> tensor +// CHECK: %[[tfrcast0:.*]] = "tfr.cast"(%[[cst_1]]) : (tensor) -> !tfr.tensor +// CHECK: %[[tfrcast1:.*]] = "tfr.cast"(%[[cst_2]]) : (tensor) -> !tfr.tensor +// CHECK: %[[list:.*]] = "tfr.build_list"(%[[tfrcast0]], %[[tfrcast1]], %[[tfrcast0]]) : (!tfr.tensor, !tfr.tensor, !tfr.tensor) -> !tfr.tensor_list +// CHECK: %[[qsf:.*]] = tfr.quant_scale_factor(%[[cst_0]], %[[list]]) : (f32, !tfr.tensor_list) -> !tfr.tensor +// CHECK: return %[[qsf]] : !tfr.tensor +} + +// CHECK-LABEL: decompose_quant_rescale +func.func @decompose_quant_rescale(%arg0: tensor<2xi32>) -> !tfr.tensor { + %zp = arith.constant 67 : i64 + %cst = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %scale_factor = "tfr.cast"(%cst) : (tensor) -> !tfr.tensor + %input = "tfr.cast"(%arg0) : (tensor<2xi32>) -> !tfr.tensor + %rescaled = "tfr.quant_rescale"(%input, %scale_factor, %zp) : (!tfr.tensor, !tfr.tensor, i64) -> !tfr.tensor + func.return %rescaled : !tfr.tensor + +// CHECK-DAG: %[[f32:.*]] = tfr.constant f32 -> !tfr.attr +// CHECK-DAG: %[[i32:.*]] = tfr.constant i32 -> !tfr.attr +// CHECK-DAG: %[[scale_cst:.*]] = "tf.Const"() <{value = dense<1.000000e+00> : tensor}> : () -> tensor +// CHECK-DAG: %false = arith.constant false +// CHECK-DAG: %[[zp_cst:.*]] = "tf.Const"() <{value = dense<67> : tensor}> : () -> tensor +// CHECK: %[[zp:.*]] = "tfr.cast"(%[[zp_cst]]) : (tensor) -> !tfr.tensor +// CHECK: %[[scale:.*]] = "tfr.cast"(%[[scale_cst]]) : (tensor) -> !tfr.tensor +// CHECK: %[[input:.*]] = "tfr.cast"(%arg0) : (tensor<2xi32>) -> !tfr.tensor +// CHECK: %[[cast:.*]] = tfr.call @tf__cast(%[[input]], %[[f32]], %false) : (!tfr.tensor, !tfr.attr, i1) -> !tfr.tensor +// CHECK: %[[rescaled:.*]] = tfr.call @tf__mul(%[[cast]], %[[scale]]) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor +// CHECK: %[[rounded:.*]] = tfr.call @tf__round(%[[rescaled]]) : (!tfr.tensor) -> !tfr.tensor +// CHECK: %[[zp_cast:.*]] = tfr.call @tf__cast(%[[zp]], %[[f32]], %false) : (!tfr.tensor, !tfr.attr, i1) -> !tfr.tensor +// CHECK: %[[recentered:.*]] = tfr.call @tf__add(%[[rounded]], %[[zp_cast]]) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor +// CHECK: %[[cast_i32:.*]] = tfr.call @tf__cast(%[[recentered]], %[[i32]], %false) : (!tfr.tensor, !tfr.attr, i1) -> !tfr.tensor +// CHECK: return %[[cast_i32]] : !tfr.tensor +} + +// CHECK-LABEL: decompose_output_type +func.func @decompose_output_type(%arg0: tensor<2xf32>) -> tensor<2xi32> { + %0 = "tf.CastFloat"(%arg0) : (tensor<2xf32>) -> tensor<2xi32> + func.return %0: tensor<2xi32> +// CHECK: %[[i32:.*]] = tfr.constant i32 -> !tfr.attr +// CHECK: tfr.call @tf__cast(%[[casted_arg:.*]], %[[i32]], %false) : (!tfr.tensor, !tfr.attr, i1) -> !tfr.tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/end2end.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/end2end.mlir new file mode 100644 index 0000000000000000000000000000000000000000..eb7e03aec77dde8fe74de731ab862a521eb154de --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/end2end.mlir @@ -0,0 +1,235 @@ +// RUN: tfr-opt %s -tfr-decompose -tfr-raise-to-tf -canonicalize -verify-diagnostics -split-input-file | FileCheck %s + +//=================> User models, from GraphDef <==================== + +// CHECK-LABEL: my_identity +func.func @my_identity(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> { + %0 = "tf.MyIdentity"(%arg0) : (tensor<2x3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> + +// CHECK-NEXT: return %arg0 : tensor<2x3xf32> +} + +// CHECK-LABEL: my_rsqrt +func.func @my_rsqrt(%arg0: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyRsqrt"(%arg0) : (tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[RE:.*]] = "tf.RiscReciprocal"(%arg0) : (tensor<2x3xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[SQRT:.*]] = "tf.RiscSqrt"(%[[RE]]) : (tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[SQRT]]) <{shape = #tf_type.shape<3x2x3>}> : (tensor<*xf32>) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<3x2x3xf32> +} + +// CHECK-LABEL: my_leaky_relu +func.func @my_leaky_relu(%arg0: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyLeakyRelu"(%arg0) {alpha=3.0 : f32} : (tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[ALPHA:.*]] = "tf.Const"() <{value = dense<3.000000e+00> : tensor}> : () -> tensor +// CHECK-NEXT: %[[SHAPE:.*]] = "tf.RiscShape"(%arg0) {T = i32} : (tensor<2x3xf32>) -> tensor<*xi32> +// CHECK-NEXT: %[[ALPHA1:.*]] = "tf.RiscBroadcast"(%[[ALPHA]], %[[SHAPE]]) : (tensor, tensor<*xi32>) -> tensor<*xf32> +// CHECK-NEXT: %[[MAX:.*]] = "tf.RiscMaximum"(%arg0, %[[ALPHA1]]) : (tensor<2x3xf32>, tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[MAX]]) <{shape = #tf_type.shape<3x2x3>}> : (tensor<*xf32>) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<3x2x3xf32> +} + +// CHECK-LABEL: my_leaky_relu_with_default +func.func @my_leaky_relu_with_default(%arg0: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyLeakyRelu"(%arg0) : (tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[ALPHA:.*]] = "tf.Const"() <{value = dense<2.000000e-01> : tensor}> : () -> tensor +// CHECK-NEXT: %[[SHAPE:.*]] = "tf.RiscShape"(%arg0) {T = i32} : (tensor<2x3xf32>) -> tensor<*xi32> +// CHECK-NEXT: %[[ALPHA1:.*]] = "tf.RiscBroadcast"(%[[ALPHA]], %[[SHAPE]]) : (tensor, tensor<*xi32>) -> tensor<*xf32> +// CHECK-NEXT: %[[MAX:.*]] = "tf.RiscMaximum"(%arg0, %[[ALPHA1]]) : (tensor<2x3xf32>, tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[MAX]]) <{shape = #tf_type.shape<3x2x3>}> : (tensor<*xf32>) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<3x2x3xf32> +} + +// CHECK-LABEL: my_cast +func.func @my_cast(%arg0: tensor<2x3xf32>) -> tensor<2x3xi32> { + %0 = "tf.MyCast"(%arg0) {Tout=i32} : (tensor<2x3xf32>) -> tensor<2x3xi32> + func.return %0 : tensor<2x3xi32> + +// CHECK-NEXT: %[[CAST:.*]] = "tf.RiscCast"(%arg0) {Tout = i32} : (tensor<2x3xf32>) -> tensor<*xi32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[CAST]]) <{shape = #tf_type.shape<2x3>}> : (tensor<*xi32>) -> tensor<2x3xi32> +// CHECK-NEXT: return %[[ES]] : tensor<2x3xi32> +} + +// CHECK-LABEL: my_pack_single_input +func.func @my_pack_single_input(%arg0: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyPack"(%arg0) {N=1:i32, axis=0:i32} : (tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor +// CHECK-NEXT: %[[ED:.*]] = "tf.ExpandDims"(%arg0, %[[AXIS]]) : (tensor<2x3xf32>, tensor) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[ED]]) <{shape = #tf_type.shape<3x2x3>}> : (tensor<*xf32>) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<3x2x3xf32> +} + +// CHECK-LABEL: my_pack_multiple_inputs +func.func @my_pack_multiple_inputs(%arg0: tensor<2x3xf32>, %arg1: tensor<2x3xf32>, %arg2: tensor<2x3xf32>) -> tensor<3x2x3xf32> { + %0 = "tf.MyPack"(%arg0, %arg1, %arg2) {N=3:i32, axis=0:i32} : (tensor<2x3xf32>, tensor<2x3xf32>, tensor<2x3xf32>) -> tensor<3x2x3xf32> + func.return %0 : tensor<3x2x3xf32> + +// CHECK-NEXT: %[[AXIS:.*]] = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor +// CHECK-NEXT: %[[ED0:.*]] = "tf.ExpandDims"(%arg0, %[[AXIS]]) : (tensor<2x3xf32>, tensor) -> tensor<*xf32> +// CHECK-NEXT: %[[ED1:.*]] = "tf.ExpandDims"(%arg1, %[[AXIS]]) : (tensor<2x3xf32>, tensor) -> tensor<*xf32> +// CHECK-NEXT: %[[CC0:.*]] = "tf.RiscConcat"(%[[ED0]], %[[ED1]]) {axis = 0 : i32} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ED2:.*]] = "tf.ExpandDims"(%arg2, %[[AXIS]]) : (tensor<2x3xf32>, tensor) -> tensor<*xf32> +// CHECK-NEXT: %[[CC1:.*]] = "tf.RiscConcat"(%[[CC0]], %[[ED2]]) {axis = 0 : i32} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[CC1]]) <{shape = #tf_type.shape<3x2x3>}> : (tensor<*xf32>) -> tensor<3x2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<3x2x3xf32> +} + +// CHECK-LABEL: my_add_n_single_input +func.func @my_add_n_single_input(%arg0: tensor<2x3xf32>) -> tensor<2x3xf32> { + %0 = "tf.MyAddN"(%arg0) {N=1:i32} : (tensor<2x3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> + +// CHECK-NEXT: return %arg0 : tensor<2x3xf32> +} + +// CHECK-LABEL: my_add_n_multiple_inputs +func.func @my_add_n_multiple_inputs(%arg0: tensor<2x3xf32>, %arg1: tensor<2x3xf32>, %arg2: tensor<2x3xf32>) -> tensor<2x3xf32> { + %0 = "tf.MyAddN"(%arg0, %arg1, %arg2) {N=3:i32} : (tensor<2x3xf32>, tensor<2x3xf32>, tensor<2x3xf32>) -> tensor<2x3xf32> + func.return %0 : tensor<2x3xf32> + +// CHECK-NEXT: %[[ADD0:.*]] = "tf.RiscAdd"(%arg0, %arg1) : (tensor<2x3xf32>, tensor<2x3xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ADD1:.*]] = "tf.RiscAdd"(%[[ADD0]], %arg2) : (tensor<*xf32>, tensor<2x3xf32>) -> tensor<*xf32> +// CHECK-NEXT: %[[ES:.*]] = "tf.EnsureShape"(%[[ADD1]]) <{shape = #tf_type.shape<2x3>}> : (tensor<*xf32>) -> tensor<2x3xf32> +// CHECK-NEXT: return %[[ES]] : tensor<2x3xf32> +} + +// CHECK-LABEL: my_map_and_batch_dataset +func.func @my_map_and_batch_dataset(%input: tensor<*x!tf_type.variant>, + %other1: tensor<*xf32>, + %other2: tensor<*xi32>) -> tensor<*x!tf_type.variant> { + %0 = "tf.MyMapAndBatchDataset"(%input, %other1, %other2) + {batch_size=1000 : i64, num_parallel_calls = 8 : i64, drop_remainder = 0 : i1, + func = @"__some_func", output_types = [f32], output_shapes = [#tf_type.shape<>], preserve_cardinality = true} + : (tensor<*x!tf_type.variant>, tensor<*xf32>, tensor<*xi32>) -> tensor<*x!tf_type.variant> + func.return %0 : tensor<*x!tf_type.variant> + +// CHECK-DAG: %[[BATCH:.*]] = "tf.Const"() <{value = dense<1000> : tensor}> : () -> tensor +// CHECK-DAG: %[[PARAL:.*]] = "tf.Const"() <{value = dense<8> : tensor}> : () -> tensor +// CHECK-DAG: %[[KEEP:.*]] = "tf.Const"() <{value = dense : tensor}> : () -> tensor +// CHECK: %[[CAST:.*]] = "tf.Cast"(%arg2) <{Truncate = false}> : (tensor<*xi32>) -> tensor<*xf32> +// CHECK: %[[RET:.*]] = "tf.MapAndBatchDatasetV0"(%arg0, %[[BATCH]], %[[PARAL]], %[[KEEP]], %arg1, %[[CAST]]) +// CHECK-SAME: {f = @__some_func, output_shapes = [#tf_type.shape<>], output_types = [f32], preserve_cardinality = true} : (tensor<*x!tf_type.variant>, tensor, tensor, tensor, tensor<*xf32>, tensor<*xf32>) -> tensor<*x!tf_type.variant> +// CHECK: return %[[RET]] : tensor<*x!tf_type.variant> +} + +//=================> decomposition functions, translated from tf.compose api <==================== +tfr.func @tf__my_identity(%value: !tfr.tensor) -> !tfr.tensor { + tfr.return %value : !tfr.tensor +} + +tfr.func @tf__my_cast(%value: !tfr.tensor, %tout: !tfr.attr{tfr.name="Tout"}) -> !tfr.tensor { + %0 = tfr.call @tf__risc_cast(%value, %tout) : (!tfr.tensor, !tfr.attr) -> !tfr.tensor + tfr.return %0 : !tfr.tensor +} + +tfr.func @tf__my_rsqrt(%value: !tfr.tensor) -> !tfr.tensor { + %1 = tfr.call @tf__risc_reciprocal(%value) : (!tfr.tensor) -> !tfr.tensor + %2 = tfr.call @tf__risc_sqrt(%1) : (!tfr.tensor) -> !tfr.tensor + tfr.return %2 : !tfr.tensor +} + +tfr.func @tf__my_leaky_relu(%value: !tfr.tensor, %alpha: f32 {tfr.name="alpha", tfr.default=0.2:f32}) -> !tfr.tensor { + %1 = tfr.call @tf__risc_shape(%value) : (!tfr.tensor) -> !tfr.tensor + %2 = "tfr.constant_tensor"(%alpha) : (f32) -> tensor + %t = "tfr.cast"(%2) : (tensor) -> !tfr.tensor + %3 = tfr.call @tf__risc_broadcast(%t, %1) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + %4 = tfr.call @tf__risc_maximum(%value, %3) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + tfr.return %4 : !tfr.tensor +} + +// TODO(fengliuai): use shape dialect to manipulate the shape then this can be decomposed further. +tfr.func @tf__my_expand_dims(%value: !tfr.tensor, %axis: i32 {tfr.name="axis"}) -> !tfr.tensor { + %axis_cst = "tfr.constant_tensor"(%axis) : (i32) -> tensor + %dim = "tfr.cast"(%axis_cst) : (tensor) -> !tfr.tensor + %0 = tfr.call @tf__expand_dims(%value, %dim) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + tfr.return %0 : !tfr.tensor +} + +tfr.func @tf__my_pack(%values: !tfr.tensor_list, + %n: i32 {tfr.name="N"}, + %axis: i32 {tfr.name="axis"}) -> !tfr.tensor { + %index = arith.constant 0 : index + %cst = arith.constant 1 : i32 + %eq = arith.cmpi eq, %n, %cst : i32 + %v1 = tfr.get_element %values[%index] : (!tfr.tensor_list, index) -> !tfr.tensor + %temp = tfr.call @tf__my_expand_dims(%v1, %axis) : (!tfr.tensor, i32) -> !tfr.tensor + %res = scf.if %eq -> !tfr.tensor { + scf.yield %temp : !tfr.tensor + } else { + %step = arith.index_cast %cst : i32 to index + %end = arith.index_cast %n : i32 to index + %reduce = scf.for %i = %step to %end step %step iter_args(%reduce_iter=%temp) -> !tfr.tensor { + %v = tfr.get_element %values[%i] : (!tfr.tensor_list, index) -> !tfr.tensor + %temp1 = tfr.call @tf__my_expand_dims(%v, %axis) : (!tfr.tensor, i32) -> !tfr.tensor + %reduce_next = tfr.call @tf__risc_concat(%reduce_iter, %temp1, %axis) : (!tfr.tensor, !tfr.tensor, i32) -> !tfr.tensor + scf.yield %reduce_next : !tfr.tensor + } + scf.yield %reduce : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +tfr.func @tf__my_add_n(%values: !tfr.tensor_list, + %n: i32 {tfr.name="N"}) -> !tfr.tensor { + %index = arith.constant 0 : index + %cst = arith.constant 1 : i32 + %eq = arith.cmpi eq, %n, %cst : i32 + %v1 = tfr.get_element %values[%index] : (!tfr.tensor_list, index) -> !tfr.tensor + %res = scf.if %eq -> !tfr.tensor { + scf.yield %v1 : !tfr.tensor + } else { + %step = arith.index_cast %cst : i32 to index + %end = arith.index_cast %n : i32 to index + %reduce = scf.for %i = %step to %end step %step iter_args(%reduce_iter=%v1) -> !tfr.tensor { + %v = tfr.get_element %values[%i] : (!tfr.tensor_list, index) -> !tfr.tensor + %reduce_next = tfr.call @tf__risc_add(%reduce_iter, %v) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor + scf.yield %reduce_next : !tfr.tensor + } + scf.yield %reduce : !tfr.tensor + } + tfr.return %res : !tfr.tensor +} + +tfr.func @tf__my_map_and_batch_dataset( + %input_dataset: !tfr.tensor, + %other_arguments: !tfr.tensor_list, + %batch_size: i64 {tfr.name="batch_size"}, + %num_parallel_calls: i64 {tfr.name="num_parallel_calls"}, + %drop_remainder: i1 {tfr.name="drop_remainder"}, + %f: !tfr.attr {tfr.name="func"}, + %output_types: !tfr.attr {tfr.name="output_types"}, + %output_shapes: !tfr.attr {tfr.name="output_shapes"}, + %preserve_cardinality: i1 {tfr.name="preserve_cardinality", tfr.default=false}) -> !tfr.tensor { + %batch = "tfr.constant_tensor"(%batch_size) : (i64) -> tensor + %batch1 = "tfr.cast"(%batch) : (tensor) -> !tfr.tensor + %calls = "tfr.constant_tensor"(%num_parallel_calls) : (i64) -> tensor + %calls1 = "tfr.cast"(%calls) : (tensor) -> !tfr.tensor + %drop = "tfr.constant_tensor"(%drop_remainder) : (i1) -> tensor + %drop1 = "tfr.cast"(%drop) : (tensor) -> !tfr.tensor + %ret = tfr.call @tf__map_and_batch_dataset_v0(%input_dataset, %batch1, %calls1, %drop1, %other_arguments, %f, %output_types, %output_shapes, %preserve_cardinality) + : (!tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor_list, !tfr.attr, !tfr.attr, !tfr.attr, i1) -> !tfr.tensor + tfr.return %ret : !tfr.tensor +} + +//=================> signatures of the primitive ops with kernels, modeled as external TFR function <== +tfr.func @tf__risc_cast_(!tfr.tensor, !tfr.attr{tfr.name="Tout"}) -> !tfr.tensor attributes{Tout} +tfr.func @tf__risc_add_(!tfr.tensor, !tfr.tensor) -> !tfr.tensor attributes{T} +tfr.func @tf__risc_concat_(!tfr.tensor, !tfr.tensor, i32{tfr.name="axis"}) -> !tfr.tensor attributes{T} +tfr.func @tf__risc_broadcast_(!tfr.tensor, !tfr.tensor) -> !tfr.tensor attributes{T, Tidx} +tfr.func @tf__risc_reciprocal_(!tfr.tensor) -> !tfr.tensor attributes{T} +tfr.func @tf__risc_sqrt_(!tfr.tensor) -> !tfr.tensor attributes{T} +tfr.func @tf__risc_shape_(!tfr.tensor, !tfr.attr{tfr.name="T", tfr.default=i32}) -> !tfr.tensor attributes{T} +tfr.func @tf__risc_maximum_(!tfr.tensor, !tfr.tensor) -> !tfr.tensor attributes{T} +tfr.func @tf__expand_dims_(!tfr.tensor, !tfr.tensor) -> !tfr.tensor attributes{T, Tdim} +tfr.func @tf__map_and_batch_dataset_v0_(!tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor, !tfr.tensor_list, + !tfr.attr{tfr.name="f"}, !tfr.attr{tfr.name="output_types"}, !tfr.attr{tfr.name="output_shapes"}, i1{tfr.name="preserve_cardinality"}) + -> !tfr.tensor attributes{T, Targuments} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..06acebf6b0cc2f07e36da6fde92910dec6f3719b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/ops.mlir @@ -0,0 +1,428 @@ +// RUN: tfr-opt %s -verify-diagnostics -split-input-file | tfr-opt | FileCheck %s + +// Tests for types, ops with custom constraints, verifiers, printer or parser +// methods. + +// CHECK-LABEL: tensor_type_noconstraint +func.func private @tensor_type_noconstraint() -> !tfr.tensor + +// ----- + +// CHECK-LABEL: tensor_type +func.func private @tensor_type() -> !tfr.tensor + +// ----- + +// CHECK-LABEL: tensor_list_type_noconstraint +func.func private @tensor_list_type_noconstraint() -> !tfr.tensor_list + +// ----- + +// CHECK-LABEL: tensor_list_type_array_like +func.func private @tensor_list_type_array_like() -> !tfr.tensor_list<[N, T]> + +// ----- + +// CHECK-LABEL: tensor_list_type_tuple_like +func.func private @tensor_list_type_tuple_like() -> !tfr.tensor_list + +// ----- + +// expected-error@+1 {{unbalanced '[' character in pretty dialect name}} +func.func private @tensor_invalid_1() -> !tfr.tensor<[N, T> + +// ----- + +// expected-error@+1 {{unbalanced}} +func.func @tensor_invalid_2() -> !tfr.tensor<[N, T] + +// ----- + +// CHECK-LABEL: call_op +func.func @call_op(%arg0: !tfr.tensor, %arg1: !tfr.tensor_list, %arg2: i32) -> !tfr.tensor { + %0 = tfr.call @Foo(%arg0, %arg1, %arg2) : (!tfr.tensor, !tfr.tensor_list, i32) -> !tfr.tensor + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: call_op_arg_attr(%arg0: i32) -> !tfr.tensor +func.func @call_op_arg_attr(%arg0: i32) -> !tfr.tensor { + %0 = tfr.call @Bar(%arg0) : (i32) -> !tfr.tensor + func.return %0 : !tfr.tensor +} + +// ----- + +func.func @call_op_invalid_1(%arg0: tensor) -> !tfr.tensor { + // expected-error@+1 {{got 'tensor'}} + %0 = tfr.call @Huu(%arg0) : (tensor) -> !tfr.tensor + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: get_shape +func.func @get_shape(%arg0: !tfr.tensor) -> (!shape.shape, !shape.shape) { + %0 = tfr.get_shape %arg0 -> !shape.shape + %1 = "tfr.get_shape"(%arg0) : (!tfr.tensor) -> !shape.shape + func.return %0, %1 : !shape.shape, !shape.shape +} + +// ----- + +// CHECK-LABEL: get_real_shape +func.func @get_real_shape(%arg0: tensor<1x2xf32>) -> tensor<2xindex> { + %0 = "tfr.cast"(%arg0) : (tensor<1x2xf32>) -> !tfr.tensor + %1 = tfr.get_shape %0 -> !shape.shape + %2 = shape.to_extent_tensor %1 : !shape.shape -> tensor<2xindex> + func.return %2 : tensor<2xindex> +} + +// ----- + +func.func @get_element_type(%arg0: !tfr.tensor) -> (!tfr.attr, !tfr.attr) { + %0 = tfr.get_element_type %arg0 -> !tfr.attr + %1 = "tfr.get_element_type"(%arg0) : (!tfr.tensor) -> !tfr.attr + func.return %0, %1 : !tfr.attr, !tfr.attr +} + +// ----- + +// CHECK-LABEL: from_tf_tensor +func.func @from_tf_tensor(%arg0: tensor) -> !tfr.tensor { + %0 = "tfr.cast"(%arg0) : (tensor) -> !tfr.tensor + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: to_tf_tensor +func.func @to_tf_tensor(%arg0: !tfr.tensor) -> tensor { + %0 = "tfr.cast"(%arg0) : (!tfr.tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: constant +func.func @constant() -> (!tfr.attr, !tfr.attr, !tfr.attr, !tfr.attr) { + %0 = tfr.constant f32 -> !tfr.attr + %1 = tfr.constant [f32, i32] -> !tfr.attr + %2 = "tfr.constant"() {value = f32} : () -> !tfr.attr + %3 = "tfr.constant"() {value = [f32, i32]} : () -> !tfr.attr + func.return %0, %1, %2, %3 : !tfr.attr, !tfr.attr, !tfr.attr, !tfr.attr +} + +// ----- + +// CHECK-LABEL: equal +func.func @equal() -> (i1, i1, i1, i1) { + %0 = tfr.constant f32 -> !tfr.attr + %1 = tfr.constant f32 -> !tfr.attr + %2 = tfr.constant i32 -> !tfr.attr + %same_type = tfr.equal %0,%1 -> i1 + %diff_type = tfr.equal %0,%2 -> i1 + + %3 = tfr.constant "hello" -> !tfr.attr + %4 = tfr.constant "hello" -> !tfr.attr + %5 = tfr.constant "how are you" -> !tfr.attr + %same_str = tfr.equal %3,%4 -> i1 + %diff_str = tfr.equal %3,%5 -> i1 + func.return %same_type, %diff_type, %same_str, %diff_str : i1, i1, i1, i1 +} + +// ----- + +// CHECK-LABEL: constant_tensor_scalar +func.func @constant_tensor_scalar(%arg0: i32) -> tensor { + %0 = "tfr.constant_tensor"(%arg0) : (i32) -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: constant_tensor_vector +func.func @constant_tensor_vector(%arg0: vector<1x2xi32>) -> tensor<1x2xi32> { + %0 = "tfr.constant_tensor"(%arg0) : (vector<1x2xi32>) -> tensor<1x2xi32> + func.return %0 : tensor<1x2xi32> +} + +// ----- + +// CHECK-LABEL: constant_tensor_array +func.func @constant_tensor_array() -> !tfr.tensor { + %0 = tfr.constant [1, -1, 3] -> !tfr.attr + %1 = "tfr.constant_tensor"(%0) : (!tfr.attr) -> !tfr.tensor + func.return %1 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: constant_tensor_scalar +func.func @constant_tensor_scalar() -> !tfr.tensor { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "tfr.constant_tensor"(%0) : (i32) -> !tfr.tensor + func.return %1 : !tfr.tensor +} + +// ----- + +func.func @constant_tensor_invalid_0(%arg0: i32) -> tensor { + // expected-error@+1 {{input and output should have the same scalar types.}} + %0 = "tfr.constant_tensor"(%arg0) : (i32) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @constant_tensor_invalid_1(%arg0: vector<1xi32>) -> tensor { + // expected-error@+1 {{output type should be static and ranked}} + %0 = "tfr.constant_tensor"(%arg0) : (vector<1xi32>) -> tensor + func.return %0 : tensor +} + +// ----- + +func.func @constant_tensor_invalid_2(%arg0: vector<1xi32>) -> tensor<1xf32> { + // expected-error@+1 {{input and output should have same shape and element type}} + %0 = "tfr.constant_tensor"(%arg0) : (vector<1xi32>) -> tensor<1xf32> + func.return %0 : tensor<1xf32> +} + +// ----- + +func.func @constant_tensor_invalid_3(%arg0: vector<1xi32>) -> tensor<1x1xi32> { + // expected-error@+1 {{input and output should have same shape and element type}} + %0 = "tfr.constant_tensor"(%arg0) : (vector<1xi32>) -> tensor<1x1xi32> + func.return %0 : tensor<1x1xi32> +} + +// ----- + +func.func @constant_tensor_invalid_4(%arg0: i32) -> tensor<1x1xi32> { + // expected-error@+1 {{input can not be converted to an output tensor}} + %0 = "tfr.constant_tensor"(%arg0) : (i32) -> tensor<1x1xi32> + func.return %0 : tensor<1x1xi32> +} + +// ----- + +// CHECK-LABEL: get_element +func.func @get_element(%arg0: !tfr.tensor_list) -> !tfr.tensor { + %cst = "arith.constant"() {value = 1 : index} : () -> index + %0 = tfr.get_element %arg0[%cst] : (!tfr.tensor_list, index) -> !tfr.tensor + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: build_list +func.func @build_list(%arg0: !tfr.tensor, %arg1: !tfr.tensor) -> !tfr.tensor_list { + %0 = "tfr.build_list"(%arg0, %arg1) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + func.return %0 : !tfr.tensor_list +} + +// ----- + +// CHECK-LABEL: quant_act_range +func.func @quant_act_range(%arg0: !tfr.attr, %arg1: f32, %arg2: i64) -> !tfr.tensor { + %0:2 = "tfr.quant_act_range"(%arg0, %arg1, %arg2) : (!tfr.attr,f32,i64) -> (!tfr.tensor,!tfr.tensor) + func.return %0#0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: quant_rescale +func.func @quant_rescale(%arg0: !tfr.tensor, %arg1: !tfr.tensor, %arg2: i64) -> !tfr.tensor { + %0 = "tfr.quant_rescale"(%arg0, %arg1, %arg2) : (!tfr.tensor, !tfr.tensor, i64) -> (!tfr.tensor) + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: quant_raw_data +func.func @quant_raw_data(%arg0: !tfr.tensor) -> !tfr.tensor { + %0 = "tfr.quant_raw_data"(%arg0) : (!tfr.tensor) -> (!tfr.tensor) + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: quant_qparam +func.func @quant_qparam(%arg0: !tfr.tensor) -> (!tfr.tensor, !tfr.tensor) { + %scale, %zp = tfr.quant_qparam(%arg0) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + func.return %scale, %zp : !tfr.tensor, !tfr.tensor +} + +// ----- + +// CHECK-LABEL: quant_scale_factor +func.func @quant_scale_factor(%arg0: f32, %arg1: !tfr.tensor_list) -> (!tfr.tensor) { + %0 = "tfr.quant_scale_factor"(%arg0, %arg1) : (f32, !tfr.tensor_list) -> (!tfr.tensor) + func.return %0 : !tfr.tensor +} + +// ----- + +// CHECK-LABEL: build_const_list +func.func @build_const_list() -> !tfr.attr { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "arith.constant"() {value = 41 : i32} : () -> i32 + %2 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + func.return %2 : !tfr.attr +} + +// ----- + +// CHECK-LABEL: build_high_dim_const_list +func.func @build_high_dim_const_list() -> !tfr.attr { + %0 = "arith.constant"() {value = 42 : i32} : () -> i32 + %1 = "arith.constant"() {value = 41 : i32} : () -> i32 + %2 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + %3 = "tfr.build_list"(%0, %1) : (i32, i32) -> !tfr.attr + %4 = "tfr.build_list"(%2, %3) : (!tfr.attr, !tfr.attr) -> !tfr.attr + func.return %4 : !tfr.attr +} + +// ----- + +// CHECK-LABEL: get_length +func.func @get_length(%arg0: !tfr.tensor, %arg1: !tfr.tensor) -> index { + %0 = "tfr.build_list"(%arg0, %arg1) : (!tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %1 = "tfr.get_length"(%0) : (!tfr.tensor_list) -> index + func.return %1 : index +} + +// ----- + +// CHECK-LABEL: tfr.func +tfr.func @External(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: !tfr.attr {tfr.name = "T"}) + -> (!tfr.tensor, !tfr.tensor_list) + attributes {A, C} + +// ----- + +// CHECK-LABEL: tfr.func +tfr.func @Foo(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) + -> (!tfr.tensor, !tfr.tensor_list) + attributes {A, C} { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// CHECK-LABEL: tfr.func +tfr.func @Bar(%arg0: !tfr.tensor, + %arg2: i32 {tfr.name = "B"}, + %arg3: vector<1xi32> {tfr.name = "C"}) + -> (!tfr.tensor, !tfr.tensor) + attributes {A} { + tfr.return %arg0, %arg0 : !tfr.tensor, !tfr.tensor +} + +// ----- + +// expected-error@+1 {{Undefined attributes are used: A}} +tfr.func @Foo_undefined_attr(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor, !tfr.tensor_list) { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{3 attribute argument doesn't have a tfr.name attribute}} +tfr.func @Foo_unnamed_attr(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32>) -> + (!tfr.tensor, !tfr.tensor_list) { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{tfr.tensor/tfr.tensor_list argument should be before non tensor arguments}} +tfr.func @Foo_invalid_arg_order(%arg0: !tfr.tensor, + %arg2: i32 {tfr.name = "A"}, + %arg1: !tfr.tensor_list, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor, !tfr.tensor_list) { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +tfr.func @Foo_valid_arg_order0( + %arg1: !tfr.tensor_list, + %arg0: !tfr.tensor, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor, !tfr.tensor_list) attributes {T}{ + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{tfr.tensor argument should be before tfr.tensor_list argument.}} +tfr.func @Foo_invalid_arg_order0( + %arg1: !tfr.tensor_list, + %arg0: !tfr.tensor, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor, !tfr.tensor_list) { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{tfr.tensor result should be before tfr.tensor_list result}} +tfr.func @Foo_invalid_result_order(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor_list, !tfr.tensor) { + tfr.return %arg1, %arg0 : !tfr.tensor_list, !tfr.tensor +} + +// ----- + +// expected-error@+1 {{More than one tfr.tensor_list argument isn't allowed}} +tfr.func @Foo_multiple_tensor_list_args(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: !tfr.tensor_list, + %arg3: i32 {tfr.name = "A"}, + %arg4: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor, !tfr.tensor_list) { + tfr.return %arg0, %arg1 : !tfr.tensor, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{More than one tfr.tensor_list result isn't allowed}} +tfr.func @Foo_multiple_tensor_list_results(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> + (!tfr.tensor_list, !tfr.tensor_list) { + tfr.return %arg1, %arg1 : !tfr.tensor_list, !tfr.tensor_list +} + +// ----- + +// expected-error@+1 {{None tfr.tensor/tfr.tensor_list results aren't allowed as a result}} +tfr.func @Foo_return_attr(%arg0: !tfr.tensor, + %arg1: !tfr.tensor_list, + %arg2: i32 {tfr.name = "A"}, + %arg3: vector<1xi32> {tfr.name = "C"}) -> i32 { + tfr.return %arg2 : i32 +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/raise_to_tf.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/raise_to_tf.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8fa852c962f35ec57c60c6925c20da6c0b947e91 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/raise_to_tf.mlir @@ -0,0 +1,126 @@ +// RUN: tfr-opt %s -tfr-raise-to-tf -verify-diagnostics -split-input-file | FileCheck %s + +tfr.func @tf__risc_same_(!tfr.tensor) -> !tfr.tensor attributes {T} +tfr.func @tf__risc_concat_(!tfr.tensor_list) -> !tfr.tensor attributes {T, N} +tfr.func @tf__risc_split_(!tfr.tensor, i32 {tfr.name="N"}) -> !tfr.tensor_list attributes {T, N} +tfr.func @tf__risc_cast_(!tfr.tensor, !tfr.attr {tfr.name="K"}) -> !tfr.tensor attributes {T, K} +tfr.func @tf__const_(!tfr.attr {tfr.name="value", tfr.type="tensor"}, + !tfr.attr {tfr.name="K",tfr.type="dtype"}) -> !tfr.tensor attributes {T, K} +tfr.func @tf__positive_(!tfr.tensor) -> !tfr.tensor attributes {T, i1_} +tfr.func @tf__invalid_type_op_(!tfr.tensor) -> !tfr.tensor attributes {T, i8_} + +// CHECK-LABEL: decompose_tf_same +func.func @decompose_tf_same(%arg0: tensor<1x2x3x4x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> { + %0 = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor + %1 = tfr.call @tf__risc_same(%0) : (!tfr.tensor) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<1x2x3x4x!tf_type.string> + func.return %2 : tensor<1x2x3x4x!tf_type.string> + +// CHECK: %[[id:.*]] = "tf.RiscSame"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> tensor<*x!tf_type.string> +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[id]]) <{shape = #tf_type.shape<1x2x3x4>}> : (tensor<*x!tf_type.string>) -> tensor<1x2x3x4x!tf_type.string> +// CHECK: return %[[es]] : tensor<1x2x3x4x!tf_type.string> +} + +// CHECK-LABEL: decompose_tf_consecutive +func.func @decompose_tf_consecutive(%arg0: tensor<1x2x3x4x!tf_type.string>, %arg1: tensor, %arg2: tensor) -> tensor { + %0 = "tfr.cast"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> !tfr.tensor + %1 = "tfr.cast"(%arg2) : (tensor) -> !tfr.tensor + %2 = tfr.call @tf__risc_same(%0) : (!tfr.tensor) -> !tfr.tensor + %3 = tfr.call @tf__risc_same(%1) : (!tfr.tensor) -> !tfr.tensor + %4 = "tfr.cast"(%3) : (!tfr.tensor) -> tensor + func.return %4 : tensor + +// CHECK: %[[id0:.*]] = "tf.RiscSame"(%arg0) : (tensor<1x2x3x4x!tf_type.string>) -> tensor<*x!tf_type.string> +// CHECK: %[[id2:.*]] = "tf.RiscSame"(%arg2) : (tensor) -> tensor<*xf32> +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[id2]]) <{shape = #tf_type.shape<>}> : (tensor<*xf32>) -> tensor +// CHECK: return %[[es]] : tensor +} + +// CHECK-LABEL: decompose_tf_concat_n +func.func @decompose_tf_concat_n(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor<3xf32> { + %0 = "tfr.cast"(%arg0) : (tensor) -> !tfr.tensor + %1 = "tfr.cast"(%arg1) : (tensor) -> !tfr.tensor + %2 = "tfr.cast"(%arg2) : (tensor) -> !tfr.tensor + %3 = "tfr.build_list"(%0, %1, %2) : (!tfr.tensor, !tfr.tensor, !tfr.tensor) -> !tfr.tensor_list + %concat = tfr.call @tf__risc_concat(%3) : (!tfr.tensor_list) -> !tfr.tensor + %4 = "tfr.cast"(%concat) : (!tfr.tensor) -> tensor<3xf32> + func.return %4 : tensor<3xf32> + +// CHECK: %[[concat:.*]] = "tf.RiscConcat"(%arg0, %arg1, %arg2) : (tensor, tensor, tensor) -> tensor<*xf32> +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[concat]]) <{shape = #tf_type.shape<3>}> : (tensor<*xf32>) -> tensor<3xf32> +// CHECK: return %[[es]] : tensor<3xf32> +} + +// CHECK-LABEL: decompose_tf_split +func.func @decompose_tf_split(%arg0: tensor<3xf32>) -> (tensor) { + %0 = "tfr.cast"(%arg0) : (tensor<3xf32>) -> !tfr.tensor + %n = arith.constant 3: i32 + %split = tfr.call @tf__risc_split(%0, %n) : (!tfr.tensor, i32) -> !tfr.tensor_list + %i0 = arith.constant 0: index + %s0 = tfr.get_element %split[%i0] : (!tfr.tensor_list, index) -> !tfr.tensor + %4 = "tfr.cast"(%s0) : (!tfr.tensor) -> tensor + func.return %4 : tensor + +// CHECK: %[[split:.*]]:3 = "tf.RiscSplit"(%arg0) {N = 3 : i32} : (tensor<3xf32>) -> (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[split]]#0) <{shape = #tf_type.shape<>}> : (tensor<*xf32>) -> tensor +// CHECK: return %[[es]] : tensor +} + +// CHECK-LABEL: decompose_tf_cast +func.func @decompose_tf_cast(%arg0: tensor) -> tensor { + %0 = "tfr.cast"(%arg0) : (tensor) -> !tfr.tensor + %t = tfr.constant i32 -> !tfr.attr + %concat = tfr.call @tf__risc_cast(%0, %t) : (!tfr.tensor, !tfr.attr) -> !tfr.tensor + %4 = "tfr.cast"(%concat) : (!tfr.tensor) -> tensor + func.return %4 : tensor + +// CHECK: %[[tfcast:.*]] = "tf.RiscCast"(%arg0) {K = i32} : (tensor) -> tensor<*xi32> +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[tfcast]]) <{shape = #tf_type.shape<>}> : (tensor<*xi32>) -> tensor +// CHECK: return %[[es]] : tensor +} + +// CHECK-LABEL: convert_to_scalar_tensor +func.func @convert_to_scalar_tensor() -> tensor { + %0 = arith.constant 3.0: f32 + %t = tfr.constant f32 -> !tfr.attr + %cst = tfr.call @tf__const(%0, %t) : (f32, !tfr.attr) -> !tfr.tensor + %4 = "tfr.cast"(%cst) : (!tfr.tensor) -> tensor + func.return %4 : tensor + +// CHECK: %[[cst:.*]] = "tf.Const"() <{value = dense<3.000000e+00> : tensor}> : () -> tensor +// CHECK: return %[[cst]] : tensor +} + +// CHECK-LABEL: attribute_propagate +func.func @attribute_propagate(%arg0: tensor) -> tensor { + %0 = "tfr.cast"(%arg0) : (tensor) -> !tfr.tensor + %t = tfr.constant i32 -> !tfr.attr + %concat = tfr.call @tf__risc_cast(%0, %t) {device = "hello", _tpu_replicate} : (!tfr.tensor, !tfr.attr) -> !tfr.tensor + %4 = "tfr.cast"(%concat) : (!tfr.tensor) -> tensor + func.return %4 : tensor + +// CHECK: %[[tfcast:.*]] = "tf.RiscCast"(%arg0) {K = i32, _tpu_replicate, device = "hello"} : (tensor) -> tensor<*xi32> +// CHECK: %[[es:.*]] = "tf.EnsureShape"(%[[tfcast]]) <{shape = #tf_type.shape<>}> : (tensor<*xi32>) -> tensor +// CHECK: return %[[es]] : tensor +} + +// CHECK-LABEL: fixed_element_attribute +func.func @fixed_element_attribute(%arg0: tensor<2xf32>) -> tensor<2xi1> { + %0 = "tfr.cast"(%arg0) : (tensor<2xf32>) -> !tfr.tensor + %1 = tfr.call @tf__positive(%0) : (!tfr.tensor) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<2xi1> + func.return %2 : tensor<2xi1> +// CHECK: %[[positive:.*]] = "tf.Positive"(%arg0) : (tensor<2xf32>) -> tensor<*xi1> +// CHECK: %[[res:.*]] = "tf.EnsureShape"(%[[positive]]) <{shape = #tf_type.shape<2>}> : (tensor<*xi1>) -> tensor<2xi1> +// CHECK: return %[[res]] : tensor<2xi1> +} + +// CHECK-LABEL: fixed_element_attribute_invalid +func.func @fixed_element_attribute_invalid(%arg0: tensor<2xf32>) -> tensor<2xi8> { + %0 = "tfr.cast"(%arg0) : (tensor<2xf32>) -> !tfr.tensor + // expected-error@+1 {{type i8_ can't be resolved for the signature of the op}} + %1 = tfr.call @tf__invalid_type_op(%0) : (!tfr.tensor) -> !tfr.tensor + %2 = "tfr.cast"(%1) : (!tfr.tensor) -> tensor<2xi8> + func.return %2 : tensor<2xi8> +// CHECK: tfr.call @tf__invalid_type_op +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/rewrite_quantized_io.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/rewrite_quantized_io.mlir new file mode 100644 index 0000000000000000000000000000000000000000..537f1f5d7b55e87a9c7831865af48d51d9cb1821 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tests/rewrite_quantized_io.mlir @@ -0,0 +1,44 @@ +// RUN: tfr-opt %s -tfr-decompose -tfr-rewrite-quantized-io -verify-diagnostics | FileCheck %s + +// CHECK-LABEL: @tf__my_requantize +tfr.func @tf__my_requantize(%input: !tfr.tensor) -> !tfr.tensor { + %raw_data = tfr.quant_raw_data(%input) : (!tfr.tensor) -> !tfr.tensor + %scale, %zp = tfr.quant_qparam(%input) : (!tfr.tensor) -> (!tfr.tensor, !tfr.tensor) + %result = tfr.call @tf__requantize(%raw_data, %scale, %zp) : (!tfr.tensor, !tfr.tensor, !tfr.tensor) -> !tfr.tensor + tfr.return %result : !tfr.tensor +} + +// CHECK-LABEL: @tf__intermediate +tfr.func @tf__intermediate(%arg0: !tfr.tensor) -> !tfr.tensor { + %0 = tfr.call @tf__risc(%arg0) : (!tfr.tensor) -> !tfr.tensor + tfr.return %0 : !tfr.tensor +} + +// CHECK-LABEL: remove_quantized_io +func.func @remove_quantized_io( + %arg0: tensor<1x10x!quant.uniform>, + %arg1: tensor<1x5xf32>) -> (tensor<1x10x!quant.uniform>, tensor<1x5xf32>) { + %0 = "tf.MyRequantize"(%arg0) : (tensor<1x10x!quant.uniform>) -> tensor<1x10x!quant.uniform> + %1 = "tf.Intermediate"(%arg1) : (tensor<1x5xf32>) -> tensor<1x5xf32> + func.return %0, %1 : tensor<1x10x!quant.uniform>, tensor<1x5xf32> + +// CHECK-DAG: %[[scale:.*]] = "tf.Const"() <{value = dense<1.000000e-01> : tensor}> : () -> tensor +// CHECK-DAG: %[[zp:.*]] = "tf.Const"() <{value = dense<-128> : tensor}> : () -> tensor +// CHECK: %[[quant:.*]] = "tfr.cast"(%arg0) : (tensor<1x10xi8>) -> !tfr.tensor +// CHECK: %[[scale_cast:.*]] = "tfr.cast"(%[[scale]]) +// CHECK: %[[zp_cast:.*]] = "tfr.cast"(%[[zp]]) +// CHECK: %[[requant:.*]] = tfr.call @tf__requantize(%[[quant]], %[[scale_cast]], %[[zp_cast]]) +// CHECK: %[[result:.*]] = "tfr.cast"(%[[requant]]) +// CHECK-NOT: quant.uniform +// CHECK: return %[[result]], %[[float_resunt:.*]] : tensor<1x10xi8>, tensor<1x5xf32> +} + + +// CHECK-LABEL: quant_input_multiple_users +// expected-error@+1 {{The argument with type tensor<1x10x!quant.uniform> should have one user}} +func.func @quant_input_multiple_users(%arg0: tensor<1x10x!quant.uniform>) -> (!tfr.tensor, !tfr.tensor) { + %0 = "tfr.cast"(%arg0) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor + %1 = "tfr.cast"(%arg0) : (tensor<1x10x!quant.uniform>) -> !tfr.tensor + func.return %0, %1 : !tfr.tensor, !tfr.tensor +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tfr_wrapper.pyi b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tfr_wrapper.pyi new file mode 100644 index 0000000000000000000000000000000000000000..7ee33642e686dbe67eff71ed7a8ee7a51f491c1b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/tfr_wrapper.pyi @@ -0,0 +1,16 @@ +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +def verify(arg0: str) -> bool: ... diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..5aa89b64c64f6a2dad351944cc32e4af310d2ef5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.cc @@ -0,0 +1,198 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfr/utils/utils.h" + +#include + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" + +namespace mlir { +namespace TFR { +namespace { + +// TODO(b/174692018): Use the official allowlist of the unregistered attrs. +const llvm::StringSet<>& GetAllowedAttributes() { + static auto* const ops = new llvm::StringSet<>({"device", "_tpu_replicate"}); + return *ops; +} + +// Some TFL optional attributes may not appear in their corresponding TF op +// attributes. +const llvm::StringSet<>& GetOptionalAttributes() { + static auto* const ops = + new llvm::StringSet<>({"asymmetric_quantize_inputs"}); + return *ops; +} + +void CollectAllowedAttrs(CallOp src, NamedAttrList* attrs) { + for (auto& attr : src->getAttrs()) { + if (GetAllowedAttributes().contains(attr.getName().strref())) { + attrs->append(attr); + } + } +} + +// Adds `attrs` to all the operations between `begin` and `end` in the same +// block. Does not include `end`. +void AddAttributesInSameBlock(Block::iterator begin, Block::iterator end, + const NamedAttrList& attrs) { + for (Block::iterator it = begin; it != end; ++it) { + for (auto& attr : attrs) { + it->setAttr(attr.getName(), attr.getValue()); + } + } +} + +// Adds `attrs` to all the operations between `begin` and `end`. Does not +// include `end`. The operations might be across multiple blocks. +void AddAttributes(Block::iterator begin, Block::iterator end, + const NamedAttrList& attrs) { + if (begin->getBlock() == end->getBlock()) { + AddAttributesInSameBlock(begin, end, attrs); + } else { + Region::iterator begin_block = Region::iterator(begin->getBlock()); + Region::iterator end_block = Region::iterator(end->getBlock()); + AddAttributesInSameBlock(begin, begin_block->end(), attrs); + for (Region::iterator it = ++begin_block; it != end_block; ++it) { + AddAttributesInSameBlock(it->begin(), it->end(), attrs); + } + } +} + +} // namespace + +std::string GetComposeFuncName(StringRef tf_op_name) { + std::string compose_func_name; + for (int i = 0; i < tf_op_name.size(); ++i) { + if (tf_op_name[i] == '_') { + // The field name must not contain "_"s. "_Arg" and "_RetVal" are special + // op names and we can return empty string to skip the decomposition. + return {}; + } + if (tf_op_name[i] == '.') { + compose_func_name.push_back('_'); + } else if (llvm::isUpper(tf_op_name[i])) { + compose_func_name.push_back('_'); + compose_func_name.push_back(llvm::toLower(tf_op_name[i])); + } else { + compose_func_name.push_back(tf_op_name[i]); + } + } + return compose_func_name; +} + +std::string GetTFOpName(StringRef compose_func_name) { + std::string tf_op_name; + bool after_underscore = false; + for (int i = 0; i < compose_func_name.size(); ++i) { + if (llvm::isUpper(compose_func_name[i])) { + // The field name must not contain uppercase letters. + return {}; + } + if (after_underscore) { + if (llvm::isLower(compose_func_name[i])) { + tf_op_name.push_back(llvm::toUpper(compose_func_name[i])); + after_underscore = false; + } else { + // The character after a "_" must be a lowercase letter. + return {}; + } + } else if (compose_func_name[i] == '_') { // first time visit '_' + if (i + 1 < compose_func_name.size() && compose_func_name[i + 1] == '_') { + tf_op_name.push_back('.'); + i++; + } + after_underscore = true; + } else { + tf_op_name.push_back(compose_func_name[i]); + } + } + if (after_underscore) { + // Trailing "_". + return {}; + } + return tf_op_name; +} + +LogicalResult ValidateAttrs(Operation* src, const StringSet<>& registered) { + for (auto& attr : src->getAttrs()) { + StringRef attr_name = attr.getName().strref(); + + if (!registered.contains(attr_name) && + !(GetAllowedAttributes().contains(attr_name) || + GetOptionalAttributes().contains(attr_name))) { + src->emitError("Denied unregistered attribute was found: " + attr_name); + return failure(); + } + } + return success(); +} + +LogicalResult CopyAllowedUnregisteredAttrs(Operation* src, CallOp dst, + const StringSet<>& registered) { + for (auto& attr : src->getAttrs()) { + StringRef attr_name = attr.getName().strref(); + // Skip the registered or optional attribute. + if (registered.contains(attr_name) || + GetOptionalAttributes().contains(attr_name)) + continue; + + // Unregistered attribute. + if (GetAllowedAttributes().contains(attr_name)) { + dst->setAttr(attr.getName(), attr.getValue()); + } else { + src->emitError("Denied unregistered attribute was found: " + attr_name); + return failure(); + } + } + return success(); +} + +LogicalResult CopyNonSymbolRefAttrs(CallOp src, Operation* dst) { + NamedAttrList attrs; + CollectAllowedAttrs(src, &attrs); + + for (auto& attr : attrs) { + dst->setAttr(attr.getName(), attr.getValue()); + } + + return success(); +} + +void PropagateAttrsToOperations(CallOp src, Block::iterator begin, + Block::iterator end) { + // Find all the attributes in the call op. These attributes are not in the + // op definition, so needs to be propagated to all the target ops. + NamedAttrList attrs; + CollectAllowedAttrs(src, &attrs); + + // Add all the attributes to the operations in the range. + if (!attrs.empty()) { + AddAttributes(begin, end, attrs); + } +} + +} // namespace TFR +} // namespace mlir diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..2c18f21b63257221562984793f0c8300a1b80118 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfr/utils/utils.h @@ -0,0 +1,66 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFR_UTILS_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFR_UTILS_UTILS_H_ + +#include + +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfr/ir/tfr_ops.h" + +namespace mlir { +namespace TFR { + +// This is a hardcoded rule for mapping a TF op name to the corresponding +// TFR function name. Examples: +// tf.Pack => tf__pack +// tf.ConcatV2 => tf__concat_v2 +// TODO(fengliuai): move to an util file. +std::string GetComposeFuncName(StringRef tf_op_name); + +// This is a hardcoded rule for mapping a TFR function op name to the +// corresponding TF opname. Examples: +// tf__pack -> tf.Pack +// tf__concat_v2 => tf.ConcatV2 +std::string GetTFOpName(StringRef compose_func_name); + +// Validate the attributes of 'src' is either contained in the registered +// attribute sets or in the allowed list. +LogicalResult ValidateAttrs(Operation* src, const StringSet<>& registered); + +// Copies all the allowed attributes in 'src' to 'dst'. The copy failed if the +// 'dst' has the attribute. Return a failure if there are any attributes are not +// allowed and also unregistered. +LogicalResult CopyAllowedUnregisteredAttrs(Operation* src, CallOp dst, + const StringSet<>& registered); + +// Copies all the allowed attributes in 'src' to 'dst'. FlatSymbolRefAttr is +// excluded. +LogicalResult CopyNonSymbolRefAttrs(CallOp src, Operation* dst); + +// Propagates all the attributes in 'src' to the operations between 'begin' and +// 'end'. Operation 'end' is excluded. +void PropagateAttrsToOperations(CallOp src, Block::iterator begin, + Block::iterator end); + +} // namespace TFR +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TFR_UTILS_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..52becaf1fe2d7cf564b2eb4d730f5c9276aa9fe1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/BUILD @@ -0,0 +1,766 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("//tensorflow:tensorflow.bzl", "if_google", "tf_cc_binary", "tf_cc_test") + +# Note: keep the following lines separate due to the way copybara works +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") +load( + "//tensorflow/core/platform:build_config.bzl", + "tf_proto_library", +) + +# TF to TFRT kernels conversion. +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [":friends"], + licenses = ["notice"], +) + +package_group( + name = "friends", + packages = [ + "//tensorflow/compiler/...", + "//tensorflow/core/tfrt/experimental/data/...", + "//tensorflow/core/tfrt/graph_executor/...", + "//tensorflow/core/tfrt/ifrt/...", + "//tensorflow/core/tfrt/kernels/...", + "//tensorflow/core/tfrt/mlrt/...", + "//tensorflow/core/tfrt/saved_model/...", + "//tensorflow/core/tfrt/tfrt_session/...", + ] + if_google([ + "//learning/brain/experimental/mlir/tflite/tfmrt/...", + "//learning/brain/experimental/mlir/tfrt_compiler/...", + "//learning/brain/experimental/tfrt/...", + "//learning/brain/tfrt/...", + "//learning/infra/mira/...", + "//learning/serving/contrib/tfrt/mlir/...", + "//learning/serving/testing/functional/...", + # Allow visibility from the mlir language server. + "//learning/brain/mlir/mlir_lsp_server/...", + "//smartass/brain/ops/...", + "//third_party/mlir_edge/tpgen/...", + "//tensorflow_serving/servables/tensorflow/...", + "//tensorflow_serving/servables/tensorflow/google/...", + "//third_party/tf_runtime_google/...", + ]), +) + +exports_files(["run_lit.sh"]) + +td_library( + name = "runtime_fallback_ops_td_files", + srcs = [ + "runtime_fallback/runtime_fallback_ops.td", + ], + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@tf_runtime//:OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "runtime_fallback_ops_inc_gen", + tbl_outs = { + "runtime_fallback_ops.h.inc": ["-gen-op-decls"], + "runtime_fallback_ops.cc.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "runtime_fallback/runtime_fallback_ops.td", + deps = [":runtime_fallback_ops_td_files"], +) + +cc_library( + name = "runtime_fallback_opdefs", + srcs = [ + "runtime_fallback/runtime_fallback_combine.cc", + "runtime_fallback/runtime_fallback_ops.cc", + ], + hdrs = [ + "runtime_fallback/runtime_fallback_ops.h", + ], + deps = [ + ":runtime_fallback_ops_inc_gen", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:tensor_opdefs", + ], +) + +cc_library( + name = "runtime_fallback_executor", + testonly = True, + srcs = [ + "runtime_fallback/runtime_fallback_executor.cc", + ], + hdrs = [ + "runtime_fallback/runtime_fallback_executor.h", + ], + deps = [ + ":host_context_util", + ":tf_to_tfrt", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:framework", + "//tensorflow/core/platform:env", + "//tensorflow/core/platform:threadpool_interface", + "//tensorflow/core/runtime_fallback/kernel:kernel_fallback_execute_compat_eager", + "//tensorflow/core/runtime_fallback/runtime:kernel_utils", + "//tensorflow/core/runtime_fallback/runtime:runtime_fallback_alwayslink", + "//tensorflow/core/tfrt/utils:fallback_tensor", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@tf_runtime//:basic_kernels_alwayslink", + "@tf_runtime//:bef", + "@tf_runtime//:befexecutor", + "@tf_runtime//:core_runtime_alwayslink", + "@tf_runtime//:hostcontext", + "@tf_runtime//:mlirtobef", + "@tf_runtime//:support", + ], +) + +cc_library( + name = "corert_converter", + srcs = [ + "transforms/corert_converter.cc", + ], + hdrs = [ + "transforms/corert_converter.h", + ], + compatible_with = get_compatible_with_portable(), # copybara: comment + deps = [ + ":attr_lowering_utils", + ":transform_utils", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core:framework", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:DerivedAttributeOpInterface", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:core_runtime_opdefs", + ], +) + +cc_library( + name = "fallback_converter", + srcs = [ + "transforms/fallback_converter.cc", + ], + hdrs = [ + "transforms/fallback_converter.h", + ], + compatible_with = get_compatible_with_portable(), # copybara: comment + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_opdefs", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:core_runtime_opdefs", + ], +) + +cc_library( + name = "transforms/gpu_passes", + srcs = ["transforms/gpu_passes.cc"], + hdrs = ["transforms/gpu_passes.h"], + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_gpu_opdefs", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "tfrt_pipeline_options", + hdrs = ["transforms/tfrt_pipeline_options.h"], + compatible_with = get_compatible_with_portable(), # copybara: comment + deps = [ + ":tfrt_compile_options", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "tf_to_tfrt", + srcs = [ + "transforms/cross_device_transfer.cc", + "transforms/deduplicate_batch_function.cc", + "transforms/deduplicate_if_result_pass.cc", + "transforms/fuse_tpu_compile_and_execute_ops.cc", + "transforms/insert_tensor_copy.cc", + "transforms/lower_saved_model.cc", + "transforms/merge_tf_if_ops.cc", + "transforms/optimize.cc", + "transforms/optimize_tf_control_flow_side_effect.cc", + "transforms/passes.cc", + "transforms/reconfig_batch_op.cc", + "transforms/remove_device_attribute.cc", + "transforms/remove_tf_if_const_args.cc", + "transforms/reorder_assert.cc", + "transforms/sink_in_invariant_ops.cc", + "transforms/tf_to_tfrt.cc", + "transforms/xla_rewrite_pass.cc", + ], + hdrs = [ + "transforms/passes.h", + ], + deps = [ + ":attr_lowering_utils", + ":corert_converter", + ":cost_analysis", + ":fallback_converter", + ":tensor_array_side_effect_analysis", + ":tfrt_compile_options", + ":tfrt_pipeline_options", + ":tpu_passes", + ":transform_utils", + ":transforms/gpu_passes", + ":transforms/set_shape_invariant_in_while_ops", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_op_interfaces", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_saved_model_asset_sinking_pass", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_gpu_opdefs", + "//tensorflow/compiler/tf2xla:tf2xla_defs", + "//tensorflow/core:framework", + "//tensorflow/core:framework_lite", + "//tensorflow/core/ir/types:Dialect", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:status", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Rewrite", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:core_runtime_opdefs", + "@tf_runtime//:stream_analysis", + "@tf_runtime//:test_kernels_opdefs", + "@tsl//tsl/platform:errors", + ], + alwayslink = 1, +) + +cc_library( + name = "host_context_util", + srcs = ["utils/host_context.cc"], + hdrs = ["utils/host_context.h"], + deps = [ + "//tensorflow/core/platform:logging", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log", + "@tf_runtime//:hostcontext", + ], +) + +cc_library( + name = "function", + srcs = [ + "function/function.cc", + ], + hdrs = [ + "function/function.h", + ], + deps = [ + ":tf_to_tfrt", + ":tfrt_compile_options", + ":tfrt_pipeline_options", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:translate_lib", + "//tensorflow/core/platform:errors", + "//tensorflow/core/platform:status", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tf_runtime//:bef", + "@tf_runtime//:core_runtime", + "@tf_runtime//:mlirtobef", + ], +) + +cc_library( + name = "saved_model", + srcs = [ + "saved_model/saved_model.cc", + ], + hdrs = [ + "saved_model/saved_model.h", + ], + deps = [ + ":tf_to_tfrt", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:translate_lib", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_dialect_passes", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:status", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tf_runtime//:bef", + "@tf_runtime//:core_runtime", + "@tf_runtime//:mlirtobef", + "@tsl//tsl/platform:errors", + ], +) + +cc_library( + name = "import_model", + srcs = [ + "translate/import_model.cc", + ], + hdrs = [ + "translate/import_model.h", + ], + visibility = [ + "//tensorflow/compiler/mlir/tfrt/tests/saved_model:__pkg__", + "//tensorflow/compiler/mlir/tfrt/transforms/mlrt:__pkg__", + "//tensorflow/core/tfrt/graph_executor:__pkg__", + "//tensorflow/core/tfrt/saved_model:__pkg__", + "//tensorflow/core/tfrt/saved_model/utils:__pkg__", + ], + deps = [ + ":backend_compiler", + ":function", + ":tf_to_tfrt", + ":tfrt_compile_options", + ":tfrt_pipeline_options", + ":tpu_passes", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms:tf_saved_model_asset_sinking_pass", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:lower_cluster_to_runtime_ops", + "//tensorflow/compiler/mlir/tf2xla/api/v2:cluster_tf", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_dialect_to_executor", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/common_runtime:function_def_utils", + "//tensorflow/core/platform:status", + "//tensorflow/core/tfrt/fallback:fallback_state", + "//tensorflow/core/tfrt/runtime", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:vlog_is_on", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncExtensions", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tf_runtime//:bef", + "@tf_runtime//:mlirtobef", + "@tf_runtime//:support", + "@tsl//tsl/platform:env", + "@tsl//tsl/platform:errors", + ], +) + +tf_cc_test( + name = "import_model_test", + srcs = [ + "translate/import_model_test.cc", + ], + tags = ["no_oss"], + deps = [ + ":import_model", + ":tfrt_compile_options", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "tfrt_compile_options", + srcs = ["translate/tfrt_compile_options.cc"], + hdrs = ["translate/tfrt_compile_options.h"], + compatible_with = get_compatible_with_portable(), # copybara: comment + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core/protobuf:for_core_protos_cc", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "cost_analysis", + srcs = ["analysis/cost_analysis.cc"], + hdrs = ["analysis/cost_analysis.h"], + deps = [ + ":constants", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/core/platform:status", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "//tensorflow/core/tfrt/fallback:op_cost_map_proto_cc", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@tf_runtime//:compiler_tfrt_op_interfaces", + ], +) + +cc_library( + name = "test_cost_analysis_pass", + srcs = ["analysis/test_cost_analysis_pass.cc"], + deps = [ + ":cost_analysis", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + ], + alwayslink = 1, +) + +cc_library( + name = "tensor_array_side_effect_analysis", + srcs = ["analysis/tensor_array_side_effect_analysis.cc"], + hdrs = ["analysis/tensor_array_side_effect_analysis.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "test_tensor_array_side_effect_analysis", + srcs = ["analysis/test_tensor_array_side_effect_analysis.cc"], + deps = [ + ":tensor_array_side_effect_analysis", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], + alwayslink = 1, +) + +tf_proto_library( + name = "analysis/analysis_proto", + srcs = ["analysis/analysis.proto"], +) + +cc_library( + name = "passes", + visibility = ["//visibility:private"], + deps = [ + "//tensorflow/compiler/mlir/tfrt:tf_to_tfrt", + ], +) + +cc_library( + name = "transforms/set_shape_invariant_in_while_ops", + srcs = ["transforms/set_shape_invariant_in_while_ops.cc"], + hdrs = ["transforms/set_shape_invariant_in_while_ops.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +cc_library( + name = "tf_tfrt_opt_lib", + testonly = True, + srcs = ["tf-tfrt-opt.cc"], + deps = [ + ":passes", + ":test_cost_analysis_pass", + ":test_opkernels", + ":test_tensor_array_side_effect_analysis", + ":tf_to_tfrt", + ":tpu_passes", + ":transforms/gpu_passes", + "//tensorflow/compiler/mlir:init_mlir", + "//tensorflow/compiler/mlir:passes", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:lower_cluster_to_runtime_ops", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_sync_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt:pack_inputs_pass", + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt:tf_ifrt_passes", + "//tensorflow/compiler/mlir/tfrt/transforms/mlrt:passes", + "//tensorflow/core:tensorflow", + "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//mlir:MlirOptLib", + "@llvm-project//mlir:RegisterAllDialects", # buildcleaner: keep + "@llvm-project//mlir:RegisterAllPasses", # buildcleaner: keep + "@llvm-project//mlir:ShapeDialect", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@stablehlo//:register", + "@tf_runtime//:init_tfrt_dialects", + "@tf_runtime//:print_stream_pass", + "@xla//xla/mlir_hlo", + ], +) + +tf_cc_binary( + name = "tf-tfrt-opt", + testonly = True, + deps = [":tf_tfrt_opt_lib"], +) + +tf_cc_binary( + name = "tfrt_translate", + srcs = ["tools/tfrt_translate/static_registration.cc"], + visibility = [":friends"], + deps = [ + "@llvm-project//mlir:IR", + "@llvm-project//mlir:TranslateLib", + "@tf_runtime//:beftomlir_translate", + "@tf_runtime//:init_tfrt_dialects", + "@tf_runtime//:mlirtobef_translate", + ] + if_google( + [ + "//third_party/tf_runtime_llvm:tfrt_translate_main", + ], + ["@tf_runtime//third_party/llvm_derived:tfrt_translate_main"], + ), +) + +tf_cc_binary( + name = "bef_executor", + testonly = True, + visibility = [":friends"], + deps = [ + "@tf_runtime//:dtype", + "@tf_runtime//:simple_tracing_sink", + "@tf_runtime//tools:bef_executor_expensive_kernels", + "@tf_runtime//tools:bef_executor_lib", + "@tf_runtime//tools:bef_executor_lightweight_kernels", + ], +) + +cc_library( + name = "tfrt_fallback_registration", + srcs = [ + "tfrt_fallback_registration.cc", + ], + hdrs = [ + "tfrt_fallback_registration.h", + ], + visibility = [":friends"] + if_google([ + "//learning/brain/experimental/tfrt/visualization:__pkg__", + # Allow visibility from the mlir language server. + "//learning/brain/mlir/mlir_lsp_server:__pkg__", + ]), + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_sync_opdefs", + "@llvm-project//mlir:IR", + ], +) + +tf_cc_binary( + name = "tfrt_fallback_translate", + srcs = [ + "tfrt_fallback_translate_registration.cc", + ], + visibility = [":friends"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_registration", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_opdefs", + "@llvm-project//mlir:TranslateLib", + "@tf_runtime//:init_tfrt_dialects", + "@tf_runtime//:mlirtobef_translate", + ] + if_google( + [ + "//third_party/tf_runtime_llvm:tfrt_translate_main", + ], + ["@tf_runtime//third_party/llvm_derived:tfrt_translate_main"], + ), +) + +cc_library( + name = "attr_lowering_utils", + srcs = [ + "transforms/attr_lowering_utils.cc", + ], + hdrs = [ + "transforms/attr_lowering_utils.h", + ], + compatible_with = get_compatible_with_portable(), # copybara: comment + visibility = [":friends"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_attributes", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@tf_runtime//:core_runtime_opdefs", + ], +) + +cc_library( + name = "transform_utils", + srcs = [ + "transforms/utils.cc", + ], + hdrs = [ + "transforms/utils.h", + ], + compatible_with = get_compatible_with_portable(), # copybara: comment + visibility = [":friends"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:core_runtime_opdefs", + ], +) + +cc_library( + name = "transforms/update_op_cost_in_tfrt_mlir", + srcs = ["transforms/update_op_cost_in_tfrt_mlir.cc"], + hdrs = ["transforms/update_op_cost_in_tfrt_mlir.h"], + deps = [ + ":cost_analysis", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "@llvm-project//mlir:IR", + ], +) + +cc_library( + name = "tpu_passes", + hdrs = ["transforms/tpu_passes.h"], + visibility = [ + ":friends", + ] + if_google([ + "//learning/brain/tfrt/ifrt/pjrt/__subpackages__", + "//learning/serving/servables/tfrt:__subpackages__", + "//smartass/brain/util:__pkg__", + ]), + deps = [ + ":fallback_converter", + ":tfrt_compile_options", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "constants", + hdrs = ["constants.h"], +) + +cc_library( + name = "test_opkernels", + testonly = True, + srcs = ["test_opkernels.cc"], + deps = ["//tensorflow/core:framework"], + alwayslink = True, +) + +cc_library( + name = "backend_compiler", + srcs = ["backend_compiler.cc"], + hdrs = ["backend_compiler.h"], + deps = [ + "//tensorflow/core/tfrt/runtime", + "@com_google_absl//absl/status", + "@llvm-project//mlir:IR", + ], +) + +cc_library( + name = "export", + srcs = ["utils/export.cc"], + hdrs = ["utils/export.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:mlir_roundtrip_flags", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tf2xla/api/v1:tf_dialect_to_executor", + "//tensorflow/compiler/mlir/tf2xla/api/v2:tf_executor_to_graph", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/functional:any_invocable", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tsl//tsl/platform:errors", + "@tsl//tsl/profiler/lib:traceme", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/analysis.proto b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/analysis.proto new file mode 100644 index 0000000000000000000000000000000000000000..260d099f326e7dffd609a78d9389983cf5aa4a66 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/analysis.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package mlir.tfrt; + +message CompatibilityAnalysisReportProto { + bool unknown_dialect = 1; + bool ref_variable = 2; + bool incompatible_variable = 3; + bool incompatible_attribute = 4; + bool control_flow_v1 = 5; + string method_name = 6; + + // TODO(chky): add more checks, eg. tensor datatypes. +} + +message CompatibilityAnalysisProto { + CompatibilityAnalysisReportProto summary = 1; + + message OpInfo { + int32 count = 1; + + CompatibilityAnalysisReportProto report = 2; + } + + map ops = 2; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.cc new file mode 100644 index 0000000000000000000000000000000000000000..60016ca7719342c09f19a2b76d44d3f861ddea6f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.cc @@ -0,0 +1,235 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h" + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tfrt/constants.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tfrt/compiler/opdefs/tfrt_op_interfaces.h" // from @tf_runtime + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +constexpr int64_t kDefaultCheapCost = 1; + +int64_t GetRankedTensorSize(mlir::TensorType type) { + auto shape = type.getShape(); + + int64_t size = 1; + for (int64_t dim : shape) { + // For unknown dimensions, use 1 as the size because it is usually the batch + // dimension. + // + // TODO(chky): Find out a better default number for this case. + size *= std::max(kDefaultCheapCost, dim); + } + + return size; +} + +int64_t InferTensorSize(const CostContext& context, mlir::TensorType type) { + if (type.hasRank()) return GetRankedTensorSize(type); + return context.default_unranked_tensor_size; +} + +// The cost function for tf.LookupTableFindV2. +int64_t InferLookupTableFindV2Cost(const CostContext& context, + mlir::TF::LookupTableFindV2Op op) { + // tf.LookupTableFindV2 ops are usually more costly than tf.AddV2 with the + // same input size, as it involves more operations like hashing, map lookup, + // etc. + constexpr int64_t kLookupTableFindCostScale = 8; + constexpr int64_t kLookupTableFindStringKeyCostScale = 16; + + auto value_type = mlir::cast(op.getValues().getType()); + auto key_type = mlir::cast(op.getKeys().getType()); + + int64_t output_size = InferTensorSize(context, value_type); + + int64_t cost = kLookupTableFindCostScale * output_size; + + if (mlir::isa(key_type.getElementType())) + cost *= kLookupTableFindStringKeyCostScale; + + return cost; +} + +// The cost function for tf.GatherV2. +int64_t InferGatherV2Cost(const CostContext& context, mlir::TF::GatherV2Op op) { + return InferTensorSize( + context, mlir::cast(op.getOutput().getType())); +} + +// The cost function for tf.SparseSegmentSumOp. +template +int64_t InferSparseSegmentOpCost(const CostContext& context, OpType op) { + return InferTensorSize( + context, mlir::cast(op.getOutput().getType())); +} + +// CostFunctionRegistry is a map from op names to their cost functions. +using CostFunctionRegistry = absl::flat_hash_map; + +void RegisterCostFunction(CostFunctionRegistry& registry, + absl::string_view op_name, + CostFunction cost_function) { + auto r = registry.try_emplace(op_name, std::move(cost_function)); + assert(r.second); + (void)r; +} + +template +void RegisterCostFunction(CostFunctionRegistry& registry, F f) { + RegisterCostFunction( + registry, OpType::getOperationName().str(), + [f = std::move(f)](const CostContext& context, mlir::Operation* op) { + return f(context, llvm::cast(op)); + }); +} + +CostFunctionRegistry& GetCostFunctionRegistry() { + static auto* const registry = []() { + auto* registry = new CostFunctionRegistry; + // TODO(chky): Find a more scalable way to register cost functions. One + // option is to incorporate it is TF MLIR ODS. + RegisterCostFunction(*registry, InferGatherV2Cost); + RegisterCostFunction( + *registry, InferSparseSegmentOpCost); + RegisterCostFunction( + *registry, InferSparseSegmentOpCost); + RegisterCostFunction( + *registry, InferSparseSegmentOpCost); + RegisterCostFunction( + *registry, InferLookupTableFindV2Cost); + return registry; + }(); + return *registry; +} + +} // namespace + +void RegisterCostFunction(absl::string_view op_name, + CostFunction cost_function) { + RegisterCostFunction(GetCostFunctionRegistry(), op_name, + std::move(cost_function)); +} + +bool HasCostFunctionRegistered(absl::string_view op_name) { + return GetCostFunctionRegistry().contains(op_name); +} + +int64_t CostAnalysis::GetCost(mlir::Operation* op) const { + assert(cost_map_.count(op) > 0); + return cost_map_.lookup(op); +} + +void CostAnalysis::AnalyzeArguments(mlir::func::FuncOp func_op) { + // Use the max size among function inputs as the default size of dynamic + // shaped tensors in the function. + for (auto arg : func_op.getArguments()) { + if (!mlir::isa(arg.getType())) continue; + auto type = mlir::cast(arg.getType()); + if (type.hasRank()) { + max_arg_size_ = std::max(max_arg_size_, GetRankedTensorSize(type)); + } + } +} + +void CostAnalysis::AnalyzeBlock(mlir::Block* block) { + for (auto& op : *block) { + EvaluateCost(&op); + } +} + +void CostAnalysis::EvaluateCost(mlir::Operation* op) { + if (auto cost_function = + mlir::dyn_cast(op)) { + cost_map_[op] = cost_function.cost(); + return; + } + + if (!llvm::isa(op->getDialect())) { + cost_map_[op] = max_arg_size_; + return; + } + + // Try to use its cost function if it is registered. + const auto& registry = GetCostFunctionRegistry(); + absl::string_view op_name = op->getName().getStringRef(); + auto iter = registry.find(op_name); + if (iter != registry.end()) { + CostContext context; + context.default_unranked_tensor_size = max_arg_size_; + cost_map_[op] = iter->second(context, op); + return; + } + + // Try to use the recorded cost if any. + if (cost_recorder_ != nullptr) { + const auto op_key_attr = + op->getAttrOfType(kOpKeyAttrName); + if (op_key_attr) { + cost_map_[op] = cost_recorder_->GetCost(op_key_attr.getInt()); + return; + } + } + + // These ops are cheap regardless of their input sizes. + // + // TODO(chky): Find a more scalable way to figure out cheap ops. + if (llvm::isa(op)) { + cost_map_[op] = kDefaultCheapCost; + return; + } + + // For other ops, use the sum of input sizes as its cost. + int64_t cost = kDefaultCheapCost; + for (auto operand : op->getOperands()) { + auto type = mlir::cast(operand.getType()); + if (type.hasRank()) { + cost += GetRankedTensorSize(type); + } else { + // For unranked tensors, use the max size among the input tensors. This is + // because the only dynamic information of the function should be the + // input, so the size of dynamic tensors should be usually capped by + // inputs' sizes. + cost += max_arg_size_; + } + } + + cost_map_[op] = cost; +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h new file mode 100644 index 0000000000000000000000000000000000000000..e2da3316b7c4d69413d18072a86c6d93cb642a70 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h @@ -0,0 +1,98 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_COST_ANALYSIS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_COST_ANALYSIS_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tensorflow/core/tfrt/fallback/op_cost_map.pb.h" + +namespace tensorflow { +namespace tfrt_compiler { + +// Analyze costs for tensorflow operations. +// +// The current heuristic used is quite simple, which is to calculate the total +// size of input tensors. The exception is that ops whose cost is irrelevant to +// input sizes, such as tf.Shape and tf.Reshape, are whitelisted to have cheap +// cost. This cost analysis is expected to be used conservatively (eg. use a low +// threshold to decide whether a cost is cheap or expensive), as it might not be +// accurate in some cases. +// +class CostAnalysis { + public: + explicit CostAnalysis( + mlir::func::FuncOp func_op, + const tfrt_stub::CostRecorder* cost_recorder = nullptr) { + cost_recorder_ = cost_recorder; + AnalyzeArguments(func_op); + AnalyzeBlock(&func_op.front()); + } + + int64_t GetCost(mlir::Operation* op) const; + + private: + void AnalyzeArguments(mlir::func::FuncOp func_op); + void AnalyzeBlock(mlir::Block* block); + void EvaluateCost(mlir::Operation* op); + + int64_t max_arg_size_ = 1; + llvm::DenseMap cost_map_; + const tfrt_stub::CostRecorder* cost_recorder_; +}; + +struct CostContext { + int64_t default_unranked_tensor_size; +}; + +using CostFunction = + std::function; + +void RegisterCostFunction(absl::string_view op_name, + CostFunction cost_function); + +template +void RegisterCostFunction(F f) { + RegisterCostFunction( + OpType::getOperationName().str(), + [f = std::move(f)](const CostContext& context, mlir::Operation* op) { + return f(context, llvm::cast(op)); + }); +} + +template +struct CostFunctionRegistration { + explicit CostFunctionRegistration( + std::function cost_function) { + RegisterCostFunction(std::move(cost_function)); + } +}; + +bool HasCostFunctionRegistered(absl::string_view op_name); + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_COST_ANALYSIS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.cc new file mode 100644 index 0000000000000000000000000000000000000000..a463a7247a151cff3af273ec163506e66fef1311 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.cc @@ -0,0 +1,53 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h" + +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +namespace tfrt_compiler { + +bool IsTensorArrayOp(mlir::Operation* op) { + return llvm::isa(op); +} + +static bool FunctionContainsOnlyNoSideEffectOpOrTensorArrayOp( + mlir::func::FuncOp func_op) { + for (mlir::Operation& op : func_op.front()) { + if (!mlir::isMemoryEffectFree(&op) && !IsTensorArrayOp(&op)) return false; + } + + return true; +} + +TensorArraySideEffectAnalysis::TensorArraySideEffectAnalysis( + mlir::ModuleOp module) { + for (auto func_op : module.getOps()) { + if (FunctionContainsOnlyNoSideEffectOpOrTensorArrayOp(func_op)) { + set_.insert(func_op); + } + } +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h new file mode 100644 index 0000000000000000000000000000000000000000..5f08f6c95d014f31696c82fa586087ab9e45e3ec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h @@ -0,0 +1,52 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_TENSOR_ARRAY_SIDE_EFFECT_ANALYSIS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_TENSOR_ARRAY_SIDE_EFFECT_ANALYSIS_H_ + +#include "llvm/ADT/DenseSet.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project + +namespace tensorflow { +namespace tfrt_compiler { + +// Return true if it is a TensorArrayOp, eg. TensorArrayV3Op. +bool IsTensorArrayOp(mlir::Operation* op); + +// This class provides utilities for analyzing side effects for TensorArray ops +// in the graph. mlir::TF::SideEffectAnalysis currently produces suboptimal +// side-effect analysis for TensorArray ops. On the other hand, control +// dependencies are already sorted out for TensorArray ops in the original TF +// graph. Each TensorArray op will take or produce a `flow` value and they are +// already properly chained in the origninal TF graph. +class TensorArraySideEffectAnalysis { + public: + explicit TensorArraySideEffectAnalysis(mlir::ModuleOp module); + + // Return if the function contains only non-side-effecting ops or TensorArray + // ops. + bool HasAtMostTensorArrayEffect(mlir::func::FuncOp func_op) const { + return set_.count(func_op) > 0; + } + + private: + llvm::DenseSet set_; +}; + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_ANALYSIS_TENSOR_ARRAY_SIDE_EFFECT_ANALYSIS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_cost_analysis_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_cost_analysis_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..be1cf3f611d09b5f54856d37a6dd6c0be30b35b9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_cost_analysis_pass.cc @@ -0,0 +1,46 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h" + +namespace tensorflow { +namespace tfrt_compiler { + +class TestCostAnalysis + : public mlir::PassWrapper> { + llvm::StringRef getArgument() const final { + return "tfrt-test-cost-analysis"; + } + llvm::StringRef getDescription() const final { + return "Add remarks based on cost analysis for testing purpose."; + } + void runOnOperation() override { + const auto& cost_analysis = getAnalysis(); + + auto func_op = getOperation(); + for (auto& op : func_op.front()) { + op.emitRemark() << "Cost: " << cost_analysis.GetCost(&op); + } + } +}; + +static mlir::PassRegistration pass; + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_tensor_array_side_effect_analysis.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_tensor_array_side_effect_analysis.cc new file mode 100644 index 0000000000000000000000000000000000000000..e452ac23c70e300e684778427ce49042d6368c22 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/analysis/test_tensor_array_side_effect_analysis.cc @@ -0,0 +1,57 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +class TestTensorArraySideEffectAnalysis + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + TestTensorArraySideEffectAnalysis) + + private: + llvm::StringRef getArgument() const final { + return "tfrt-test-tensor-array-effect"; + } + llvm::StringRef getDescription() const final { + return "Test TensorArraySideEffectAnalysis"; + } + void runOnOperation() override { + auto module = getOperation(); + TensorArraySideEffectAnalysis tensor_array_side_effect_analysis(module); + + for (auto func_op : module.getOps()) { + func_op.emitRemark() << "HasAtMostTensorArrayEffect: " + << tensor_array_side_effect_analysis + .HasAtMostTensorArrayEffect(func_op); + } + } +}; + +mlir::PassRegistration pass; + +} // namespace +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.cc new file mode 100644 index 0000000000000000000000000000000000000000..efeeae89e247b383392fa48abf8981da91b35403 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.cc @@ -0,0 +1,22 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/backend_compiler.h" + +namespace tensorflow { + +BackendCompiler::~BackendCompiler() = default; + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.h new file mode 100644 index 0000000000000000000000000000000000000000..b3eefec186c93b1de8d928f492487cbb6c22eae9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/backend_compiler.h @@ -0,0 +1,41 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_BACKEND_COMPILER_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_BACKEND_COMPILER_H_ + +#include "absl/status/status.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "tensorflow/core/tfrt/runtime/runtime.h" + +namespace tensorflow { + +class BackendCompiler { + public: + virtual ~BackendCompiler(); + + virtual void GetDependentDialects(mlir::DialectRegistry& registry) const {} + + // Compile the `module` in TF dialect. The result module should be also in TF + // dialect. + virtual absl::Status CompileTensorflow( + tfrt_stub::ModelRuntimeContext& model_context, + mlir::ModuleOp module) const = 0; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_BACKEND_COMPILER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/constants.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/constants.h new file mode 100644 index 0000000000000000000000000000000000000000..a313e7e49ddfb7d954250790aabf7c6265037474 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/constants.h @@ -0,0 +1,28 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_CONSTANTS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_CONSTANTS_H_ + +namespace tensorflow { +namespace tfrt_compiler { + +// Use __ prefix to indicate this is internal attribute. +inline constexpr char kOpKeyAttrName[] = "__op_key"; + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_CONSTANTS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.cc new file mode 100644 index 0000000000000000000000000000000000000000..8ae72003d8b61ead47944e2a1c73451a9a6582d2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.cc @@ -0,0 +1,100 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/function/function.h" + +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "absl/strings/match.h" +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_converter/mlir_to_bef.h" // from @tf_runtime + +namespace tensorflow { + +absl::Status CompileTFMLIRToBEF(const TfrtFunctionCompileOptions& options, + mlir::ModuleOp module, + tfrt::BefBuffer* bef_buffer) { + mlir::OpPrintingFlags print_flags; + print_flags.elideLargeElementsAttrs(); + + if (VLOG_IS_ON(1)) { + VLOG(1) << "Input TF Executor dialect:"; + DumpMlirOpToFile("tf_to_tfrt_tf_executor_dialect", module); + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + + // Lower MLIR TF Dialect to MLIR TFRT CoreRT dialect. + mlir::PassManager pm(module.getContext()); + tensorflow::applyTensorflowAndCLOptions(pm); + + tensorflow::TfrtPipelineOptions pass_options; + if (!options.default_device.empty()) { + pass_options.default_device = options.default_device; + } + if (!options.force_data_format.empty()) { + pass_options.force_data_format = options.force_data_format; + } + // TODO(tfrt-devs): Current MaxPoolingOp only supports NHWC on device type + // CPU. Enable this layout optimization after we introduce TFRT native ops + // for training. + if (absl::StrContains(pass_options.default_device, "CPU")) { + pass_options.skip_fold_transpose_in_ops = true; + } + pass_options.enable_optimizer = options.enable_optimizer; + // Use TFRT TPU OpKernel for training. + pass_options.target_tpurt = false; + pass_options.tpu_use_core_selector = options.tpu_use_core_selector; + pass_options.tpu_use_bundled_transfer = options.tpu_use_bundled_transfer; + pass_options.tpu_lower_to_fallback = options.tpu_lower_to_fallback; + pass_options.tpu_fuse_ops = options.tpu_fuse_ops; + pass_options.tpu_transfer_result_to_host = + options.tpu_transfer_result_to_host; + absl::Status status = + tensorflow::CreateTfExecutorToTfrtPipeline(pm, pass_options); + if (!status.ok()) { + return diag_handler.Combine(status); + } + + if (mlir::failed(pm.run(module))) + return diag_handler.Combine( + absl::InternalError("failed to lower TF Dialect to CoreRT dialect.")); + + if (VLOG_IS_ON(1)) { + VLOG(1) << "TFRT dialect: "; + DumpMlirOpToFile("tf_to_tfrt_tfrt_dialect", module); + } + + *bef_buffer = + tfrt::ConvertMLIRToBEF(module, /* disable_optional_sections = */ true); + if (bef_buffer->empty()) + return diag_handler.Combine( + absl::InternalError("failed to convert MLIR to BEF.")); + + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.h new file mode 100644 index 0000000000000000000000000000000000000000..1cd09562c0eff072c26565087a93411ab4287dd6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/function/function.h @@ -0,0 +1,83 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_FUNCTION_FUNCTION_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_FUNCTION_FUNCTION_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/core/platform/status.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/core_runtime/tensor_handle.h" // from @tf_runtime + +namespace tfrt { +class CoreRuntime; +} + +namespace mlir { +class ModuleOp; +} + +namespace tensorflow { + +struct TfrtFunctionCompileOptions : public TfrtCompileOptions { + // Currently only SavedModel API inference uses the tpu_fuse_ops option + TfrtFunctionCompileOptions() { + tpu_fuse_ops = false; + // Currently grappler is not correctly applied in the eager execution of TF + // functions, as it may sometimes remove arguments and results. + enable_grappler = false; + } + + // If true, use ServingCoreSelector to pick TPU core. Otherwise, obtain core + // location from assigned device name. + // Currently we don't use core_selector for training use cases. + bool tpu_use_core_selector = false; + + // If true, use BundledTransferToTpuOp to transfer variables and input tensors + // to TPU. + bool tpu_use_bundled_transfer = false; + + // If true, lower an TF op that's placed on TPU device to be executed with + // tfrt_fallback.execute. + // Currently for training use cases we need to lower the op to corert.execute + // to execute with TPU OpHandler, and with TFRT's native implementation. + // TODO(b/188940204): remove this config after we clear up the TPU variable + // implementation. + bool tpu_lower_to_fallback = false; + // If true, transfer the result of TPUExecuteOp from TPU to host. + // Currently for training and Python bulk inference use cases, we don't need + // to proactively transfer the result to host since the consumer op (or + // function) of the result may still be on TPU. + // TODO(b/194081364): remove this option once we unify servo TPU serving + // result transfer behavior. + bool tpu_transfer_result_to_host = false; +}; + +// Compile MLIR generated by tf.function in TF dialect into BEF. +absl::Status CompileTFMLIRToBEF(const TfrtFunctionCompileOptions& options, + mlir::ModuleOp module, + tfrt::BefBuffer* bef_buffer); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_FUNCTION_FUNCTION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..e95997eb8b4550844db5f5cf0bbeb99365d5d34b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/BUILD @@ -0,0 +1,235 @@ +# BUILD targets for runtime fallback opdefs. + +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +# Note: keep the following lines separate due to the way copybara works +load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +cc_library( + name = "tfrt_fallback_opdefs", + srcs = [ + "tfrt_fallback.cc", + ], + hdrs = ["tfrt_fallback.h"], + compatible_with = get_compatible_with_portable(), # copybara: comment + visibility = [ + # copybara:uncomment "//learning/brain/experimental/tfrt:__subpackages__", + # copybara:uncomment "//learning/brain/tfrt/tpu/compiler:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + "//tensorflow/core/runtime_fallback:internal", + "//tensorflow/core/tfrt/saved_model:friends", + # copybara:uncomment "//third_party/tf_runtime_google:__subpackages__", + ], + deps = [ + ":tfrt_fallback_opdefs_inc_gen", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tfrt_fallback_async_opdefs", + srcs = [ + "tfrt_fallback_async.cc", + ], + hdrs = ["tfrt_fallback_async.h"], + compatible_with = get_compatible_with_portable(), # copybara: comment + visibility = [ + # copybara:uncomment "//learning/brain/experimental/tfrt:__subpackages__", + # copybara:uncomment "//learning/brain/tfrt/tpu/compiler/mlir:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + "//tensorflow/core/tfrt/saved_model:friends", + ], + deps = [ + ":tfrt_fallback_async_opdefs_inc_gen", + ":tfrt_fallback_common", + ":tfrt_fallback_opdefs", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@llvm-project//mlir:Transforms", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:compiler_tfrt_op_interfaces", + "@tf_runtime//:compiler_tfrt_traits", + "@tf_runtime//:core_runtime_opdefs", + "@tf_runtime//:core_runtime_sync_opdefs", + ], +) + +cc_library( + name = "tfrt_fallback_sync_opdefs", + srcs = [ + "tfrt_fallback_sync.cc", + ], + hdrs = ["tfrt_fallback_sync.h"], + visibility = [ + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + "//tensorflow/core/runtime_fallback:internal", + ], + deps = [ + ":tfrt_fallback_common", + ":tfrt_fallback_opdefs", + ":tfrt_fallback_sync_opdefs_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + "@tf_runtime//:basic_kernels_opdefs", + "@tf_runtime//:core_runtime_opdefs", + "@tf_runtime//:tensor_opdefs", + ], +) + +cc_library( + name = "tfrt_fallback_common", + srcs = ["tfrt_fallback_common.cc"], + hdrs = ["tfrt_fallback_common.h"], + compatible_with = get_compatible_with_portable(), # copybara: comment + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@tf_runtime//:basic_kernels_opdefs", + ], +) + +cc_library( + name = "tfrt_fallback_util", + srcs = ["tfrt_fallback_util.cc"], + hdrs = ["tfrt_fallback_util.h"], + visibility = ["//tensorflow/compiler/mlir/tfrt:__subpackages__"], + deps = [ + ":tfrt_fallback_async_opdefs", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + ], +) + +td_library( + name = "tfrt_fallback_td_files", + srcs = [ + "tfrt_fallback.td", + "tfrt_fallback_async.td", + "tfrt_fallback_sync.td", + ], + compatible_with = get_compatible_with_portable(), # copybara: comment + includes = ["."], + visibility = [ + "//learning/brain/experimental/tfrt:__subpackages__", + "//learning/brain/tfrt/tpu/compiler:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@tf_runtime//:CoreRTTdFiles", + "@tf_runtime//:OpBaseTdFiles", + "@tf_runtime//:compiler_td_files", + ], +) + +gentbl_cc_library( + name = "tfrt_fallback_opdefs_inc_gen", + compatible_with = get_compatible_with_portable(), # copybara: comment + tbl_outs = { + "tfrt_fallback.h.inc": ["-gen-op-decls"], + "tfrt_fallback.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tfrt_fallback.td", + deps = [":tfrt_fallback_td_files"], +) + +gentbl_cc_library( + name = "tfrt_fallback_async_opdefs_inc_gen", + compatible_with = get_compatible_with_portable(), # copybara: comment + tbl_outs = { + "tfrt_fallback_async.h.inc": ["-gen-op-decls"], + "tfrt_fallback_async.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tfrt_fallback_async.td", + deps = [":tfrt_fallback_td_files"], +) + +gentbl_cc_library( + name = "tfrt_fallback_sync_opdefs_inc_gen", + tbl_outs = { + "tfrt_fallback_sync.h.inc": ["-gen-op-decls"], + "tfrt_fallback_sync.cpp.inc": ["-gen-op-defs"], + "tfrt_fallback_sync_dialect.h.inc": [ + "-gen-dialect-decls", + "-dialect=tfrt_fallback_sync", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tfrt_fallback_sync.td", + test = True, + visibility = [ + "//learning/brain/experimental/tfrt:__subpackages__", + ], + deps = [":tfrt_fallback_td_files"], +) + +td_library( + name = "gpu_ops_td_file", + srcs = [ + "gpu_ops.td", + ], + includes = ["."], + visibility = [ + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + deps = [ + ":tfrt_fallback_td_files", + "@tf_runtime//:OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "tfrt_gpu_opdefs_inc_gen", + tbl_outs = { + "gpu_ops.h.inc": ["-gen-op-decls"], + "gpu_ops.cpp.inc": ["-gen-op-defs"], + "gpurt_dialect.h.inc": [ + "-gen-dialect-decls", + "-dialect=gpurt", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "gpu_ops.td", + test = True, + visibility = [ + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + deps = [":gpu_ops_td_file"], +) + +cc_library( + name = "tfrt_gpu_opdefs", + srcs = [ + "gpu_ops.cc", + ], + hdrs = ["gpu_ops.h"], + visibility = [ + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + deps = [ + ":tfrt_fallback_opdefs", + ":tfrt_gpu_opdefs_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..0af589f3a34e6f7d93117110f2ab50e841e27400 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.cc @@ -0,0 +1,41 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h" + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" + +namespace tfrt { +namespace gpu { + +GpuRuntimeDialect::GpuRuntimeDialect(MLIRContext *context) + : Dialect(/*name=*/"gpurt", context, TypeID::get()) { + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.cpp.inc" + >(); +} + +} // namespace gpu +} // namespace tfrt + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..9e15851e4045be10241a171734ffc519f5d6bae6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h @@ -0,0 +1,41 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_GPU_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_GPU_OPS_H_ + +#include "mlir/Bytecode/BytecodeOpInterface.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project + +using namespace mlir; // NOLINT + +namespace tfrt { +namespace gpu { + +// Dialect for TFRT GPU operations. +class GpuRuntimeDialect : public Dialect { + public: + explicit GpuRuntimeDialect(MLIRContext *context); + static StringRef getDialectNamespace() { return "gpurt"; } +}; + +} // namespace gpu +} // namespace tfrt + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_GPU_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..75bfa094f3bfb89da2b6173f7b762de30ae6aa5d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/gpu_ops.td @@ -0,0 +1,114 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifdef TFRT_GPU_OPS +#else +#define TFRT_GPU_OPS + +include "tfrt/tfrt_op_base.td" +include "tfrt/compiler/opdefs/tfrt_op_interfaces.td" +include "tfrt/compiler/opdefs/tfrt_traits.td" +include "tfrt_fallback.td" + +def TFRT_GPU_Dialect : Dialect { + let name = "gpurt"; + + let description = [{ + The TFRT GPU Dialect. + }]; + + let cppNamespace = "::tfrt::gpu"; +} + +class Gpu_Op traits = []> : + Op { +} + +// TODO(b/260267885): We may add a device argument when we want to support +// GPU MIG. +def TransferToDeviceOp: Gpu_Op<"transfer_to_device"> { + let summary = "Transfer a CPU tensor to device."; + + let description = [{ + Transfer a CPU tensor to device. + + Example: + %device_tensor = gpurt.transfer_to_device %cpu_tensor + }]; + + let arguments = (ins TFTensorType); + let results = (outs TFTensorType); + let assemblyFormat = "operands attr-dict"; +} + +// TODO(b/260267885): We may add a device argument when we want to support +// GPU MIG. +def TransferFromDeviceOp: Gpu_Op<"transfer_from_device"> { + let summary = "Transfer a tensor from device."; + + let description = [{ + Transfer a tensor from device. + + Example: + %cpu_tensor = gpurt.transfer_from_device %device_tensor + }]; + + let arguments = (ins TFTensorType); + let results = (outs TFTensorType); + let assemblyFormat = "operands attr-dict"; +} + +// TODO(b/260267885): We may add a device argument when we want to support +// GPU MIG. +def MaybeTransferVariableOp: Gpu_Op<"maybe_transfer_variable"> { + let summary = "Transfer a CPU variable tensor to device."; + let description = [{ + Transfer a CPU variable tensor to device if the variable has not been + transferred before. + + Example: + %device_var = gpurt.maybe_transfer_variable %cpu_var + }]; + + let arguments = (ins TFTensorType); + let results = (outs TFTensorType); + let assemblyFormat = "operands attr-dict"; +} + +def CompileAndExecuteOp: Gpu_Op<"compile_and_execute"> { + let summary = "GPU compile and execute operation."; + let description = [{ + The op compiles and executes a GPU cluster function. + + func_name is the name of the function to be executed on GPU. + resource_indices are the indices of inputs that are resources. + used_output_indices are the indices of outputs that have users. + + Example: + %results = gpurt.compile_and_execute {func_name = "xla_func_0", resource_indices = [1] ...} + }]; + + let arguments = (ins + Variadic:$operands, + StrAttr:$func_name, + I64ArrayAttr:$resource_indices, + I64ArrayAttr:$used_output_indices + ); + let results = (outs + Variadic:$results + ); +} + +#endif // TFRT_GPU_OPS diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..6bda8945e349054bc45106c606d0e7cfba4bd563 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/BUILD @@ -0,0 +1,157 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +td_library( + name = "mlrt_td_files", + srcs = [ + "mlrt_dialect.td", + "mlrt_ops.td", + ], + includes = ["."], + visibility = ["//visibility:private"], + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "mlrt_ops_inc_gen", + tbl_outs = { + "mlrt_ops.h.inc": ["-gen-op-decls"], + "mlrt_ops.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "mlrt_ops.td", + deps = [":mlrt_td_files"], +) + +cc_library( + name = "mlrt_ops", + srcs = [ + "mlrt_dialect.cc", + "mlrt_ops.cc", + ], + hdrs = [ + "mlrt_dialect.h", + "mlrt_ops.h", + ], + visibility = ["//tensorflow/compiler/mlir/tfrt:__subpackages__"], + deps = [ + ":mlrt_ops_inc_gen", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:Support", + ], +) + +td_library( + name = "tf_mlrt_td_files", + srcs = [ + "tf_mlrt_dialect.td", + "tf_mlrt_ops.td", + "tf_ops.td", + ], + includes = ["."], + deps = [ + ":mlrt_td_files", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@tf_runtime//:compiler_td_files", + ], +) + +td_library( + name = "tf_mlrt_tpu_td_files", + srcs = [ + "tf_mlrt_tpu_ops.td", + ], + includes = ["."], + visibility = ["//visibility:private"], + deps = [ + ":mlrt_td_files", + ":tf_mlrt_td_files", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_td_files", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "tf_mlrt_ops_inc_gen", + tbl_outs = { + "tf_mlrt_ops.h.inc": ["-gen-op-decls"], + "tf_mlrt_ops.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_mlrt_ops.td", + deps = [":tf_mlrt_td_files"], +) + +gentbl_cc_library( + name = "tf_mlrt_tpu_ops_inc_gen", + tbl_outs = { + "tf_mlrt_tpu_ops.h.inc": ["-gen-op-decls"], + "tf_mlrt_tpu_ops.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_mlrt_tpu_ops.td", + deps = [":tf_mlrt_tpu_td_files"], +) + +gentbl_cc_library( + name = "tf_ops_inc_gen", + tbl_outs = { + "tf_ops.h.inc": ["-gen-op-decls"], + "tf_ops.cpp.inc": ["-gen-op-defs"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tf_ops.td", + deps = [":tf_mlrt_td_files"], +) + +cc_library( + name = "tf_mlrt_ops", + srcs = ["tf_mlrt_ops.cc"], + hdrs = ["tf_mlrt_ops.h"], + visibility = [ + # copybara:uncomment "//learning/brain/experimental/tfrt/mlrt/application/tensorflow/tests:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + deps = [ + ":mlrt_ops", + ":tf_mlrt_ops_inc_gen", + ":tf_ops_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_side_effects", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tf_runtime//:compiler_tfrt_op_interfaces", + "@tf_runtime//:compiler_tfrt_traits", + ], +) + +cc_library( + name = "tf_mlrt_tpu_ops", + srcs = ["tf_mlrt_tpu_ops.cc"], + hdrs = ["tf_mlrt_tpu_ops.h"], + visibility = ["//tensorflow/compiler/mlir/tfrt/transforms/mlrt:__subpackages__"], + deps = [ + ":mlrt_ops", + ":tf_mlrt_ops", + ":tf_mlrt_tpu_ops_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.cc new file mode 100644 index 0000000000000000000000000000000000000000..1eddd082b12717429005e054428bb71fdc0d32fc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.cc @@ -0,0 +1,96 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h" + +namespace mlrt { +namespace compiler { + +namespace { + +struct MlrtInlinerInterface : public mlir::DialectInlinerInterface { + using DialectInlinerInterface::DialectInlinerInterface; + bool isLegalToInline(mlir::Operation *op, mlir::Region *dest, + bool would_be_cloned, + mlir::IRMapping &mapping) const final { + // All mlrt dialect ops can be inlined. + return true; + } +}; + +} // namespace + +MlrtDialect::MlrtDialect(mlir::MLIRContext *context) + : mlir::Dialect(/*name=*/"mlrt", context, + mlir::TypeID::get()) { + addTypes(); + addTypes(); + addTypes(); + addInterfaces(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.cpp.inc" + >(); +} + +// Parse a type registered to this dialect. +mlir::Type MlrtDialect::parseType(mlir::DialectAsmParser &parser) const { + llvm::StringRef keyword; + if (parser.parseKeyword(&keyword)) return mlir::Type(); + + if (keyword == "future") return FutureType::get(getContext()); + if (keyword == "promise") return PromiseType::get(getContext()); + if (keyword == "async_handle") return AsyncHandleType::get(getContext()); + + parser.emitError(parser.getNameLoc(), "unknown type: ") << keyword; + return mlir::Type(); +} + +// Print a type registered to this dialect. +void MlrtDialect::printType(mlir::Type type, + mlir::DialectAsmPrinter &os) const { + if (mlir::isa(type)) { + os << "future"; + return; + } + + if (mlir::isa(type)) { + os << "promise"; + return; + } + + if (mlir::isa(type)) { + os << "async_handle"; + return; + } + + llvm_unreachable("unexpected mlrt type kind"); +} + +} // namespace compiler +} // namespace mlrt diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h new file mode 100644 index 0000000000000000000000000000000000000000..3057553dc23c1a8f0153303e909c15e4f9310db3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h @@ -0,0 +1,62 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_DIALECT_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_DIALECT_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project + +namespace mlrt { +namespace compiler { + +class MlrtDialect : public mlir::Dialect { + public: + explicit MlrtDialect(mlir::MLIRContext *context); + static llvm::StringRef getDialectNamespace() { return "mlrt"; } + + mlir::Type parseType(mlir::DialectAsmParser &parser) const override; + void printType(mlir::Type type, mlir::DialectAsmPrinter &os) const override; +}; + +// The MLIR type represents a C++ mlrt::Future. +class FutureType + : public mlir::Type::TypeBase { + public: + using Base::Base; + static constexpr mlir::StringLiteral name = "mlrt.compiler.future"; +}; + +// The MLIR type represents a C++ mlrt::Promise. +class PromiseType + : public mlir::Type::TypeBase { + public: + using Base::Base; + static constexpr mlir::StringLiteral name = "mlrt.compiler.promise"; +}; + +// The MLIR type represents a C++ mlrt::AsyncHandle. +class AsyncHandleType : public mlir::Type::TypeBase { + public: + using Base::Base; + static constexpr mlir::StringLiteral name = "mlrt.compiler.async_handle"; +}; + +} // namespace compiler +} // namespace mlrt + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_DIALECT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td new file mode 100644 index 0000000000000000000000000000000000000000..6d083e01ebce1368cadf17cd2104cf8e94997b88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td @@ -0,0 +1,55 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef MLRT_DIALECT +#else +#define MLRT_DIALECT + +include "mlir/IR/OpBase.td" + +def Mlrt_Dialect : Dialect { + let name = "mlrt"; + + let description = [{ + The MLRT Dialect. + }]; + + let cppNamespace = "::mlrt::compiler"; +} + +def MlrtFutureType : DialectType($_self)">, "!mlrt.future type">, + BuildableType<"$_builder.getType<::mlrt::compiler::FutureType>()"> { + let description = [{ + `!mlrt.future type` represents a C++ mlrt::Future. + }]; +} + +def MlrtPromiseType : DialectType($_self)">, "!mlrt.promise type">, + BuildableType<"$_builder.getType<::mlrt::compiler::PromiseType>()"> { + let description = [{ + `!mlrt.promise type` represents a C++ mlrt::Promise. + }]; +} + +def MlrtAsyncHandleType : DialectType($_self)">, "!mlrt.async_handle type">, + BuildableType<"$_builder.getType<::mlrt::compiler::AsyncHandleType>()"> { + let description = [{ + `!mlrt.async_handle type` represents a C++ mlrt::AsyncHandle. + }]; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..59d0a17518a59c8be1a97041348934aae26b7f3b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.cc @@ -0,0 +1,28 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h" + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..42ab78ed8455f1570b57ed0e2cd8f36e10291224 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h @@ -0,0 +1,27 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_OPS_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_MLRT_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..ead776ea7c923307963901dc86d80bf1479d60ec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.td @@ -0,0 +1,240 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef MLRT_OPS +#else +#define MLRT_OPS + +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td" + +class Mlrt_Op traits = []> : + Op { +} + +def CondOp: Mlrt_Op<"cond", []> { + let summary = "mlrt.cond op"; + + let description = [{ + Execute $a_true_fn with $args if $cond is true; otherwise, %b_false_fn is + executed. + }]; + + let arguments = (ins + I1:$cond, + Variadic:$args, + SymbolRefAttr:$a_true_fn, + SymbolRefAttr:$b_false_fn + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = [{ + $cond $a_true_fn $b_false_fn `(` $args `)` attr-dict `:` `(` type($args) `)` `->` `(` type($results) `)` + }]; +} + +def AsyncOp: Mlrt_Op<"async", []> { + let summary = "Launches a function asynchronously."; + + let description = [{ + Launch a function asynchronously. + + $args: a list of arguments to be passed. + $callee: The function to be launched. Its return op must not have operands. + + $handle: This op returns a handle object that manages the context of the async execution. + }]; + + let arguments = (ins + Variadic:$args, + SymbolRefAttr:$callee + ); + + let results = (outs + MlrtAsyncHandleType:$handle + ); + + let assemblyFormat = "`(` $args `)` attr-dict `:` functional-type($args, $handle)"; +} + +def AwaitHandleOp: Mlrt_Op<"await_handle", []> { + let summary = "Awaits an async execution "; + + let description = [{ + Awaits an async execution. + + $handle: The handle returned by mlrt.async op. + }]; + + let arguments = (ins + MlrtAsyncHandleType:$handle + ); + + let assemblyFormat = "operands attr-dict"; +} + +def AwaitAllHandleOp: Mlrt_Op<"await_all_handle", []> { + let summary = "Awaits multiple async executions"; + + let description = [{ + Awaits multiple async execution. + + $handles: A list of handles returned by mlrt.async ops. + }]; + + let arguments = (ins + Variadic:$handles + ); + + let assemblyFormat = "operands attr-dict `:` type($handles)"; +} + +def AwaitControlOp: Mlrt_Op<"await_control", []> { + let summary = "Await a signal from a future"; + + let description = [{ + Await a signal, instead of a value, from a future. + + $future: A value of !mlrt.future type. + }]; + + let arguments = (ins + MlrtFutureType:$future + ); + + let assemblyFormat = "operands attr-dict"; +} + +def AwaitAllControlOp: Mlrt_Op<"await_all_control", []> { + let summary = "Awaits multiple signals"; + + let description = [{ + Awaits multiple signals + + $futures: A list of !mlrt.futures + }]; + + let arguments = (ins + Variadic:$futures + ); + + let assemblyFormat = "operands attr-dict `:` type($futures)"; +} + +def PromiseControlOp: Mlrt_Op<"promise_control", []> { + let summary = "Set a control promise"; + + let description = [{ + Set a control promise. + + $promise: A value of !mlrt.promise type. + }]; + + let arguments = (ins + MlrtPromiseType:$promise + ); + + let assemblyFormat = "operands attr-dict"; +} + +def CaseOp : Mlrt_Op<"case"> { + let summary = "An n-way switch statement which calls a single branch function."; + let description = [{ + An n-way switch statement, implementing the following: + ``` + switch (branch_index) { + case 0: + outputs = branches[0](inputs); + break; + case 1: + outputs = branches[1](inputs); + break; + ... + case [[nbranches-1]]: + default: + outputs = branches[nbranches-1](inputs); + break; + } + ``` + Example: %res = mlrt.case %branch_idx [@branch0, @branch1] (%arg0, %arg1) : (i32, i32) -> (i32) + }]; + + let arguments = (ins I32:$branch_index, + ConfinedAttr]>:$branches, + Variadic:$branch_operands); + + let results = (outs Variadic:$branch_outputs); + let assemblyFormat = [{ + $branch_index $branches `(` $branch_operands `)` attr-dict `:` `(` type($branch_operands) `)` `->` `(` type($branch_outputs) `)` + }]; +} + +def AllocateControlFuturesOp: Mlrt_Op<"allocate_control_futures", [AttrSizedResultSegments]> { + let summary = "Allocate futures and corresponding promises"; + + let description = [{ + Allocate futures and corresponding promises. + + $num: The number of futures to be allocated. + + $promises: There are $num promises, and promises[i] shares the state with futures[i]. + $futures: There are $num futures, and futures[i] shares the state with promises[i]. + }]; + + let arguments = (ins + I32Attr:$num + ); + + let results = (outs + Variadic:$promises, + Variadic:$futures + ); +} + +def WhileOp : Mlrt_Op<"while", []> { + let summary = "while operation"; + let description = [{ + cond: The boolean to control whether the first iteration should be + executed. + operands: The arguments to the first iteration. + results: The results of the last iteration. The number and types of results + excluding the last one are the same as the number and types of operands. The + last element of results is an I1 value that is false. + body_fn: The body function that takes the arguments and returns the results + that includes an I1 value to indicate whether next iteration should be executed. + + The pseudo code: + + while(cond) { + results = body_fn(operands) + cond = results#1 + } + return results + + }]; + + let arguments = (ins I1:$cond, + Variadic:$arguments, + FlatSymbolRefAttr:$body_fn); + + let results = (outs Variadic); + + let assemblyFormat = [{ + $cond $body_fn `(` $arguments `)` attr-dict `:` `(` type($arguments) `)` `->` `(` type(results) `)` + }]; +} +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td new file mode 100644 index 0000000000000000000000000000000000000000..7fc6322c47584dfa8a97d9d9478a21dcfa3242c7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td @@ -0,0 +1,56 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef TF_MLRT_DIALECT +#else +#define TF_MLRT_DIALECT + +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td" + +// TODO(chky,rohitju): Unify this dialect with tfrt_fallback_sync dialect after +// vrooml is using the new interpreter. +def TensorflowMlrt_Dialect : Dialect { + let name = "tf_mlrt"; + + let description = [{ + The TF MLRT Dialect. + }]; + + let cppNamespace = "::tensorflow::tf_mlrt"; +} + +class TensorflowMlrt_Op traits = []> : + Op { +} + +// This corresponds to tensorflow::Tensor. +def TFTensorType : DialectType($_self)">, "!tf_mlrt.tensor type">, + BuildableType<"$_builder.getType<::tensorflow::tf_mlrt::TFTensorType>()"> { + let description = [{ + `!tf_mlrt.tensor type` represents a tensorflow::Tensor. + }]; +} + +// This corresponds to tensorflow::Device* . +def TFDeviceType : DialectType($_self)">, "!tf_mlrt.device type">, + BuildableType<"$_builder.getType<::tensorflow::tf_mlrt::TFDeviceType>()"> { + let description = [{ + `!tf_mlrt.device type` represents a tensorflow::device. + }]; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..a93ca6482acfc23d18779371c5ed79ee3bf6c783 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.cc @@ -0,0 +1,96 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" + +namespace tensorflow { +namespace tf_mlrt { + +namespace { + +struct TensorflowMlrtInlinerInterface : public mlir::DialectInlinerInterface { + using DialectInlinerInterface::DialectInlinerInterface; + bool isLegalToInline(mlir::Operation *op, mlir::Region *dest, + bool would_be_cloned, + mlir::IRMapping &mapping) const final { + // All tf_mlrt dialect ops can be inlined. + return true; + } + // Note that CallOp and ReturnOp are handled by func; so need to implement + // handleTerminator. +}; + +} // namespace + +TensorflowMlrtDialect::TensorflowMlrtDialect(mlir::MLIRContext *context) + : mlir::Dialect(/*name=*/"tf_mlrt", context, + mlir::TypeID::get()) { + addTypes(); + addInterfaces(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.cpp.inc" + >(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.cpp.inc" + >(); +} + +// Parse a type registered to this dialect. +mlir::Type TensorflowMlrtDialect::parseType( + mlir::DialectAsmParser &parser) const { + llvm::StringRef keyword; + if (parser.parseKeyword(&keyword)) return mlir::Type(); + + if (keyword == "tensor") return TFTensorType::get(getContext()); + + parser.emitError(parser.getNameLoc(), "unknown type: ") << keyword; + return mlir::Type(); +} + +// Print a type registered to this dialect. +void TensorflowMlrtDialect::printType(mlir::Type type, + mlir::DialectAsmPrinter &os) const { + if (mlir::isa(type)) { + os << "tensor"; + return; + } + + llvm_unreachable("unexpected tf_mlrt type kind"); +} + +} // namespace tf_mlrt +} // namespace tensorflow + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.cpp.inc" +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..e97d5adce0755859aa2fc532883e32dafbe88991 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h @@ -0,0 +1,63 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_OPS_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_side_effects.h" +#include "tfrt/compiler/opdefs/tfrt_op_interfaces.h" // from @tf_runtime +#include "tfrt/compiler/opdefs/tfrt_traits.h" // from @tf_runtime + +namespace tensorflow { +namespace tf_mlrt { + +class TensorflowMlrtDialect : public mlir::Dialect { + public: + explicit TensorflowMlrtDialect(mlir::MLIRContext *context); + static llvm::StringRef getDialectNamespace() { return "tf_mlrt"; } + + mlir::Type parseType(mlir::DialectAsmParser &parser) const override; + void printType(mlir::Type type, mlir::DialectAsmPrinter &os) const override; +}; + +// The MLIR type represents a tensorflow::Tensor. +class TFTensorType + : public mlir::Type::TypeBase { + public: + using Base::Base; + static constexpr mlir::StringLiteral name = "tensorflow.tf_mlrt.tf_tensor"; +}; + +// The MLIR type represents a tensorflow::Device* +class TFDeviceType + : public mlir::Type::TypeBase { + public: + using Base::Base; + static constexpr mlir::StringLiteral name = "tensorflow.tf_mlirt.tf_device"; +}; + +} // namespace tf_mlrt +} // namespace tensorflow + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h.inc" +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..f3afb395360d23255df4b8a6b3c619080e338979 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.td @@ -0,0 +1,577 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef TF_MLRT_OPS +#else +#define TF_MLRT_OPS + +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td" + +def CreateOp: TensorflowMlrt_Op<"createop", []> { + let summary = "The Fallback CreateOp"; + + let description = [{ + The CreateOp creates the tensorflow::OpKernel in the fallback context. + }]; + + let arguments = (ins + StrAttr:$node_def, + I32Attr:$op_key + ); + + let assemblyFormat = "attr-dict"; +} + +def ConstOp: TensorflowMlrt_Op<"constop", []> { + let summary = "The tf_mlrt ConstOp"; + + let description = [{ + The ConstOp creates a constant tensorflow::Tensor from serialized proto. + }]; + + let arguments = (ins + StrAttr:$tensor_proto + ); + + let results = (outs + TFTensorType:$result + ); + + let assemblyFormat = "attr-dict"; +} + +def ExecuteOp : TensorflowMlrt_Op<"executeop", []> { + let summary = "The Fallback ExecuteOp"; + let description = [{ + The ExecuteOp executes an operation on the specified device. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +def ExecuteOpWithDevice: TensorflowMlrt_Op<"executeop.device", []> { + let summary = "The Fallback ExecuteOp with custom device"; + let description = [{ + The ExecuteOp executes an operation on the specified device using a custom device. + }]; + + let arguments = (ins + TFDeviceType:$device, + Variadic:$args, + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $device`)` `(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +def AsyncExecuteOp : TensorflowMlrt_Op<"async_executeop", []> { + let summary = "The Fallback ExecuteOp for tensorflow::AsyncOpKernel"; + let description = [{ + The ExecuteOp executes an operation on the specified device asynchronously. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +def AsyncExecuteOpWithDevice : TensorflowMlrt_Op<"async_executeop.device", []> { + let summary = "The Fallback ExecuteOp for tensorflow::AsyncOpKernel"; + let description = [{ + The ExecuteOp executes an operation on the specified device asynchronously. + }]; + + let arguments = (ins + TFDeviceType:$device, + Variadic:$args, + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $device`)` `(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +def SetResourceOp : TensorflowMlrt_Op<"set_resource", []> { + let summary = "Set a tensor in resource array"; + + let description = [{ + Set a tensor in resource array. + + arg: the tensor to be set in the resource array. + index: the index in the resource array + }]; + + let arguments = (ins + TFTensorType:$arg, + I64Attr:$index + ); + + let results = (outs); + + let assemblyFormat = "operands attr-dict"; +} + +def GetResourceOp : TensorflowMlrt_Op<"get_resource", []> { + let summary = "get a tensor in resource array"; + + let description = [{ + Get a tensor in resource array. + + indices: the indices in the resource array. + results: the tensor values for the corresponding indices. + }]; + + let arguments = (ins + I64ArrayAttr:$indices + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "attr-dict `:` type($results)"; +} + +def AwaitOp: TensorflowMlrt_Op<"await", [Pure]> { + let summary = "Await a tensor from a !mlrt.future"; + + let description = [{ + Await a tensor from a !mlrt.future. + + $future: A value of type !mlrt.future. The underlying value must be a tensorflow tensor. + + $result: a tensorflow tensor. + }]; + + let arguments = (ins + MlrtFutureType:$future + ); + + let results = (outs + TFTensorType:$result + ); + + let assemblyFormat = "operands attr-dict"; +} + +def AwaitAllOp: TensorflowMlrt_Op<"await_all", [Pure]> { + let summary = "Await tensors from a list of !mlrt.future"; + + let description = [{ + Await tensors from a list of !mlrt.future. + + $futures: A list of !mlrt.future. The underlying value must be tensorflow tensors. + + $results: A list of tensorflow tensors. + }]; + + let arguments = (ins + Variadic:$futures + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "$futures attr-dict `:` type($results)"; +} + +def PromiseOp: TensorflowMlrt_Op<"promise", []> { + let summary = "Set a tensor in a promise"; + + let description = [{ + Set a tensor in a promise. + + $promise: A value of type !mlrt.promise. The underlying value must be a tensorflow tensor. + $tensor: A tensorflow tensor. + }]; + + let arguments = (ins + MlrtPromiseType:$promise, + TFTensorType:$tensor + ); + + let assemblyFormat = "operands attr-dict"; +} + +def PromiseFutureOp: TensorflowMlrt_Op<"promise_future", []> { + let summary = "Set a tensor future in a promise"; + + let description = [{ + Set a tensor future in a promise. + + $promise: A value of type !mlrt.promise. The underlying value must be a tensorflow tensor. + $future: A value of type !mlrt.future. Must represents a tensorflow tensor. + }]; + + let arguments = (ins + MlrtPromiseType:$promise, + MlrtFutureType:$tensor + ); + + let assemblyFormat = "operands attr-dict"; +} + +def PromiseReturnOp: TensorflowMlrt_Op<"promise_return", [Terminator]> { + let summary = "Set a tensor in a promise and return from a function with no output"; + + let description = [{ + Set a tensor in a promise and return from a function with no output. + + $promise: A value of type !mlrt.promise. The underlying value must be a tensorflow tensor. + $tensor: A tensorflow tensor. + }]; + + let arguments = (ins + MlrtPromiseType:$promise, + TFTensorType:$tensor + ); + + let assemblyFormat = "operands attr-dict"; +} + +def AllocateFuturesOp: TensorflowMlrt_Op<"allocate_futures", [AttrSizedResultSegments]> { + let summary = "Allocate futures and promsies for tensorflow tensors"; + + let description = [{ + Allocate futures and promsies for tensorflow tensors. + + $num_futures: The number of futures to be allocated. + + $promises: There are $num_futures promises. promises[i] shares the state with futures[i]. + $futures: There are $num_futures futures. futures[i] shares the state with promises[i]. + }]; + + let arguments = (ins + I32Attr:$num_futures + ); + + let results = (outs + Variadic:$promises, + Variadic:$futures + ); +} + +def TensorToIntOp : TensorflowMlrt_Op<"tensor_to_int32", [Pure]> { + let summary = "Cast a Tensor to int32."; + let description = [{ + Cast a Tensor to int32. + + Example: + %one = tf_mlrt.tensor_to_int32 %src_tenosr + }]; + + let arguments = (ins TFTensorType:$src); + let results = (outs I32:$result); + let assemblyFormat = "operands attr-dict"; +} + +def BoolToTensorOp : TensorflowMlrt_Op<"bool_to_tensor", [Pure]> { + let summary = "Cast a boolean to a tensor."; + let description = [{ + Cast a boolean to a tensor. + + Example: + %one = tf_mlrt.bool_to_tensor %src_bool + }]; + + let arguments = (ins I1: $src); + let results = (outs TFTensorType:$result); + + let assemblyFormat = "operands attr-dict"; +} + +def PredicateOp : TensorflowMlrt_Op<"predicate", [Pure]> { + let summary = "Converts a fallback tensor to a bool"; + + let description = [{ + Note: this kernel is used for CPU tensors. + + Converts a fallback tensor to a bool with the following rules: + + - For 0D tensors, truthiness is determined by comparing against a "zero" + value. For numerical types it is the obvious zero. For strings it is the + empty string. + + - For >0D tensors, truthiness is determined by looking at the number of + elements. If has zero elements, then the result is false. Otherwise the + result is true. + + input: a fallback tensor representing the condition. + device: the name of the tensorflow device that is associated with the + input fallback tensor. + + output: the converted bool. + }]; + + let arguments = (ins + TFTensorType:$input + ); + + let results = (outs + I1:$output + ); + + let assemblyFormat = "$input attr-dict"; +} + +def BatchFunctionOp : TensorflowMlrt_Op<"batch_function", [Pure]> { + let summary = "Fallback ExecuteOp specialized for tf.BatchFunction."; + + let description = [{ + This kernel executes a variant tf.BatchFunction kernel that supports having + the `f` attribute as a bytecode function. + + Example: + %res = tf_mlrt.batch_function(%input, %captured_input) { + device = "/device:CPU:0", + f = @batch_function, + node_def = "..." + } : (!tf_mlrt.tensor,!tf_mlrt.tensor) -> (!tf_mlrt.tensor) + + Note that the trailing number indicates the number of results. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$device, + SymbolRefAttr:$f, + StrAttr:$node_def + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +def BatchFunctionWithDeviceOp : TensorflowMlrt_Op<"batch_function.device", [Pure]> { + let summary = "Fallback ExecuteOp specialized for tf.BatchFunction with custom device."; + + let description = [{ + This kernel executes a variant tf.BatchFunction kernel that supports having + the `f` attribute as a bytecode function with a custom device. + + Example: + %res = tf_mlrt.batch_function.device(%custom_device, %input, %captured_input) { + device = "/device:CPU:0", + f = @batch_function, + node_def = "..." + } : (!tf_mlrt.tensor,!tf_mlrt.tensor) -> (!tf_mlrt.tensor) + + Note that the trailing number indicates the number of results. + Also note that the custom device argument is not the same as the device attribute. + The latter is just the name in which the enclosure graph is executed. + }]; + + let arguments = (ins + TFDeviceType:$custom_device, + Variadic:$args, + StrAttr:$device, + SymbolRefAttr:$f, + StrAttr:$node_def + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = " `(` $custom_device`)` `(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + + +def CancelOp: TensorflowMlrt_Op<"cancel", []> { + let summary = "Handle cancellation request."; + + let description = [{ + This kernel will early terminate the program upon cancellation request (e.g. time out). + }]; +} + +def MapFnOp : TensorflowMlrt_Op<"map_fn", [AttrSizedOperandSegments, Pure]> { + let summary = "The Parallel Map for tf_mlrt dialect"; + let description = [{ + The Pmap executes body function in parallel for all ranges up to $max_iterations. + + The pseudo code: + for(int i = 0; i < $max_iterations; i++) { + body_fn(MlrtFture($tensor_list_or_flow_in[i]), + MlrtPromise($tensor_list_or_flow_in[i+1]), + i, i, $invariant_args); + } + + return $tensor_list_or_flow_in[$max_iterations] + }]; + + let arguments = (ins + TFTensorType:$max_iterations, + Variadic:$tensor_list_or_flow_in, + Variadic:$invariant_args, + FlatSymbolRefAttr:$body_fn, + I32Attr:$num_tensor_list_or_flow_in + ); + + let results = (outs + Variadic:$result + ); + + let assemblyFormat = "`(`$max_iterations`,` $tensor_list_or_flow_in`,` $invariant_args `)` attr-dict `:` functional-type(operands, results)"; +} + +def AsyncWhileOp : TensorflowMlrt_Op<"async_while", [Pure]> { + let summary = "Asynchronously execution of while op for tf_mlrt"; + let description = [{ + cond: The boolean to control whether the first iteration should be executed. + arguments: the last $immutable_size elements are invariants between iterations. + results: a list of futures. + body_fn: its input are [predicate_promise, arg0_future, arg0_promise, arg1_future, arg1_promise, ..., invariant_args] and has no returned results. + }]; + + let arguments = (ins TFTensorType:$cond, + Variadic:$arguments, + FlatSymbolRefAttr:$body_fn, + I32Attr:$invariant_size); + + let results = (outs Variadic); + + let assemblyFormat = [{ + $body_fn `(` $cond `,` $arguments `)` attr-dict `:` `(` type($cond) `,` type($arguments) `)` `->` `(` type(results) `)` + }]; +} + +def IfrtLoadVariableOp: TensorflowMlrt_Op<"ifrt_load_variable", [Pure]> { + let summary = "Loads a variable tensor as an IFRT array for mlrt"; + + let description = [{ + This op loads a restored variable tensor as a tensor future. It is a + replacement of `tf.ReadVariableOp`. + + This op returns a scalar string tensor containing the restored variable name, which can be + used as a key within the runtime, as well as a future for the tensor. + + The `tf.IfrtCall` kernel uses the output $array_key. + Other ops executed by TFRT may make use of $tensor_future. + }]; + + let arguments = (ins + TFTensorType:$variable, + DefaultValuedAttr:$used_by_host + ); + + let results = (outs + TFTensorType:$array_key, + MlrtFutureType: $tensor_future + ); +} + +def IfrtRestoreVariableOp: TensorflowMlrt_Op<"ifrt_restore_variable", []> { + let summary = "Restore variable tensors"; + let description = [{ + This is the MLRT version of tf.IfrtRestoreVariableOp. + + This op restores tensors from a checkpoint asynchronously, similar to tf.RestoreV2. + It fuses variable assignment: tensors that are consumed by tf.AssignVariableOp + are handled internally via the $var_handles operands and do not appear in + the results. + + Tensors that are consumed by other operations (e.g. tf.LookupTableImportV2) + will be returned as results in $tensors. Consequently, the number of + results is dynamic, containing only those tensors not assigned to variables + but consumed directly by other operations. A restored tensor with no + consumer will cause an error in the kernel. + This Op is specific to MLRT runtime and is not a stable interface for + serialization. + + }]; + + let arguments = (ins + TFTensorType:$prefix, + TFTensorType:$tensor_names, + TFTensorType:$shape_and_slices, + Variadic:$var_handles, + TypeArrayAttr: $restored_dtypes, + DenseBoolArrayAttr: $truncate_in_cast, + // The names of the returned tensors. The order of names corresponds to the + // order of the returned tensors. + StrArrayAttr: $returned_tensor_names + ); + + let results = (outs + Res, [{The restored tensor futures as requested by the $returned_Tensor_names attribute.}]>:$tensors + ); +} + +def MlrtIfrtResourceDeserializeOp: TensorflowMlrt_Op<"ifrt_resource_deserialize", []> { + let summary = "Deserialize resource vars."; + let description = [{ + This is the MLRT version of the IfrtResourceDeserialize op. + }]; + + let arguments = (ins + TFTensorType:$resource_var, + TFTensorType:$input_dir, + StrAttr:$tensor_name + ); +} + +def AsyncIfrtCallOp : TensorflowMlrt_Op<"async_ifrt_call", []> { + let summary = "IFRT execution op for mlrt"; + let description = [{ + This op executes an IFRT program asynchronously. + }]; + + let arguments = (ins + I64Attr:$program_id, + I32ArrayAttr:$variable_arg_indices, + Variadic:$args + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "`(` $args `)` attr-dict `:` functional-type($args, $results)"; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..86478a7b3d43b1ee8e931a4c41de24e28d7221bd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.cc @@ -0,0 +1,46 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h" + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +namespace tensorflow { +namespace tf_mlrt_tpu { + +TensorflowMlrtTpuDialect::TensorflowMlrtTpuDialect(mlir::MLIRContext *context) + : mlir::Dialect(/*name=*/"tf_mlrt_tpu", context, + mlir::TypeID::get()) { + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.cpp.inc" + >(); +} + +} // namespace tf_mlrt_tpu +} // namespace tensorflow + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..bf6fa810b95c7737e7601b65a05601aad529a829 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h @@ -0,0 +1,39 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_TPU_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_TPU_OPS_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project + +namespace tensorflow { +namespace tf_mlrt_tpu { + +class TensorflowMlrtTpuDialect : public mlir::Dialect { + public: + explicit TensorflowMlrtTpuDialect(mlir::MLIRContext *context); + static llvm::StringRef getDialectNamespace() { return "tf_mlrt_tpu"; } +}; + +} // namespace tf_mlrt_tpu +} // namespace tensorflow + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_MLRT_TF_MLRT_TPU_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..c03d164b504b1fbbb16b4c834a8be25142475476 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.td @@ -0,0 +1,82 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef TF_MLRT_TPU_OPS +#else +#define TF_MLRT_TPU_OPS + +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td" + +def TensorflowMlrtTpu_Dialect : Dialect { + let name = "tf_mlrt_tpu"; + + let description = [{ + The TF MLRT TPU Dialect. + }]; + + let cppNamespace = "::tensorflow::tf_mlrt_tpu"; +} + +class TensorflowMlrtTpu_Op traits = []> : + Op { +} + +def GetTpuHostDeviceOp : TensorflowMlrtTpu_Op<"get_tpu_host_device", [Pure]> { + let summary = "get the tpu host allocator that implements tensorflow::Device"; + + let results = (outs + TFDeviceType:$device + ); + + let assemblyFormat = "attr-dict"; +} + +def CompileAndExecuteOp : TensorflowMlrtTpu_Op<"compile_and_execute"> { + let summary = "tpu compile and execute operation"; + let description = [{ + tf_mlrt_tpu.compile_and_execute compiles a mlir tpu program and executes the compiled tpu program. + + $mlir_module is a serialized MLIR module with a `main` function that contains target computation. + $metadata is a serialized TPUCompileMetadataProto describing the shapes and types of the inputs to the computation, as well as a mapping onto the TPU pod topology. + $constant_operand_indices are the indices of the inputs that are constant to the TPU program (e.g. weights in inference), the rest of the inputs are input tensors. + constant_operand_indices is sorted in ascending order. + $operands_with_static_shape are indices of operands that are tagged with a maximum static shape. + $producer_name is a string describing the name of the framework that added support for running this portion of the model on TPUs. + + Example: + %rendezvous_key_base, %result = tf_mlrt_tpu.compile_and_execute (%operands) constant_operand_indices = [1, 3] metadata = "metadata..." mlir_module = "mlir_module..." + }]; + let arguments = (ins + Variadic:$operands_and_static_shapes, + DenseI32ArrayAttr:$constant_operand_indices, + StrAttr:$metadata, + StrAttr:$mlir_module, + UI32Attr:$num_operands, + DenseI32ArrayAttr:$operands_with_static_shape, + StrAttr:$producer_name + ); + + let results = (outs + TFTensorType:$rendezvous_key_base, + Variadic:$results + ); + + let assemblyFormat = [{ + `(` $operands_and_static_shapes `)` attr-dict `:` functional-type($operands_and_static_shapes, results) + }]; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..3b7be2880baef072e03d4a6accd69c4f80739fe2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_ops.td @@ -0,0 +1,182 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifdef MLRT_TF_OPS +#else +#define MLRT_TF_OPS + +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_dialect.td" +include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td" +include "tfrt/compiler/opdefs/tfrt_op_interfaces.td" // from @tf_runtime +include "tfrt/compiler/opdefs/tfrt_traits.td" // from @tf_runtime + +// tf_mlrt.tf_await returns a tensorflow Tensor. It is a fake op that is only +// used during parallelization and has no runtime implementation. +def TFAwaitOp: TensorflowMlrt_Op<"tf_await", [Pure, TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Await a tensor from a !mlrt.future"; + + let description = [{ + Await a tensor from a !mlrt.future. + + $future: A value of type !mlrt.future. The underlying value must be a tensorflow tensor. + + $result: a tensorflow tensor. + }]; + + let arguments = (ins + MlrtFutureType:$future + ); + + let results = (outs + TF_Tensor:$result + ); +} + +// tf_mlrt.tf_promise takes a tensorflow Tensor. It is a fake op that is only +// used during parallelization and has no runtime implementation. +def TFPromiseOp: TensorflowMlrt_Op<"tf_promise", [TF_MustExecute, TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Set a tensor in a promise"; + + let description = [{ + Set a tensor in a promise. + + $promise: A value of type !mlrt.promise. The underlying value will always be a tensorflow tensor. + $tensor: A tensorflow tensor. + }]; + + let arguments = (ins + MlrtPromiseType:$promise, + TF_Tensor:$tensor + ); +} + +def TFMapFnOp : TensorflowMlrt_Op<"tf_map_fn", [AttrSizedOperandSegments, Pure]> { + let summary = "The Parallel Map for tf_mlrt dialect"; + let description = [{ + The Pmap executes body function in parallel for all ranges up to $max_iterations. + + The pseudo code: + for(int i = 0; i < $max_iterations; i++) { + body_fn(MlrtFture($tensor_list_or_flow_in[i]), + MlrtPromise($tensor_list_or_flow_in[i+1]), + i, i, $invariant_args); + } + + return $tensor_list_or_flow_in[$max_iterations] + }]; + + let arguments = (ins + TF_Tensor:$max_iterations, + Variadic:$tensor_list_or_flow_in, + Variadic:$invariant_args, + FlatSymbolRefAttr:$body_fn, + I32Attr:$num_tensor_list_or_flow_in + ); + + let results = (outs + Variadic:$result + ); + + let assemblyFormat = "`(`$max_iterations`,` $tensor_list_or_flow_in`,` $invariant_args `)` attr-dict `:` functional-type(operands, results)"; +} + +def TFAsyncWhileOp : TensorflowMlrt_Op<"tf_async_while", [Pure]> { + let summary = "Asynchronously execution of while op for tf_mlrt"; + let description = [{ + cond: The boolean to control whether the first iteration should be executed. + arguments: The last $invariant_size elements are invariants between iterations. + results: a list of futures. + body_fn: input is [predicate_promise, arg0_future, arg0_promise, arg1_future, arg1_promise, ..., invariant_args] and has no returned results. + }]; + + let arguments = (ins TF_Tensor:$cond, + Variadic:$arguments, + FlatSymbolRefAttr:$body_fn, + I32Attr:$invariant_size); + + let results = (outs Variadic); + + let assemblyFormat = [{ + $body_fn `(` $cond `,` $arguments `)` attr-dict `:` `(` type($cond) `,` type($arguments) `)` `->` `(` type(results) `)` + }]; +} + +def TFTPUCompileAndExecuteOp : TensorflowMlrt_Op<"tf_tpu_compile_and_execute", [TF_MustExecute]> { + let summary = "tpu compile and execute operation"; + let description = [{ + tf_mlrt_tpu.compile_and_execute compiles a mlir tpu program and executes the compiled tpu program. + + $mlir_module is a serialized MLIR module with a `main` function that contains target computation. + $metadata is a serialized TPUCompileMetadataProto describing the shapes and types of the inputs to the computation, as well as a mapping onto the TPU pod topology. + $constant_operand_indices are the indices of the inputs that are constant to the TPU program (e.g. weights in inference), the rest of the inputs are input tensors. + constant_operand_indices is sorted in ascending order. + $operands_with_static_shape are indices of operands that are tagged with a maximum static shape. + $producer_name is a string describing the name of the framework that added support for running this portion of the model on TPUs. + + Example: + %rendezvous_key_base, %result = tf_mlrt_tpu.compile_and_execute (%operands) constant_operand_indices = [1, 3] metadata = "metadata..." mlir_module = "mlir_module..." + }]; + let arguments = (ins + Variadic:$operands_and_static_shapes, + DenseI32ArrayAttr:$constant_operand_indices, + StrAttr:$metadata, + StrAttr:$mlir_module, + UI32Attr:$num_operands, + DenseI32ArrayAttr:$operands_with_static_shape, + StrAttr:$producer_name + ); + + let results = (outs + TF_Tensor:$rendezvous_key_base, + Variadic:$results + ); + + let assemblyFormat = [{ + `(` $operands_and_static_shapes `)` attr-dict `:` functional-type($operands_and_static_shapes, results) + }]; +} + +def TFIfrtLoadVariableOp: TensorflowMlrt_Op<"tf_ifrt_load_variable", [Pure]> { + let summary = "Loads a variable tensor as an IFRT array for mlrt"; + + let description = [{ + This is the MLRT version of tf.IfrtLoadVariableOp. + + This op loads a variable tensor as an IFRT array and binds it with the specified name. + + This op is an replacement of `tf.ReadVariableOp` in the case that a constant + variable tensor is an input to the tpu program invoked by `tf.IfrtCall`. + + After a `tf.ReadVariableOp` is lowered into `tf.IfrtLoadVariableOp`, the `tf.IfrtCall` kernel + will bind the loaded IFRT array by name with the tpu program's input. + + This op returns a scalar string tensor as a key for user to look for the loaded array + and a future containing the restored tensor. + }]; + + let arguments = (ins + TF_Tensor:$variable, + DefaultValuedAttr:$used_by_host + ); + + let results = (outs + TF_Tensor:$array_key, + MlrtFutureType: $tensor_future + ); +} + + + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.cc new file mode 100644 index 0000000000000000000000000000000000000000..bb1ad2ccac039912a58caf5eb6fa0cc0b393008b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.cc @@ -0,0 +1,72 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" + +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace tfrt { +namespace fallback { + +FallbackDialect::FallbackDialect(MLIRContext *context) + : Dialect(/*name=*/"tfrt_fallback", context, + TypeID::get()) { + addTypes(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.cpp.inc" + >(); +} + +/// Parse a type registered to this dialect. +Type FallbackDialect::parseType(DialectAsmParser &parser) const { + StringRef keyword; + if (parser.parseKeyword(&keyword)) return Type(); + + if (keyword == "tf_tensor") return TFTensorType::get(getContext()); + if (keyword == "tf_allocator") return TFAllocatorType::get(getContext()); + + parser.emitError(parser.getNameLoc(), "unknown type: ") << keyword; + return Type(); +} + +/// Print a type registered to this dialect. +void FallbackDialect::printType(Type type, DialectAsmPrinter &os) const { + if (mlir::isa(type)) { + os << "tf_tensor"; + return; + } + + if (mlir::isa(type)) { + os << "tf_allocator"; + return; + } + + llvm_unreachable("unexpected fallback type kind"); +} + +} // namespace fallback +} // namespace tfrt + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h new file mode 100644 index 0000000000000000000000000000000000000000..04e535b7da355bbb82c30bafd588ff77028ecbe1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h @@ -0,0 +1,60 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project + +using namespace mlir; // NOLINT + +namespace tfrt { +namespace fallback { + +// Dialect for fallback operations. +class FallbackDialect : public Dialect { + public: + explicit FallbackDialect(MLIRContext *context); + static StringRef getDialectNamespace() { return "tfrt_fallback"; } + + Type parseType(DialectAsmParser &parser) const override; + void printType(Type type, DialectAsmPrinter &os) const override; +}; + +// The MLIR type represents a tensorflow::Tensor. +class TFTensorType : public Type::TypeBase { + public: + using Base::Base; + static constexpr StringLiteral name = "tfrt.tf_tensor"; +}; + +// The MLIR type represents a tensorflow::Allocator. +class TFAllocatorType + : public Type::TypeBase { + public: + using Base::Base; + static constexpr StringLiteral name = "tfrt.tf_allocator"; +}; + +} // namespace fallback +} // namespace tfrt + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.td new file mode 100644 index 0000000000000000000000000000000000000000..544b2d2b11eeac0341477b2ccc4542d1005cf1c7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.td @@ -0,0 +1,50 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifdef TFRT_FALLBACK_OPS +#else +#define TFRT_FALLBACK_OPS + +include "mlir/Interfaces/SideEffectInterfaces.td" + +def Fallback_Dialect : Dialect { + let name = "tfrt_fallback"; + + let description = [{ + The TFRT Fallback Dialect. + }]; + + let cppNamespace = "::tfrt::fallback"; +} + +// This corresponds to tensorflow::Tensor. +def TFTensorType : DialectType($_self)">, "!tfrt_fallback.tf_tensor type">, + BuildableType<"$_builder.getType<::tfrt::fallback::TFTensorType>()"> { + let description = [{ + `!tfrt_fallback.tf_tensor type` represents a tensorflow::Tensor. + }]; +} + +// This corresponds to tensorflow::Allocator. +def TFAllocatorType : DialectType($_self)">, "!tfrt_fallback.tf_allocator type">, + BuildableType<"$_builder.getType<::tfrt::fallback::TFAllocatorType>()"> { + let description = [{ + `!tfrt_fallback.tf_tensor type` represents a tensorflow::Tensor. + }]; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.cc new file mode 100644 index 0000000000000000000000000000000000000000..ec6e98a602f43d9e278e0496d25debd544fe2815 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.cc @@ -0,0 +1,407 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" + +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/InliningUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h" +#include "tfrt/basic_kernels/opdefs/tfrt_base.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/attributes.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/sync/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tfrt { +namespace fallback_async { + +namespace { + +struct FallbackInlinerInterface : public mlir::DialectInlinerInterface { + using DialectInlinerInterface::DialectInlinerInterface; + + bool isLegalToInline(Operation *op, Region *dest, bool would_be_cloned, + IRMapping &) const final { + return true; + } +}; + +} // namespace + +FallbackAsyncDialect::FallbackAsyncDialect(MLIRContext *context) + : Dialect(/*name=*/"tfrt_fallback_async", context, + TypeID::get()) { + context->getOrLoadDialect(); + context->getOrLoadDialect(); + context->getOrLoadDialect(); + + allowUnknownTypes(); + + allowUnknownOperations(); + + addInterfaces(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.cpp.inc" + >(); +} + +static Type GetChainType(Builder *builder) { + return builder->getType(); +} + +LogicalResult CreateOp::verify() { + return fallback_common::VerifyFallbackExecuteOp(*this); +} +LogicalResult ExecuteOp::verify() { + return fallback_common::VerifyFallbackExecuteOp(*this); +} +LogicalResult ExecuteOpSeq::verify() { + return fallback_common::VerifyFallbackExecuteOp(*this); +} +LogicalResult ExecuteOpWithAllocator::verify() { + return fallback_common::VerifyExecuteOpCommon(*this); +} +LogicalResult ExecuteOpSeqWithAllocator::verify() { + return fallback_common::VerifyExecuteOpCommon(*this); +} +LogicalResult BatchFunctionOp::verify() { + return fallback_common::VerifyExecuteOpCommon(*this); +} + +ParseResult CreateOp::parse(OpAsmParser &parser, OperationState &result) { + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = true; + parse_options.has_key = true; + parse_options.has_device = true; + parse_options.has_func_attr = true; + parse_options.has_cost = false; + + auto &builder = parser.getBuilder(); + if (mlir::failed(fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options))) + return mlir::failure(); + + mlir::IntegerAttr num_args; + if (parser.parseKeyword("num_args") || parser.parseLParen() || + parser.parseAttribute(num_args, "num_args", result.attributes) || + parser.parseRParen()) + return mlir::failure(); + + return mlir::success(); +} +ParseResult ExecuteOp::parse(OpAsmParser &parser, OperationState &result) { + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = false; + parse_options.has_key = true; + parse_options.has_device = true; + parse_options.has_func_attr = true; + parse_options.has_cost = true; + + auto &builder = parser.getBuilder(); + return fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options); +} +ParseResult ExecuteOpSeq::parse(OpAsmParser &parser, OperationState &result) { + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = true; + parse_options.has_key = true; + parse_options.has_device = true; + parse_options.has_func_attr = true; + parse_options.has_cost = true; + + auto &builder = parser.getBuilder(); + return fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options); +} +ParseResult ExecuteOpWithAllocator::parse(OpAsmParser &parser, + OperationState &result) { + auto &builder = parser.getBuilder(); + llvm::SmallVector allocator; + if (parser.parseOperandList(allocator, + /*requiredOperandCount=*/1, + mlir::OpAsmParser::Delimiter::Paren)) + return mlir::failure(); + + if (parser.resolveOperand(allocator.front(), + builder.getType(), + result.operands)) + return mlir::failure(); + + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = false; + parse_options.has_key = true; + parse_options.has_device = true; + parse_options.has_func_attr = true; + parse_options.has_cost = true; + + return fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options); +} +ParseResult ExecuteOpSeqWithAllocator::parse(OpAsmParser &parser, + OperationState &result) { + auto &builder = parser.getBuilder(); + llvm::SmallVector + chain_and_allocator; + if (parser.parseOperandList(chain_and_allocator, + /*requiredOperandCount=*/2, + mlir::OpAsmParser::Delimiter::Paren)) + return mlir::failure(); + + auto &chain = chain_and_allocator[0]; + auto &allocator = chain_and_allocator[1]; + + if (parser.resolveOperand(chain, builder.getType(), + result.operands)) + return mlir::failure(); + + if (parser.resolveOperand(allocator, + builder.getType(), + result.operands)) + return mlir::failure(); + + // The first result is a chain. + result.types.push_back(builder.getType()); + + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = false; + parse_options.has_key = true; + parse_options.has_device = true; + parse_options.has_func_attr = true; + parse_options.has_cost = true; + + return fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options); +} + +ParseResult BatchFunctionOp::parse(OpAsmParser &parser, + OperationState &result) { + fallback_common::ParseExecuteOpOptions parse_options; + parse_options.has_chain = false; + parse_options.has_key = false; + parse_options.has_device = true; + parse_options.has_func_attr = false; + parse_options.has_cost = false; + parse_options.has_op_name = false; + parse_options.has_symbol_ref = true; + + auto &builder = parser.getBuilder(); + return fallback_common::ParseExecuteOpCommon( + parser, builder, result, builder.getType(), + parse_options); +} + +void CreateOp::print(OpAsmPrinter &p) { + CreateOp op = *this; + p << "(" << op.getInCh() << ") key(" + << op->getAttrOfType("op_key").getInt() << ") device(" + << op->getAttr("device") << ") " << op->getAttr("op_name") << "()"; + + fallback_common::PrintExecuteOpCommon(p, op); + fallback_common::PrintExecuteOpFuncAttribute(p, op); + + p << " num_args(" << op->getAttrOfType("num_args").getInt() + << ')'; +} + +void ExecuteOp::print(OpAsmPrinter &p) { + ExecuteOp op = *this; + p << " key(" << op->getAttrOfType("op_key").getInt() + << ") cost(" << op->getAttrOfType("_tfrt_cost").getInt() + << ") device(" << op->getAttr("device") << ") " << op->getAttr("op_name") + << '(' << op.getArgs() << ')'; + + fallback_common::PrintExecuteOpCommon(p, op); + fallback_common::PrintExecuteOpFuncAttribute(p, op); + if (!op.getResults().empty()) p << " : " << op.getResults().size(); +} + +void ExecuteOpSeq::print(OpAsmPrinter &p) { + ExecuteOpSeq op = *this; + p << "(" << op.getInOpChain() << ") key(" + << op->getAttrOfType("op_key").getInt() << ") cost(" + << op->getAttrOfType("_tfrt_cost").getInt() + << ") device(" << op->getAttr("device") << ") " << op->getAttr("op_name") + << '(' << op.getArgs() << ')'; + + fallback_common::PrintExecuteOpCommon(p, op); + fallback_common::PrintExecuteOpFuncAttribute(p, op); + if (!op.getResults().empty()) p << " : " << op.getResults().size(); +} + +void ExecuteOpWithAllocator::print(OpAsmPrinter &p) { + ExecuteOpWithAllocator op = *this; + p << "(" << op.getAllocator() << ") key(" + << op->getAttrOfType("op_key").getInt() << ") cost(" + << op->getAttrOfType("_tfrt_cost").getInt() + << ") device(" << op->getAttr("device") << ") " << op->getAttr("op_name") + << '(' << op.getArgs() << ')'; + + fallback_common::PrintExecuteOpCommon(p, op); + fallback_common::PrintExecuteOpFuncAttribute(p, op); + if (!op.getResults().empty()) p << " : " << op.getResults().size(); +} + +void ExecuteOpSeqWithAllocator::print(OpAsmPrinter &p) { + ExecuteOpSeqWithAllocator op = *this; + p << "(" << op.getInOpChain() << ", " << op.getAllocator() << ") key(" + << op->getAttrOfType("op_key").getInt() << ") cost(" + << op->getAttrOfType("_tfrt_cost").getInt() + << ") device(" << op->getAttr("device") << ") " << op->getAttr("op_name") + << '(' << op.getArgs() << ')'; + + fallback_common::PrintExecuteOpCommon(p, op); + fallback_common::PrintExecuteOpFuncAttribute(p, op); + if (!op.getResults().empty()) p << " : " << op.getResults().size(); +} + +void BatchFunctionOp::print(OpAsmPrinter &p) { + BatchFunctionOp op = *this; + p << " device(" << op->getAttr("device") << ") " << op->getAttr("f") << " (" + << op.getArgs() << ")"; + + fallback_common::PrintExecuteOpCommon(p, op); +} + +void ExecuteOp::getOpAttrs( + SmallVectorImpl> *op_attrs) { + fallback_common::GetExecuteOpAttrsCommon( + this->getContext(), this->getOpAttrs().getValue(), op_attrs); +} + +//===----------------------------------------------------------------------===// +// ConstDenseTensorOp +//===----------------------------------------------------------------------===// + +OpFoldResult ConstDenseTensorOp::fold(FoldAdaptor) { return getValue(); } + +//===----------------------------------------------------------------------===// +// CoreRTTensorHandleToFallbackTensorOp +//===----------------------------------------------------------------------===// + +namespace { + +// Simplifies pattern containing a corert const tensor op followed by a +// `tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor` op to a single +// tfrt_fallback_async const tensor. +struct ConstCoreRTTensorHandleToFallbackTensorCanonicalization + : public OpRewritePattern { + using OpRewritePattern< + CoreRTTensorHandleToFallbackTensorOp>::OpRewritePattern; + + LogicalResult matchAndRewrite(CoreRTTensorHandleToFallbackTensorOp op, + PatternRewriter &rewriter) const override { + SmallVector new_values; + bool should_rewrite = false; + for (auto operand : op.getArgs()) { + if (auto corert_const_dense_tensor_op = + operand.getDefiningOp()) { + new_values.push_back(fallback_async::ConstDenseTensorOp::create( + rewriter, op.getLoc(), rewriter.getType(), + corert_const_dense_tensor_op.getValue())); + should_rewrite = true; + continue; + } + if (auto corert_const_string_tensor_op = + operand.getDefiningOp()) { + new_values.push_back(fallback_async::ConstStringTensorOp::create( + rewriter, op.getLoc(), rewriter.getType(), + corert_const_string_tensor_op.getShape(), + corert_const_string_tensor_op.getValue())); + should_rewrite = true; + continue; + } + // To guarantee that the new values are in the same order as the old + // ones, we create individual ops for the non-canonicalizable operands. + // For simplicity, we don't consolidate these ops when all the + // non-canonicalizable operands are adjacent. + new_values.push_back( + fallback_async::CoreRTTensorHandleToFallbackTensorOp::create( + rewriter, op.getLoc(), rewriter.getType(), + operand, op->getAttrOfType("device")) + .getResult(0)); + } + + if (!should_rewrite) return failure(); + rewriter.replaceOp(op, new_values); + return success(); + } +}; + +// Removes the following double tensor conversion: +// %1 = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %0 +// %2 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %1 +struct RemoveDoubleTensorConversion + : mlir::OpRewritePattern { + using OpRewritePattern< + CoreRTTensorHandleToFallbackTensorOp>::OpRewritePattern; + + mlir::LogicalResult matchAndRewrite( + CoreRTTensorHandleToFallbackTensorOp op, + mlir::PatternRewriter &rewriter) const override { + // Currently only handles the case where there is only one value in the + // conversion op. This should be enough for most of the cases. + if (op.getNumOperands() > 1) return mlir::failure(); + + auto def = + op.getOperand(0).getDefiningOp(); + if (!def) return mlir::failure(); + + if (def.getNumResults() > 1) return mlir::failure(); + + rewriter.replaceOp(op, def.getOperand(0)); + + return mlir::success(); + } +}; + +} // namespace + +void CoreRTTensorHandleToFallbackTensorOp::getCanonicalizationPatterns( + RewritePatternSet &results, MLIRContext *context) { + results.add(context); +} + +} // namespace fallback_async +} // namespace tfrt + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h new file mode 100644 index 0000000000000000000000000000000000000000..fcf1405bd360ff56c81d3cfad1e03c7b3dfc3426 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h @@ -0,0 +1,45 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_ASYNC_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_ASYNC_H_ + +#include "mlir/Bytecode/BytecodeOpInterface.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "tfrt/compiler/opdefs/tfrt_op_interfaces.h" // from @tf_runtime +#include "tfrt/compiler/opdefs/tfrt_traits.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/traits.h" // from @tf_runtime + +using namespace mlir; // NOLINT + +namespace tfrt { +namespace fallback_async { + +// Dialect for fallback async operations. +class FallbackAsyncDialect : public Dialect { + public: + explicit FallbackAsyncDialect(MLIRContext *context); + static StringRef getDialectNamespace() { return "tfrt_fallback_async"; } +}; + +} // namespace fallback_async +} // namespace tfrt + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_ASYNC_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.td new file mode 100644 index 0000000000000000000000000000000000000000..935b32ae14fd4b7d479ddcc0b624f9544e6a9f44 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.td @@ -0,0 +1,531 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifdef TFRT_FALLBACK_ASYNC_OPS +#else +#define TFRT_FALLBACK_ASYNC_OPS + +include "tfrt/tfrt_op_base.td" +include "tfrt/compiler/opdefs/tfrt_op_interfaces.td" +include "tfrt/compiler/opdefs/tfrt_traits.td" +include "tfrt/core_runtime/opdefs/corert_base.td" +include "tfrt/core_runtime/opdefs/corert_traits.td" +include "tfrt_fallback.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +def FallbackAsync_Dialect : Dialect { + let name = "tfrt_fallback_async"; + + let description = [{ + The TFRT Fallback Async Dialect. + }]; + + let cppNamespace = "::tfrt::fallback_async"; +} + +class FallbackAsync_Op traits = []> : + Op { +} + +def SetResourceOp : FallbackAsync_Op<"set_resource", [CoreRT_TypedAttributeTrait]> { + let summary = "Set a tensor in resource array"; + + let description = [{ + Set a tensor in resource array. + + arg: the tensor to be set in the resource array. + index: the index in the resource array + + in_ch and out_ch are for control dependencies. + }]; + + let arguments = (ins + TFRT_ChainType:$in_ch, + TFTensorType:$arg, + StrAttr:$device, + I64Attr:$index + ); + + let results = (outs + TFRT_ChainType:$out_ch + ); + + let assemblyFormat = "operands attr-dict"; +} + +def GetResourceOp : FallbackAsync_Op<"get_resource", + [CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "get a tensor in resource array"; + + let description = [{ + Get a tensor in resource array. + + indices: the indices in the resource array. + results: the tensor values for the corresponding indices. + + in_ch and out_ch are for control dependencies. + }]; + + let arguments = (ins + TFRT_ChainType:$in_ch, + StrAttr:$device, + I64ArrayAttr:$indices + ); + + let results = (outs + TFRT_ChainType:$out_ch, + Variadic:$results + ); + + let assemblyFormat = "operands attr-dict `:` `(` type($in_ch) `)` `->` `(` type($out_ch) `,` type($results) `)`"; +} + +def ConstTensorProtoOp : FallbackAsync_Op<"const_tensor_proto", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Create a tensorflow tensor from serialized tensor proto"; + + let description = [{ + Create a tensorflow tensor from serilaized tensor proto. + + Example: + %tensor = tfrt_fallback_async.const_tensor_proto "" + }]; + + let arguments = (ins + StrAttr:$serialized_tensor_proto + ); + + let results = (outs + TFTensorType:$result + ); + + let assemblyFormat = "$serialized_tensor_proto attr-dict"; +} + +def ConstDenseTensorOp : FallbackAsync_Op<"const_dense_tensor", + [ConstantLike, Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Create a constant dense tensorflow tensor."; + + let description = [{ + ConstDenseTensorOp creates a constant dense tensorflow tensor. + + Example: + %tensor = tfrt_fallback_async.const_dense_tensor dense<[0, 1, 2]> : tensor<3xi32> + }]; + + let arguments = (ins + ElementsAttr:$value + ); + + let results = (outs + TFTensorType:$result + ); + + let assemblyFormat = "$value attr-dict"; + + let hasFolder = 1; +} + +// TODO(b/181257202): This should be `ConstantLike`. +def ConstStringTensorOp : FallbackAsync_Op<"const_string_tensor", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Create a constant string tensorflow tensor."; + let description = [{ + ConstStringTensorOp creates a constant string tensorflow tensor. + + Example: + %tensor = tfrt_fallback_async.const_string_tensor {shape = [1, 2], value = ["const", "string"]} + + Note: if the entries in `value` are all the same, we can just keep one copy. + In other words, the following two lines are effectively the same. + %tensor = tfrt_fallback_async.const_string_tensor {shape = [1, 2], value = ["string", "string"]} + %tensor = tfrt_fallback_async.const_string_tensor {shape = [1, 2], value = ["string"]} + }]; + + let arguments = (ins + I64ArrayAttr:$shape, + StrArrayAttr:$value + ); + + let results = (outs + TFTensorType:$result + ); + + let assemblyFormat = "attr-dict"; +} + +def FallbackTensorToCoreRTTensorHandleOp : FallbackAsync_Op< + "fallback_tensor_to_corert_tensorhandle", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Convert fallback tensor to corert tensorhandle"; + + let description = [{ + Convert the fallback tensor to corert tensorhandle. + + args: a list of fallback tensors. + device: the name of the tensorflow device that is associated with the + input fallback tensors. + + results: the converted corert tensorhandles. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$device + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "operands attr-dict `:` `(` type($args) `)` `->` `(` type($results) `)`"; +} + +def CoreRTTensorHandleToFallbackTensorOp : FallbackAsync_Op< + "corert_tensorhandle_to_fallback_tensor", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Convert corert tensorhandle to fallback tensor"; + + let description = [{ + Convert the corert tensorhandle to the fallback tensor. + + args: a list of corert tensorhandles. + device: the name of the tensorflow device that is associated with the + output fallback tensor. + + results: the converted fallback tensors. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$device + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "operands attr-dict `:` `(` type($args) `)` `->` `(` type($results) `)`"; + + let hasCanonicalizer = 1; +} + +def PredicateOp : FallbackAsync_Op<"predicate", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Converts a fallback tensor to a bool"; + + let description = [{ + Note: this kernel is used for CPU tensors. + + Converts a fallback tensor to a bool with the following rules: + + - For 0D tensors, truthiness is determined by comparing against a "zero" + value. For numerical types it is the obvious zero. For strings it is the + empty string. + + - For >0D tensors, truthiness is determined by looking at the number of + elements. If has zero elements, then the result is false. Otherwise the + result is true. + + input: a fallback tensor representing the condition. + device: the name of the tensorflow device that is associated with the + input fallback tensor. + + output: the converted bool. + }]; + + let arguments = (ins + TFTensorType:$input + ); + + let results = (outs + I1:$output + ); + + let assemblyFormat = "$input attr-dict"; +} + +def CopyIfSmallOp : FallbackAsync_Op<"copy_if_small", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_FixedCost<1>]> { + let summary = "Copy the CPU fallback tensor if it is small"; + + let description = [{ + This kernel performs deep copy on the input tensor if it is small, to avoid + atomic contention on its refcount. + + Note that this kernel always create a new AsyncValue for each result to + avoid atomic contention on AsyncValue's refcount. + }]; + + let arguments = (ins + TFTensorType:$operand + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "$operand attr-dict `:` `(` type($operand) `)` `->` `(` type($results) `)`"; +} + +def CreateOp: FallbackAsync_Op<"createop", [CoreRT_TypedAttributeTrait]> { + let summary = "The Fallback CreateOp"; + + let description = [{ + The CreateOp creates the tensorflow::OpKernel in the fallback context. + + op_key: The unique key to identify this op. + num_args: The number of inputs. + device: The tensorflow device. eg. "/CPU:0" + op_attrs: The tensorflow attributes excluding the func attrs. + op_func_attrs: The func attrs. + op_name: The tensorflow op name. eg. "tf.AddV2" + + Example: + %out_ch = tfrt_fallback_async.createop(%in_ch) key(0) device("/CPU:0") + "some.op"() {attr1 = value, attr2 = value2} {f_attr1 = "fn1", f_attr2 = "fn2"} num_args(1) + }]; + + let arguments = (ins + TFRT_ChainType:$in_ch, + I64Attr:$num_args, + StrAttr:$device, + ArrayAttr:$op_attrs, + // TODO(b/173025975): consider using DictionaryAttr after we support + // BEF conversion for this type. + ArrayAttr:$op_func_attrs, + I64Attr:$op_key, + StrAttr:$op_name + ); + + let results = (outs + TFRT_ChainType:$out_ch + ); + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def ExecuteOp : FallbackAsync_Op<"executeop", + [Pure, CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_AttrCostTrait]> { + let summary = "The Fallback ExecuteOp"; + let description = [{ + The ExecuteOp executes an operation on the specified device. + + Example: + %res = tfrt_fallback_async.executeop key(0) cost(100) device("/CPU:0") "some.op"(%arg) : 1 + + Note that the trailing number indicates the number of results. + + The func attrs and op attrs are placed in two separate regions. + + Example: + %res = tfrt_fallback_async.executeop key(0) device("/CPU:0") "some.op"(%arg) + {attr1 = value, attr2 = value2} {f_attr1 = "fn1", f_attr2 = "fn2"} : 1 + + Note that in this case, the first bracket contains op attributes other than + function attributes, and the second braket contains function attributes, + which represent function names. + + Refer to b/172597167 for approaches considered for supporting function + attributes. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$device, + ArrayAttr:$op_attrs, + // TODO(b/173025975): consider using DictionaryAttr after we support + // BEF conversion for this type. + ArrayAttr:$op_func_attrs, + I64Attr:$op_key, + StrAttr:$op_name, + I64Attr:$_tfrt_cost + ); + + let results = (outs + Variadic:$results + ); + + let extraClassDeclaration = [{ + void getOpAttrs(SmallVectorImpl>* op_attrs); + }]; + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def ExecuteOpSeq : FallbackAsync_Op<"executeop.seq", + [CoreRT_TypedAttributeTrait, + TFRT_CostFunctionInterface, TFRT_AttrCostTrait]> { + let summary = "The sequenced version of Fallback ExecuteOp"; + let description = [{ + The ExecuteOpSeq executes an operation on the specified device, similar to + ExecuteOp. In addition, it takes an in_op_chain and produces an out_op_chain + that can be used to sequence op executions. ExecuteOpSeq is nonstrict on + in_op_chain. + + Example: + %op_ch_out, %res = tfrt_fallback_async.executeop.seq(%op_ch_in) + key(0) cost(100) device("/CPU:0") "some.op"(%arg) : 1 + + Note that the trailing number indicates the number of results. + + The func attrs and op attrs are placed in two separate regions. + + Example: + %op_ch_out, %res = tfrt_fallback_async.executeop.seq(%op_ch_in) + key(0) cost(100) device("/CPU:0") "some.op"(%arg) + {fn = function_1} {attr1 = value, attr2 = value2} : 1 + + Note that in this case, the first braket contains op attributes other than + function attributes, and the second braket contains function attributes, + which represent function names. + + Refer to b/172597167 for approaches considered for supporting function + attributes. + }]; + + let arguments = (ins + TFRT_ChainType:$in_op_chain, + Variadic:$args, + StrAttr:$device, + ArrayAttr:$op_attrs, + // TODO(b/173025975): consider using DictionaryAttr after we support + // BEF conversion for this type. + ArrayAttr:$op_func_attrs, + I64Attr:$op_key, + StrAttr:$op_name, + I64Attr:$_tfrt_cost + ); + + let results = (outs + TFRT_ChainType:$out_op_chain, + Variadic:$results + ); + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def BatchFunctionOp : FallbackAsync_Op<"batch_function", + [Pure, CoreRT_TypedAttributeTrait]> { + let summary = "Fallback ExecuteOp specialized for tf.BatchFunction."; + let description = [{ + This kernel executes a variant tf.BatchFunction kernel that supports having + the `f` attribute as a BEF function. + + Example: + %res = tfrt_fallback_async.batch_function device("/CPU:0") @f(%input, %captured_input) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [2 , 4], + batch_timeout_micros = 50, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [f32], + Tcaptured = [f32], + Tout = [f32]} : 1 + + Note that the trailing number indicates the number of results. + }]; + + let arguments = (ins + Variadic:$args, + StrAttr:$device, + SymbolRefAttr:$f, + ArrayAttr:$op_attrs + ); + + let results = (outs + Variadic:$results + ); + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def ExecuteOpWithAllocator : FallbackAsync_Op<"executeop.allocator", + [Pure, CoreRT_TypedAttributeTrait, TFRT_CostFunctionInterface, TFRT_AttrCostTrait]> { + let summary = "The Fallback ExecuteOp with custom allocator"; + let description = [{ + Similar to ExecuteOp but takes a custom allocator for allocating output tensors. + }]; + + let arguments = (ins + TFAllocatorType:$allocator, + Variadic:$args, + StrAttr:$device, + ArrayAttr:$op_attrs, + ArrayAttr:$op_func_attrs, + I64Attr:$op_key, + StrAttr:$op_name, + I64Attr:$_tfrt_cost + ); + + let results = (outs + Variadic:$results + ); + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +def ExecuteOpSeqWithAllocator : FallbackAsync_Op<"executeop.seq.allocator", + [CoreRT_TypedAttributeTrait, TFRT_CostFunctionInterface, TFRT_AttrCostTrait]> { + let summary = "The sequenced version of Fallback ExecuteOp with custom allocator"; + let description = [{ + Similar to ExecuteOpSeq but takes a custom allocator for allocating output tensors. + + Example: + %op_ch_out, %res = tfrt_fallback_async.executeop.seq.allocator(%op_ch_in, %allocator) + key(0) cost(100) device("/CPU:0") "some.op"(%arg) : 1 + }]; + + let arguments = (ins + TFRT_ChainType:$in_op_chain, + TFAllocatorType:$allocator, + Variadic:$args, + StrAttr:$device, + ArrayAttr:$op_attrs, + // TODO(b/173025975): consider using DictionaryAttr after we support + // BEF conversion for this type. + ArrayAttr:$op_func_attrs, + I64Attr:$op_key, + StrAttr:$op_name, + I64Attr:$_tfrt_cost + ); + + let results = (outs + TFRT_ChainType:$out_op_chain, + Variadic:$results + ); + + let hasVerifier = 1; + let hasCustomAssemblyFormat = 1; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.cc new file mode 100644 index 0000000000000000000000000000000000000000..98cdf7bd5fa891b88951c98183360b1fdd8577dc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.cc @@ -0,0 +1,145 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h" + +#include +#include +#include + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project + +namespace tfrt { +namespace fallback_common { + +void GetExecuteOpAttrsCommon( + mlir::MLIRContext *context, llvm::ArrayRef op_attr_array, + llvm::SmallVectorImpl> + *op_attrs) { + assert(op_attrs); + op_attrs->clear(); + + mlir::Builder builder(context); + for (auto iter : op_attr_array) { + auto key_value = mlir::cast(iter).getValue(); + llvm::StringRef key = mlir::cast(key_value[0]).getValue(); + mlir::Attribute value = key_value[1]; + op_attrs->push_back({key, value}); + } +} + +mlir::ParseResult ParseExecuteOpCommon(mlir::OpAsmParser &parser, + mlir::Builder &builder, + mlir::OperationState &result, + mlir::Type tensor_type, + const ParseExecuteOpOptions &options) { + auto chain_type = builder.getType(); + + mlir::IntegerAttr op_key; + mlir::IntegerAttr cost; + mlir::StringAttr device; + mlir::StringAttr op_name; + mlir::SymbolRefAttr f; + llvm::SmallVector in_chains; + llvm::SmallVector operands; + mlir::NamedAttrList op_attrs; + mlir::NamedAttrList op_func_attrs; + auto loc = parser.getNameLoc(); + + if (options.has_chain && + parser.parseOperandList(in_chains, + /*requiredOperandCount=*/1, + mlir::OpAsmParser::Delimiter::Paren)) + return mlir::failure(); + + if (options.has_key && + (parser.parseKeyword("key") || parser.parseLParen() || + parser.parseAttribute(op_key, "op_key", result.attributes) || + parser.parseRParen())) + return mlir::failure(); + + if (options.has_cost && + (parser.parseKeyword("cost") || parser.parseLParen() || + parser.parseAttribute(cost, "_tfrt_cost", result.attributes) || + parser.parseRParen())) + return mlir::failure(); + + if (options.has_device && + (parser.parseKeyword("device") || parser.parseLParen() || + parser.parseAttribute(device, "device", result.attributes) || + parser.parseRParen())) + return mlir::failure(); + + if (options.has_op_name && + parser.parseAttribute(op_name, "op_name", result.attributes)) + return mlir::failure(); + + if (options.has_symbol_ref && + parser.parseAttribute(f, "f", result.attributes)) + return mlir::failure(); + + if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::Paren) || + parser.parseOptionalAttrDict(op_attrs) || + parser.parseOptionalAttrDict(op_func_attrs)) + return mlir::failure(); + + int64_t num_results = 0; + if (succeeded(parser.parseOptionalColon())) { + mlir::IntegerAttr attr; + mlir::NamedAttrList attrs; + if (failed(parser.parseAttribute(attr, "num_results", attrs))) + return mlir::failure(); + num_results = attr.getValue().getSExtValue(); + } + if (num_results < 0) return mlir::failure(); + + llvm::SmallVector operand_types; + if (options.has_chain) operand_types.push_back(chain_type); + if (parser.resolveOperands(in_chains, operand_types, loc, result.operands) || + parser.resolveOperands(operands, tensor_type, result.operands)) + return mlir::failure(); + + if (options.has_chain) result.types.push_back(chain_type); + result.types.append(num_results, tensor_type); + + llvm::SmallVector op_attr_array; + for (const auto &key_value : op_attrs) { + auto key = key_value.getName(); + auto value = key_value.getValue(); + op_attr_array.push_back(builder.getArrayAttr({key, value})); + } + + result.attributes.push_back( + builder.getNamedAttr("op_attrs", builder.getArrayAttr(op_attr_array))); + + // TODO(tfrt-devs): support func attributes in tfrt_fallback_sync. + if (options.has_func_attr) { + llvm::SmallVector op_func_attr_array; + for (const auto &key_value : op_func_attrs) { + auto key = key_value.getName(); + auto value = key_value.getValue(); + op_func_attr_array.push_back(builder.getArrayAttr({key, value})); + } + + result.attributes.push_back(builder.getNamedAttr( + "op_func_attrs", builder.getArrayAttr(op_func_attr_array))); + } + + return mlir::success(); +} + +} // namespace fallback_common +} // namespace tfrt diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h new file mode 100644 index 0000000000000000000000000000000000000000..eb71a9fdac258642e7af9cd74da5db506b32bb43 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h @@ -0,0 +1,127 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_COMMON_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_COMMON_H_ + +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime + +namespace tfrt { +namespace fallback_common { + +template +mlir::LogicalResult VerifyExecuteOpCommon(OpTy op) { + auto op_attr_array = op.getOpAttrs().getValue(); + for (auto op_attr : op_attr_array) { + auto key_value = mlir::dyn_cast(op_attr); + if (!key_value || key_value.getValue().size() != 2 || + !mlir::isa(key_value.getValue()[0])) + return op.emitOpError() << "each op_attr should be a key-value pair, " + "where the key is a string"; + } + return mlir::success(); +} + +template +mlir::LogicalResult VerifyFallbackExecuteOp(OpTy op) { + auto result = VerifyExecuteOpCommon(op); + if (failed(result)) return result; + + // Verify function attributes. + auto op_func_attr_array = op.getOpFuncAttrs().getValue(); + for (auto op_attr : op_func_attr_array) { + auto key_value = mlir::dyn_cast(op_attr); + if (!key_value || key_value.getValue().size() != 2 || + !mlir::isa(key_value.getValue()[0]) || + !mlir::isa(key_value.getValue()[1])) + return op.emitOpError() << "each op_func_attr should be a key-value " + "pair, where both the key and the value are " + "strings"; + } + return mlir::success(); +} + +template +void PrintExecuteOpFuncAttribute(mlir::OpAsmPrinter &p, OpTy op) { + auto op_func_attrs = op.getOpFuncAttrs(); + if (!op_func_attrs.empty()) { + auto print_key_value = [&](mlir::Attribute attr) { + auto key_value = mlir::cast(attr).getValue(); + auto key = key_value[0]; + auto value = key_value[1]; + + p << mlir::cast(key).getValue(); + p << " = "; + p << value; + }; + + auto op_func_attr_array = op_func_attrs.getValue(); + p << " {"; + llvm::interleaveComma(op_func_attr_array, p, print_key_value); + p << '}'; + } +} + +template +void PrintExecuteOpCommon(mlir::OpAsmPrinter &p, OpTy op) { + auto op_attrs = op.getOpAttrs(); + if (!op_attrs.empty()) { + auto print_key_value = [&](mlir::Attribute attr) { + auto key_value = mlir::cast(attr).getValue(); + auto key = key_value[0]; + auto value = key_value[1]; + + p << mlir::cast(key).getValue(); + p << " = "; + p << value; + }; + + auto op_attr_array = op_attrs.getValue(); + p << " {"; + llvm::interleaveComma(op_attr_array, p, print_key_value); + p << '}'; + } +} + +void GetExecuteOpAttrsCommon( + mlir::MLIRContext *context, llvm::ArrayRef op_attr_array, + llvm::SmallVectorImpl> + *op_attrs); + +struct ParseExecuteOpOptions { + bool has_chain = false; + bool has_key = false; + bool has_device = false; + bool has_func_attr = false; + bool has_cost = false; + bool has_op_name = true; + bool has_symbol_ref = false; +}; + +mlir::ParseResult ParseExecuteOpCommon(mlir::OpAsmParser &parser, + mlir::Builder &builder, + mlir::OperationState &result, + mlir::Type tensor_type, + const ParseExecuteOpOptions &options); +} // namespace fallback_common +} // namespace tfrt + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_COMMON_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.cc new file mode 100644 index 0000000000000000000000000000000000000000..36c929a1d5640d6ac8efefc6cf842afcc12dcb2a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.cc @@ -0,0 +1,65 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h" + +#include "llvm/ADT/STLExtras.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectImplementation.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_common.h" +#include "tfrt/basic_kernels/opdefs/tfrt_base.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/attributes.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime +#include "tfrt/tensor/opdefs/tensor.h" // from @tf_runtime + +namespace tfrt { +namespace fallback_sync { + +FallbackSyncDialect::FallbackSyncDialect(MLIRContext *context) + : Dialect(/*name=*/"tfrt_fallback_sync", context, + TypeID::get()) { + context->getOrLoadDialect(); + context->getOrLoadDialect(); + context->getOrLoadDialect(); + + allowUnknownTypes(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.cpp.inc" + >(); +} + +static Type GetTensorType(Builder *builder) { + return tfrt::tfrt_tensor::TensorType::get(builder->getContext()); +} + +} // namespace fallback_sync +} // namespace tfrt + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.cpp.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h new file mode 100644 index 0000000000000000000000000000000000000000..6048e5e639f4dcb96a53a071d740f111c87b7045 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h @@ -0,0 +1,45 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_SYNC_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_SYNC_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OpImplementation.h" // from @llvm-project +#include "mlir/Interfaces/InferTypeOpInterface.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "tfrt/core_runtime/opdefs/traits.h" // from @tf_runtime +#include "tfrt/tensor/opdefs/tensor.h" // from @tf_runtime + +using namespace mlir; // NOLINT + +namespace tfrt { +namespace fallback_sync { + +// Dialect for fallback operations. +class FallbackSyncDialect : public Dialect { + public: + explicit FallbackSyncDialect(MLIRContext *context); + static StringRef getDialectNamespace() { return "tfrt_fallback_sync"; } +}; + +} // namespace fallback_sync +} // namespace tfrt + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_SYNC_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.td new file mode 100644 index 0000000000000000000000000000000000000000..9e3bdfbef967da6edd4236f6f1dcf79488661bec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.td @@ -0,0 +1,286 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifdef TFRT_FALLBACK_SYNC_OPS +#else +#define TFRT_FALLBACK_SYNC_OPS + +include "tfrt/tfrt_op_base.td" +include "tfrt/core_runtime/opdefs/corert_base.td" +include "tfrt/core_runtime/opdefs/corert_traits.td" +include "tfrt_fallback.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td" + +def FallbackSync_Dialect : Dialect { + let name = "tfrt_fallback_sync"; + + let description = [{ + The TFRT Fallback Sync Dialect. + }]; + + let cppNamespace = "::tfrt::fallback_sync"; +} + +class FallbackSync_Op traits = []> : + Op { +} + +// TODO(rohitju): Add a verifier if there is anything that should be verified +// compile time. +def SetResourceOp : FallbackSync_Op<"set_resource", [CoreRT_TypedAttributeTrait]> { + let summary = "Set a tensor in resource array"; + + let description = [{ + Set a tensor in resource array. + + arg: the tensor to be set in the resource array. + index: the index in the resource array + }]; + + let arguments = (ins + TFTensorType:$arg, + I64Attr:$index + ); + + let results = (outs); + + let assemblyFormat = "operands attr-dict"; +} + +def SetResourceDhtOp : FallbackSync_Op<"set_resource_dht", [CoreRT_TypedAttributeTrait]> { + let summary = "Set a DHT in resource array"; + + let description = [{ + Set a DHT in resource array. + + arg: the tensor to be set in the resource array. + index: the index in the resource array + }]; + + let arguments = (ins + TensorType:$arg, + I64Attr:$index + ); + + let results = (outs); + + let assemblyFormat = "operands attr-dict"; +} + +def GetResourceOp : FallbackSync_Op<"get_resource", + [CoreRT_TypedAttributeTrait]> { + let summary = "get a tensor in resource array"; + + let description = [{ + Get a tensor in resource array. + + indices: the indices in the resource array. + results: the tensor values for the corresponding indices. + }]; + + let arguments = (ins + I64ArrayAttr:$indices + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "attr-dict `:` type($results)"; +} + +def GetResourceDhtOp : FallbackSync_Op<"get_resource_dht", + [CoreRT_TypedAttributeTrait]> { + let summary = "get a DHT in resource array"; + + let description = [{ + Get a tensor in resource array. + + indices: the indices in the resource array. + results: the tensor values for the corresponding indices. + }]; + + let arguments = (ins + I64ArrayAttr:$indices + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "attr-dict `:` type($results)"; +} + +def CreateOp: FallbackSync_Op<"createop", []> { + let summary = "The Fallback CreateOp"; + + let description = [{ + The CreateOp creates the tensorflow::OpKernel in the fallback context. + + op_key: The unique key to identify this op. + num_args: The number of inputs. + device: The tensorflow device. eg. "/CPU:0" + op_attrs: The tensorflow attributes excluding the func attrs. + op_name: The tensorflow op name. eg. "tf.AddV2" + + Example: + tfrt_fallback_sync.createop() key(0) device("/CPU:0") + "some.op"() {attr1 = value, attr2 = value2} {f_attr1 = "fn1", f_attr2 = "fn2"} num_args(1) + }]; + + let arguments = (ins + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs); + + let assemblyFormat = "attr-dict"; +} + +def SyncExecuteOp : FallbackSync_Op<"executeop", [Pure]> { + let summary = "The Fallback Sync ExecuteOp"; + let description = [{ + The ExecuteOp executes an operation on the specified device. + + Example: + %res = tfrt_fallback_sync.executeop "some.op"(%arg) : 1 + + Note that the trailing number indicates the number of results. + }]; + + let arguments = (ins + Variadic, + StrAttr:$node_def, + I32Attr:$op_key + ); + + let results = (outs Variadic); + + let assemblyFormat = "`(`operands`)` attr-dict `:` functional-type(operands, results)"; +} + + +def ConvertDhtToFallbackTensorOp : FallbackSync_Op<"convert_dht_to_fallback_tensor", + [CoreRT_TypedAttributeTrait]> { + let summary = "convert a DHT to a fallback tensor"; + + let description = [{ + Convert a TFRT DHT to a fallback tensor. + + dht: the input DHT. + fallback_tensor: The output fallback tensor + }]; + + let arguments = (ins + TensorType:$dht + ); + + let results = (outs + TFTensorType:$fallback_tensor + ); + + let assemblyFormat = "operands attr-dict `:` type($fallback_tensor)"; +} + +def ConvertShtToFallbackTensorOp : FallbackSync_Op<"convert_sht_to_fallback_tensor", + [CoreRT_TypedAttributeTrait]> { + let summary = "convert a SHT to a fallback tensor"; + + let description = [{ + Convert a TFRT SHT to a fallback tensor. + + sht: the input SHT. + fallback_tensor: The output fallback tensor + }]; + + let arguments = (ins + TensorType:$sht + ); + + let results = (outs + TFTensorType:$fallback_tensor + ); + + let assemblyFormat = "operands attr-dict `:` type($fallback_tensor)"; +} + +def ConvertFallbackTensorToDhtOp : FallbackSync_Op<"convert_fallback_tensor_to_dht", + [CoreRT_TypedAttributeTrait]> { + let summary = "convert fallback tensor to a DHT"; + + let description = [{ + Convert a fallback tensor to a TFRT DHT. + + fallback_tensor: The input fallback tensor + dht: the output DHT. + }]; + + let arguments = (ins + TFTensorType:$fallback_tensor + ); + + let results = (outs + TensorType:$dht + ); + + let assemblyFormat = "operands attr-dict `:` type($dht)"; +} + +// TODO(rohitju): This is Ads specific, need to find an appropriate home for it. +def SetSparseMatrixResourceOp : FallbackSync_Op<"set_sparse_matrix_resource", [CoreRT_TypedAttributeTrait]> { + let summary = "Set a Sparse matrix in resource array"; + + let description = [{ + Set a sparse matrix in resource array. + + arg: the matrix to be set in the resource array. + index: the index in the resource array + }]; + + let arguments = (ins + TFTensorType:$arg, + I64Attr:$index + ); + + let results = (outs); + + let assemblyFormat = "operands attr-dict"; +} + +def GetSparseMatrixResourceOp : FallbackSync_Op<"get_sparse_matrix_resource", + [CoreRT_TypedAttributeTrait]> { + let summary = "get a sparse matrix from resource array"; + + let description = [{ + Get a sparse matrix from resource array. + + indices: the indices in the resource array. + results: the tensor values for the corresponding indices. + }]; + + let arguments = (ins + I64ArrayAttr:$indices + ); + + let results = (outs + Variadic:$results + ); + + let assemblyFormat = "attr-dict `:` type($results)"; +} + +#endif diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.cc new file mode 100644 index 0000000000000000000000000000000000000000..fccf8998176f63288ce34d9d0f56ea59e0e0adc1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.cc @@ -0,0 +1,56 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.h" + +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" + +namespace tfrt { +namespace fallback_async { + +bool IsArgConsumedByFallback(mlir::func::FuncOp func, int arg_index) { + auto arg = func.getArgument(arg_index); + + // Return true if any user is a fallback op. It is more interesting to know + // whether it is consumed by any fallback op than whether it is only consumed + // by fallback ops. For example, the conversion from a DenseHostTensor to a + // fallback tensor is more costly than the conversion from fallback tensor to + // a DenseHostTensor. So as long as one of the users of a captured variable is + // a fallback op, we should keep this variable as a fallback tensor. + for (mlir::Operation *user : arg.getUsers()) { + if (llvm::isa(user->getDialect())) return true; + } + return false; +} + +void ForEachArgConsumedByFallback( + mlir::func::FuncOp func, llvm::function_ref action) { + for (int arg_index = 0; arg_index < func.getNumArguments(); ++arg_index) { + if (IsArgConsumedByFallback(func, arg_index)) action(arg_index); + } +} + +void ForEachArgConsumedByFallback( + mlir::ModuleOp module, + llvm::function_ref action) { + for (auto func : module.getOps()) { + ForEachArgConsumedByFallback( + func, [func_name = func.getName(), action](int arg_index) { + action(func_name, arg_index); + }); + } +} + +} // namespace fallback_async +} // namespace tfrt diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.h new file mode 100644 index 0000000000000000000000000000000000000000..40a25229a590ecef68d31fadfaee15e781e5bb47 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.h @@ -0,0 +1,36 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_UTIL_H_ + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project + +namespace tfrt { +namespace fallback_async { + +bool IsArgConsumedByFallback(mlir::func::FuncOp func, int arg_index); + +void ForEachArgConsumedByFallback( + mlir::func::FuncOp func, llvm::function_ref action); + +void ForEachArgConsumedByFallback( + mlir::ModuleOp module, + llvm::function_ref action); + +} // namespace fallback_async +} // namespace tfrt + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_IR_TFRT_FALLBACK_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_combine.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_combine.cc new file mode 100644 index 0000000000000000000000000000000000000000..2828162a9cd69e209a072bbd0b77f1273125f6ec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_combine.cc @@ -0,0 +1,80 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +//===----------------------------------------------------------------------===// +// +// This file implements a set of simple combiners for optimizing operations in +// the Runtime Fallback dialect. +// +//===----------------------------------------------------------------------===// + +#include "mlir/IR/Matchers.h" +#include "mlir/IR/PatternMatch.h" +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.h" + +// This optimizes the following scenario: +// %tft0, %c2 = "tfd.move_dht_to_tft"(%dht0, %c1) +// : (!dht.host_tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) +// %dht1, %c3 = "tfd.convert_tft_to_dht"(%tft0, %c2) +// : (!tfd.tf_tensor, !tfrt.chain) -> (!dht.host_tensor, !tfrt.chain) +// some_op %dht1, %c3 +// +// becomes +// some_op %dht0, %c1 + +struct SimplifyDoubleConversion + : public mlir::OpRewritePattern { + // We register this pattern to match every tfd.move_dht_to_tft op. + // The "benefit" is used by the framework to order the patterns and process + // them in order of profitability. + explicit SimplifyDoubleConversion(mlir::MLIRContext* context) + : mlir::OpRewritePattern(context, + /*benefit=*/1) {} + + // This method attempts to match a pattern and rewrite it. The rewriter + // argument is the orchestrator of the sequence of rewrites. The pattern is + // expected to interact with it to perform any changes to the IR from here. + mlir::LogicalResult matchAndRewrite( + mlir::tfd::ConvertTftToDhtOp op, + mlir::PatternRewriter& rewriter) const override { + // Look through the inputs of the ConvertTftToDhtOp. + mlir::Value convert_op_input_0 = op.getOperand(0); + mlir::Value convert_op_input_1 = op.getOperand(1); + mlir::tfd::MoveDhtToTftOp move_input_op_0 = + llvm::dyn_cast_or_null( + convert_op_input_0.getDefiningOp()); + mlir::tfd::MoveDhtToTftOp move_input_op_1 = + llvm::dyn_cast_or_null( + convert_op_input_1.getDefiningOp()); + + // The inputs should be MoveDhtToTftOp. + if (!move_input_op_0 || !move_input_op_1) return mlir::failure(); + // Both inputs are the same MoveDhtToTftOp. + if (move_input_op_0 != move_input_op_1) return mlir::failure(); + + // Use the rewriter to replace the ConvertTftToDhtOp's users with the + // operands of MoveDhtToTftOp. + rewriter.replaceOp( + op, {move_input_op_0.getOperand(0), move_input_op_0.getOperand(1)}); + return mlir::success(); + } +}; + +// Register rewrite pattern as "canonicalization" patterns on the MoveDhtToTftOp +// so that they can be picked up by the Canonicalization framework. +void mlir::tfd::ConvertTftToDhtOp::getCanonicalizationPatterns( + RewritePatternSet& results, MLIRContext* context) { + results.add(context); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.cc new file mode 100644 index 0000000000000000000000000000000000000000..e069c3597bd862193ed5289cc93d6a355db1bd1b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.cc @@ -0,0 +1,230 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.h" + +#include +#include +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SourceMgr.h" +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/utils/host_context.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/threadpool.h" +#include "tensorflow/core/platform/threadpool_interface.h" +#include "tensorflow/core/runtime_fallback/kernel/kernel_fallback_execute_compat_eager.h" +#include "tensorflow/core/runtime_fallback/runtime/kernel_utils.h" +#include "tensorflow/core/tfrt/utils/fallback_tensor.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_converter/mlir_to_bef.h" // from @tf_runtime +#include "tfrt/bef_executor/bef_file.h" // from @tf_runtime +#include "tfrt/host_context/async_value.h" // from @tf_runtime +#include "tfrt/host_context/chain.h" // from @tf_runtime +#include "tfrt/host_context/execution_context.h" // from @tf_runtime +#include "tfrt/host_context/function.h" // from @tf_runtime +#include "tfrt/host_context/host_context.h" // from @tf_runtime +#include "tfrt/host_context/resource_context.h" // from @tf_runtime +#include "tfrt/support/ref_count.h" // from @tf_runtime + +namespace tensorflow { + +using ::tfrt::AsyncValue; +using ::tfrt::BEFFile; +using ::tfrt::ExecutionContext; +using ::tfrt::Function; +using ::tfrt::MakeAvailableAsyncValueRef; +using ::tfrt::RCReference; +using ::tfrt::RequestContextBuilder; + +using ::tensorflow::Env; +using ::tensorflow::thread::ThreadPool; +using ::tensorflow::thread::ThreadPoolInterface; + +using ::tensorflow::tfrt_stub::FallbackTensor; + +// -------------------------------------------------------------------------- // +// Run function via the TF->TFRT fallback lowering. +// -------------------------------------------------------------------------- // + +namespace { +// Thread pool for running `intra-op` tasks scheduled by the fallback kernels. +class IntraOpThreadPool : public ThreadPoolInterface { + public: + explicit IntraOpThreadPool(int64_t num_threads) + : tpool_(Env::Default(), "intra-op", + std::max(1, static_cast(num_threads))) {} + + void Schedule(std::function fn) override { + tpool_.Schedule(std::move(fn)); + } + + int NumThreads() const override { return tpool_.NumThreads(); } + int CurrentThreadId() const override { return tpool_.CurrentThreadId(); } + void Cancel() override {} + + private: + ThreadPool tpool_; +}; +} // namespace + +RuntimeFallbackExecutor::RuntimeFallbackExecutor(int64_t num_threads) + : intra_op_(std::make_unique(num_threads)) { + // Create a HostContext for running TFRT functions. Concurrent work queue acts + // similar to the Tensorflow `inter-op` thread pool, so we'll match the size. + host_context_ = num_threads ? CreateMultiThreadedHostContext(num_threads) + : CreateSingleThreadedHostContext(); + tfrt::RegisterStaticKernels(host_context_->GetMutableRegistry()); + + // Build an ExecutionContext from the HostContext. + auto builder = RequestContextBuilder(host_context_.get(), &resource_context_); + + // Get tensorflow::EagerContext for the kernel fallback. + auto* eager_context_resource = + resource_context_ + .GetOrCreateResource( + tensorflow::tfd::kEagerContextResourceName); + auto expected_eager_context = eager_context_resource->GetTFEagerContext(); + auto* eager_context = expected_eager_context.get(); + + // Initialize fallback kernels state with a custom intra-op thread pool. + auto status = tensorflow::tfd::SetUpKernelFallbackCompatRequestContext( + &builder, /*runner_table=*/nullptr, eager_context, intra_op_.get()); + CHECK(status.ok()) << "Failed to setup request context: " << status.message(); + + auto req_ctx = std::move(builder).build(); + if (auto err = req_ctx.takeError()) + LOG(FATAL) << "Failed to build a request context"; + + exec_ctx_ = std::make_unique(std::move(*req_ctx)); +} + +void RuntimeFallbackExecutor::Prepare(llvm::StringRef mlir_input) { + // We only support IR written in the Tensorflow dialect. + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + + llvm::SourceMgr source_mgr; + source_mgr.AddNewSourceBuffer( + llvm::MemoryBuffer::getMemBuffer(mlir_input, "test_ir"), llvm::SMLoc()); + + // Parse a kernel source code into the MLIR Module. + mlir::OwningOpRef module( + mlir::parseSourceFile(source_mgr, &context)); + CHECK(module) << "failed to parse mlir module"; + + // Collect all diagnostics emitted while lowering parsed kernel module. + std::string diagnostic_str; + llvm::raw_string_ostream os(diagnostic_str); + mlir::SourceMgrDiagnosticHandler handler(source_mgr, module->getContext(), + os); + + // Convert TF to TFRT fallback dialect. + TfrtPipelineOptions pipeline_opts; + pipeline_opts.default_device = kDefaultHostDeviceName; + pipeline_opts.hoist_invariant_ops = true; + pipeline_opts.sink_in_invariant_ops = false; + pipeline_opts.cost_threshold = 1024; + pipeline_opts.merge_inter_dependent_streams = true; + + mlir::PassManager pm(module->getContext()); + pm.addPass(CreateTfToTfrtConversionPass(pipeline_opts)); + + CHECK(mlir::succeeded(pm.run(*module))) + << "Failed to lower module to TFRT: " << os.str(); + + // Convert module to BEF. + bef_buffer_ = + tfrt::ConvertMLIRToBEF(*module, /*disable_optional_sections=*/false); + CHECK(!bef_buffer_.empty()) << "Failed to convert module to BEF"; + + bef_file_ = + BEFFile::Open(bef_buffer_, host_context_->GetKernelRegistry(), + host_context_->diag_handler(), host_context_->allocator()); + CHECK(bef_file_) << "Failed to open BEF"; + + // Run TFRT initialization function to pre-instantiate fallback kernels. + RunTfrtInitializer(); +} + +llvm::SmallVector RuntimeFallbackExecutor::Execute( + llvm::StringRef function_name, llvm::ArrayRef arguments) { + // Get the kernel entrypoint function. + const Function* compute = bef_file_->GetFunction(function_name); + CHECK(compute) << "Entrypoint function not found"; + CHECK_EQ(arguments.size() + 1, compute->num_arguments()) + << "Wrong number of arguments for function " << function_name.str(); + + // Prepare function arguments from ready Chain and input Tensors. + llvm::SmallVector exec_arguments; + exec_arguments.reserve(compute->num_arguments()); + exec_arguments.push_back(tfrt::GetReadyChain().release()); + for (const Tensor& input_tensor : arguments) { + auto av = MakeAvailableAsyncValueRef(input_tensor); + exec_arguments.push_back(av.release()); + } + + // Space for returned values. + llvm::SmallVector> results(compute->num_results()); + + compute->Execute(*exec_ctx_, exec_arguments, results); + + // Wait for the function execution to finish, as well as the side-effects. + host_context_->Await(results); + + // Check that all results are available. + llvm::SmallVector ret_values; + for (unsigned i = 1; i < results.size(); ++i) { + if (auto* error = results[i]->GetErrorIfPresent()) + LOG(FATAL) << "Failed to execute a function: " << error->message(); + ret_values.push_back(results[i]->get().tensor()); + } + + // Deallocate arguments. + for (auto* argument : exec_arguments) argument->DropRef(); + return ret_values; +} + +// Run TFRT fallback initialization function to instantiate all fallback +// kernels ahead of executing the compute function. +void RuntimeFallbackExecutor::RunTfrtInitializer() { + const Function* func = bef_file_->GetFunction("_tfrt_fallback_init"); + CHECK(func) << "TFRT initialization function was not found"; + CHECK_EQ(func->argument_types().size(), 1); + + llvm::SmallVector, 1> results; + results.resize(func->result_types().size()); + CHECK_EQ(results.size(), 1); + + func->Execute(*exec_ctx_, tfrt::GetReadyChain().GetAsyncValue(), results); + + host_context_->Await(results); + + CHECK(!results[0]->IsError()) << "Failed to run TFRT initialization function"; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.h new file mode 100644 index 0000000000000000000000000000000000000000..a0b313367e45ab90b4a4ef1dbaed161b4f7db80f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_executor.h @@ -0,0 +1,63 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_EXECUTOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_EXECUTOR_H_ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/threadpool_interface.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_executor/bef_file.h" // from @tf_runtime +#include "tfrt/host_context/execution_context.h" // from @tf_runtime +#include "tfrt/host_context/host_context.h" // from @tf_runtime +#include "tfrt/host_context/resource_context.h" // from @tf_runtime +#include "tfrt/support/ref_count.h" // from @tf_runtime + +namespace tensorflow { + +class RuntimeFallbackExecutor { + public: + explicit RuntimeFallbackExecutor(int64_t num_threads); + + // Prepare() needs to be called once before calling Execute(). It sets up all + // things necessary to execute the given 'mlir_input' with the fallback to + // tensorflow. + void Prepare(llvm::StringRef mlir_input); + + // Execute() can be called several times after the call to Prepare() (e.g. for + // benchmarking). + llvm::SmallVector Execute(llvm::StringRef function_name, + llvm::ArrayRef arguments); + + private: + void RunTfrtInitializer(); + + std::unique_ptr intra_op_; + std::unique_ptr host_context_; + tfrt::ResourceContext resource_context_; + std::unique_ptr exec_ctx_; + tfrt::BefBuffer bef_buffer_; + tfrt::RCReference bef_file_; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_EXECUTOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..9c9828a5a9fd568ecb36ffbde82ac34e37753e8a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.cc @@ -0,0 +1,47 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.h" + +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime + +namespace mlir { +namespace tfd { + +//===----------------------------------------------------------------------===// +// TfrtDelegate Dialect +//===----------------------------------------------------------------------===// + +RuntimeFallbackDialect::RuntimeFallbackDialect(MLIRContext *context) + : Dialect(/*name=*/"tfd", context, TypeID::get()) { + allowUnknownTypes(); + + allowUnknownOperations(); + + addOperations< +#define GET_OP_LIST +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback_ops.cc.inc" + >(); +} + +} // namespace tfd +} // namespace mlir + +//===----------------------------------------------------------------------===// +// TableGen'd op method definitions +//===----------------------------------------------------------------------===// + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback_ops.cc.inc" diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..3e3a5ae7e9c923b9a759d2b0d97a1bba5e6f632c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.h @@ -0,0 +1,45 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file defines the operations used in the Runtime Fallback dialect. + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_OPS_H_ + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "tfrt/tensor/opdefs/tensor.h" // from @tf_runtime + +namespace mlir { +namespace tfd { + +// Dialect for TFRT delegate operations. +class RuntimeFallbackDialect : public Dialect { + public: + explicit RuntimeFallbackDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "tfd"; } +}; + +} // namespace tfd +} // namespace mlir + +#define GET_OP_CLASSES +#include "tensorflow/compiler/mlir/tfrt/runtime_fallback_ops.h.inc" + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_RUNTIME_FALLBACK_RUNTIME_FALLBACK_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.td new file mode 100644 index 0000000000000000000000000000000000000000..de963edddd63c3db33e39bd28a193ea0529b224f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/runtime_fallback/runtime_fallback_ops.td @@ -0,0 +1,158 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is the definition file for the Runtime Fallback Dialect. + +#ifdef TFRT_DELEGATE_DIALECT +#else +#define TFRT_DELEGATE_DIALECT + +include "tfrt/tfrt_op_base.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +//===----------------------------------------------------------------------===// +// Type definitions +//===----------------------------------------------------------------------===// +def TfTensorType : OpaqueType<"tfd", "tf_tensor", "!tfd.tf_tensor type">; + +//===----------------------------------------------------------------------===// +// Runtime Fallback Dialect definitions +//===----------------------------------------------------------------------===// + +def RuntimeFallback_Dialect : Dialect { + let name = "tfd"; + + let description = [{ + The Runtime Fallback dialect. + + This dialect contains operations to run existing TF kernels on TFRT by + invoking TF Eager API. + }]; + + let cppNamespace = "::mlir::tfd"; +} + +//===----------------------------------------------------------------------===// +// Runtime Fallback Dialect Ops definitions +//===----------------------------------------------------------------------===// + +// Base class for the operation in this dialect. +class RuntimeFallbackDialect_Op traits = []> : + Op { } + +def InitEagerContextOp : RuntimeFallbackDialect_Op<"init_eager_context"> { + let summary = "eager context initialization operation"; + let description = [{ + The "tfd.init_eager_context" operation takes an input chain, creates and + initializes the TF EagerContext and returns an output chain. + + Example: + %c1 = "tfd.init_eager_context"(%c0): (!tfrt.chain) -> !tfrt.chain + }]; + + let arguments = (ins TFRT_ChainType); + let results = (outs TFRT_ChainType); +} + +def DelegateKernelOp : RuntimeFallbackDialect_Op<"delegate_kernel"> { + let summary = "delegate kernel operation"; + let description = [{ + The "tfd.delegate_kernel" operation takes an input chain, and arbitrary + number of input arguments, and runs a specified TF op via TFE C API. It + returns an output chain and variable number of outputs from the TF op. + + The input arguments and attributes are passed to the TF op. The outputs are + outputs of the TF op. + + Note that `_name` is a required attribute specifying the TF op to run. + TFRT attributes are sorted alphabetically, passed in as positional + attributes to the TFRT kernel, rather than as named attributes. + + Example: + To run "tf.MatMul" op, which has two boolean attributes, + 1. Set _name = "MatMul" + 2. For each TF attribute, split it into two attributes, one for name of + the TF attribute, and the other for the type and value of the + attribute value. Attribute value is a string with the format of + "type$val", where type can be "bool", "string", "tfdtype", "tfshape", + "tftensor". + The value serialization format can be found in attr_util.h. + + %out_c, %out_tensor = "tfd.delegate_kernel"( + %in_c, %in1_tensor, %in2_tensor) { + _name = "MatMul", + attr1_name = "transpose_a", attr1_value = "bool$false", + attr2_name = "transpose_b", attr2_value = "bool$false" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> ( + !tfrt.chain, !tfd.tf_tensor) + }]; + + let arguments = (ins TFRT_ChainType, Variadic); + let results = (outs TFRT_ChainType, Variadic); +} + +def PrintTftOp : RuntimeFallbackDialect_Op<"print_tft"> { + let summary = "print TF tensor operation"; + let description = [{ + The "tfd.print_tft" operation prints the input TF tensor. It takes an input + TF tensor to be printed and an input chain, and returns an output chain. + + Example: + %c1 = "tfd.print_tft"(%t, %c) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + }]; + + let arguments = (ins TfTensorType, TFRT_ChainType); + let results = (outs TFRT_ChainType); +} + +def ConvertTftToDhtOp : RuntimeFallbackDialect_Op<"convert_tft_to_dht", [Pure]> { + let summary = "convert TF tensor to TFRT DHT tensor operation"; + let description = [{ + The "tfd.convert_tft_to_dht" operation converts a TF tensor to a TFRT + DenseHostTensor. + + It takes as input a TF Tensor and an input chain, and returns a converted + TFRT DHT tensor and an output chain. + + Example: + %dht, %c0 = "tfd.convert_tft_to_dht"(%tft, %c) + : (!tfd.tf_tensor, !tfrt.chain) -> (!dht.host_tensor, !tfrt.chain) + }]; + + let arguments = (ins TfTensorType, TFRT_ChainType); + // Enable registering canonicalization patterns with this operation. + let hasCanonicalizer = 1; + let results = (outs TensorType, TFRT_ChainType); +} + +def MoveDhtToTftOp : RuntimeFallbackDialect_Op<"move_dht_to_tft", [Pure]> { + let summary = "convert TFRT DHT tensor to DHT tensor operation"; + let description = [{ + The "tfd.move_dht_to_tft" operation moves a TFRT tensor into a TF Tensor. + + It takes as input a TFRT Tensor and an input chain, and returns a TF tensor + with the same underlying buffer and an output chain. + + Example: + %dht, %c0 = "tfd.convert_tft_to_dht"(%tft, %c) + : (!tfd.tf_tensor, !tfrt.chain) -> (!dht.host_tensor, !tfrt.chain) + }]; + + let arguments = (ins TensorType, TFRT_ChainType); + let results = (outs TfTensorType, TFRT_ChainType); +} + +#endif // TFRT_DELEGATE_DIALECT diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.cc new file mode 100644 index 0000000000000000000000000000000000000000..62f21bdb26561e826ef79dcc023589c350bf7052 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.cc @@ -0,0 +1,154 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/saved_model/saved_model.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tsl/platform/errors.h" +#include "tfrt/bef_converter/mlir_to_bef.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +using ::mlir::tf_saved_model::kTfSavedModelIndexPathAttr; + +llvm::StringRef ProcessIndexPath(mlir::ArrayAttr index_path) { + if (index_path.size() == 1 && mlir::isa(index_path[0])) { + // TODO(chky): Support cases where index_path is not a single string. + return mlir::cast(index_path[0]).getValue(); + } + return ""; +} + +absl::StatusOr> +ProcessTensorSpec(mlir::TensorType type) { + tensorflow::DataType dtype; + TF_RETURN_IF_ERROR( + ConvertScalarTypeToDataType(type.getElementType(), &dtype)); + + if (!type.hasRank()) + return std::make_pair(dtype, tensorflow::PartialTensorShape()); + + auto shape = type.getShape(); + llvm::SmallVector dims; + dims.assign(shape.begin(), shape.end()); + return std::make_pair(dtype, tensorflow::PartialTensorShape(dims)); +} + +} // namespace + +absl::Status MapFunctionSignaturesFromTFSavedModelMLIR( + mlir::ModuleOp module, + llvm::function_ref map_fn) { + // Create bound inputs for each functions. + mlir::SymbolTable symbol_table(module); + absl::Status status = absl::OkStatus(); + module.walk([&symbol_table, map_fn, &status](mlir::func::FuncOp func) { + // Use the exported name as the function name, and skip non-exported + // functions. + auto func_names = mlir::tf_saved_model::GetExportedNames(func); + if (func_names.empty()) return mlir::WalkResult::advance(); + + auto func_type = func.getFunctionType(); + + // Here we walk through each arguments and find out the input/output names, + // and input devices, variables used by this function. + llvm::SmallVector input_names; + llvm::SmallVector< + std::pair, 4> + input_specs; + llvm::SmallVector input_devices; + llvm::SmallVector bound_inputs; + for (unsigned i = 0, e = func.getNumArguments(); i != e; ++i) { + if (auto input_index_path = func.getArgAttrOfType( + i, kTfSavedModelIndexPathAttr)) { + input_names.push_back(ProcessIndexPath(input_index_path)); + auto statusor_spec = ProcessTensorSpec( + mlir::cast(func_type.getInput(i))); + if (!statusor_spec.ok()) { + status = std::move(statusor_spec).status(); + return mlir::WalkResult::interrupt(); + } + input_specs.push_back(std::move(statusor_spec).value()); + if (auto input_device = + func.getArgAttrOfType(i, "tf.device")) { + input_devices.push_back(input_device.getValue()); + } else { + input_devices.push_back(""); + } + } + if (auto* bound_input = + mlir::tf_saved_model::LookupBoundInput(func, i, symbol_table)) { + bound_inputs.push_back(bound_input); + } + } + + llvm::SmallVector output_names; + llvm::SmallVector< + std::pair, 4> + output_specs; + for (unsigned i = 0, e = func.getNumResults(); i != e; ++i) { + if (auto output_index_path = func.getResultAttrOfType( + i, kTfSavedModelIndexPathAttr)) { + output_names.push_back(ProcessIndexPath(output_index_path)); + auto statusor_spec = ProcessTensorSpec( + mlir::cast(func_type.getResult(i))); + if (!statusor_spec.ok()) { + status = std::move(statusor_spec).status(); + return mlir::WalkResult::interrupt(); + } + output_specs.push_back(std::move(statusor_spec).value()); + } + } + + for (auto func_name : func_names) { + TFRTSavedModelSignatureInfo sig_info; + sig_info.func_name = func_name; + sig_info.input_names = input_names; + sig_info.input_specs = input_specs; + sig_info.input_devices = input_devices; + sig_info.output_names = output_names; + sig_info.output_specs = output_specs; + sig_info.bound_inputs = bound_inputs; + map_fn(sig_info); + } + + return mlir::WalkResult::advance(); + }); + + return status; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.h new file mode 100644 index 0000000000000000000000000000000000000000..6060afaa97010c29d5e6f005eda1ff466368e6d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/saved_model/saved_model.h @@ -0,0 +1,81 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_SAVED_MODEL_SAVED_MODEL_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_SAVED_MODEL_SAVED_MODEL_H_ + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/core_runtime/tensor_handle.h" // from @tf_runtime + +namespace tfrt { +class CoreRuntime; +} + +namespace mlir { +class ModuleOp; +} + +namespace tensorflow { + +// TFRTSavedModelSignatureInfo contains the metadata for a signature in the +// savedmodel such as function name, inputs/outputs' names and types. This can +// be used to retrieve these information in a tf_saved_model module. +struct TFRTSavedModelSignatureInfo { + llvm::StringRef func_name; + + // The following are metadata for inputs. + llvm::ArrayRef input_names; + llvm::ArrayRef< + std::pair> + input_specs; + llvm::ArrayRef input_devices; + + // The following are metadata for outputs. + llvm::ArrayRef output_names; + llvm::ArrayRef< + std::pair> + output_specs; + + // The following are metadata for bound_inputs, ie. captures. + llvm::ArrayRef bound_inputs; +}; + +// Apply `map_fn` on every exported function in the module with the +// corresponding signature metadata populated in TFRTSavedModelSignatureInfo for +// the function. +absl::Status MapFunctionSignaturesFromTFSavedModelMLIR( + mlir::ModuleOp module, + llvm::function_ref map_fn); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_SAVED_MODEL_SAVED_MODEL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/test_opkernels.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/test_opkernels.cc new file mode 100644 index 0000000000000000000000000000000000000000..bd0f665914201bda4afcf776c3a5d9f7579ca943 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/test_opkernels.cc @@ -0,0 +1,52 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" + +namespace tensorflow { +namespace tf_mlrt { + +REGISTER_OP("TestAsyncIdentity") + .Input("in: T") + .Output("out: T") + .Attr( + "T: {bfloat16, half, float, double, uint8, int8, int16, uint32, int32, " + "int64, complex64, complex128}") + .SetShapeFn(::tensorflow::shape_inference::UnchangedShape); + +class TestAsyncIdentityKernel : public AsyncOpKernel { + public: + explicit TestAsyncIdentityKernel(OpKernelConstruction* context) + : AsyncOpKernel(context) {} + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override { + const Tensor& in = ctx->input(0); + ctx->set_output(0, in); + done(); + } + + private: + TestAsyncIdentityKernel(const TestAsyncIdentityKernel&) = delete; + void operator=(const TestAsyncIdentityKernel&) = delete; +}; + +REGISTER_KERNEL_BUILDER(Name("TestAsyncIdentity").Device(DEVICE_CPU), + TestAsyncIdentityKernel); + +} // namespace tf_mlrt +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..76db403017429fee432ca2b9cbb7f86f602de1a4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/BUILD @@ -0,0 +1,29 @@ +load("//tensorflow:tensorflow.bzl", "if_oss") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + features = if_oss(["--path=org_tensorflow/tensorflow/compiler/mlir/tfrt"]), + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-mlir-translate", + "//tensorflow/compiler/mlir:tf-opt", + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..d8150f8941fe64d13e15390594e254c931cf8eee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/BUILD @@ -0,0 +1,50 @@ +load("//tensorflow:tensorflow.bzl", "if_oss", "tf_cc_test") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + exclude = ["testdata/**"], + features = if_oss(["--path=org_tensorflow/tensorflow/compiler/mlir/tfrt"]), + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir:tf-mlir-translate", + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) + +tf_cc_test( + name = "update_op_cost_in_tfrt_mlir_test", + srcs = ["update_op_cost_in_tfrt_mlir_test.cc"], + data = [ + "testdata/test.mlir", + ], + deps = [ + "//tensorflow/compiler/mlir/tfrt:transforms/update_op_cost_in_tfrt_mlir", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_sync_opdefs", + "//tensorflow/core:test", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@tf_runtime//:init_tfrt_dialects", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/cost_analysis.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/cost_analysis.mlir new file mode 100644 index 0000000000000000000000000000000000000000..35caad2000ea42a5f7f11920daf4e75821c480b3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/cost_analysis.mlir @@ -0,0 +1,151 @@ +// RUN: tf-tfrt-opt -tfrt-test-cost-analysis -verify-diagnostics %s | FileCheck %s + +// CHECK-LABEL: test_cheap_ops_0 +func.func @test_cheap_ops_0(%arg: tensor) -> (tensor) { + // expected-remark@+1 {{Cost: 1}} + %0 = "tf.Const"() {value = dense<> : tensor<0xi64>} : () -> tensor<0xi64> + // expected-remark@+1 {{Cost: 1}} + %1 = "tf.Const"() {value = dense<"has_login_page_feature"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %2 = "tf.Const"() {value = dense<"num_terms_inside_postform"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %3 = "tf.Const"() {value = dense<"num_terms_outside_postform"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %4 = "tf.Const"() {value = dense<"num_terms_outside_postform_without_bp"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %5 = "tf.Const"() {value = dense<"password_not_in_bp_area"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %6 = "tf.Const"() {value = dense<"query_params_contains_url"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %7 = "tf.Const"() {value = dense<"title_with_login_phase"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %8 = "tf.Const"() {value = dense<"url_contains_login_terms"> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %9 = "tf.Const"() {value = dense<> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // expected-remark@+1 {{Cost: 1}} + %10 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %11 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 19}} + %dense_values:8 = "tf.ParseExample"(%arg, %9, %1, %2, %3, %4, %5, %6, %7, %8, %0, %0, %0, %0, %0, %0, %0, %0) {dense_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>], device = "/job:localhost/replica:0/task:0/device:CPU:0", operandSegmentSizes = array, resultSegmentSizes = array} : (tensor, tensor<0x!tf_type.string>, tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>, tensor<0xi64>) -> (tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor) + // expected-remark@+1 {{Cost: 2}} + %28 = "tf.Cast"(%dense_values#0) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %29 = "tf.Cast"(%dense_values#1) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %30 = "tf.Cast"(%dense_values#2) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %31 = "tf.Cast"(%dense_values#3) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %32 = "tf.Cast"(%dense_values#4) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %33 = "tf.Cast"(%dense_values#5) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %34 = "tf.Cast"(%dense_values#6) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 2}} + %35 = "tf.Cast"(%dense_values#7) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %36 = "tf.ExpandDims"(%28, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %37 = "tf.ExpandDims"(%29, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %38 = "tf.ExpandDims"(%30, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %39 = "tf.ExpandDims"(%31, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %40 = "tf.ExpandDims"(%32, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %41 = "tf.ExpandDims"(%33, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %42 = "tf.ExpandDims"(%34, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %43 = "tf.ExpandDims"(%35, %11) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 10}} + %44 = "tf.ConcatV2"(%36, %37, %38, %39, %40, %41, %42, %43, %10) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor, tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + func.return %44 : tensor +} + +// CHECK-LABEL: test_cheap_ops_1 +func.func @test_cheap_ops_1(%arg: tensor) -> (tensor<4xi32>, tensor) { + // expected-remark@+1 {{Cost: 1}} + %0 = "tf.Const"() {value = dense<8> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %1 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 1}} + %2 = "tf.Const"() {value = dense<64> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %3 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 1}} + %4 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 1}} + %5 = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 1}} + %6 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 9}} + %7 = "tf.Softmax"(%arg) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + // expected-remark@+1 {{Cost: 1}} + %8 = "tf.Shape"(%7) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor<4xi32> + // expected-remark@+1 {{Cost: 1}} + %9 = "tf.StridedSlice"(%8, %5, %4, %4) {begin_mask = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // expected-remark@+1 {{Cost: 1}} + %10 = "tf.StridedSlice"(%8, %3, %6, %4) {begin_mask = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // expected-remark@+1 {{Cost: 5}} + %11 = "tf.Pack"(%9, %0, %10, %2) {axis = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor, tensor) -> tensor<4xi32> + // expected-remark@+1 {{Cost: 1}} + %12 = "tf.StridedSlice"(%8, %6, %1, %4) {begin_mask = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // expected-remark@+1 {{Cost: 5}} + %13 = "tf.Pack"(%9, %0, %10, %12) {axis = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor, tensor) -> tensor<4xi32> + // expected-remark@+1 {{Cost: 1}} + %14 = "tf.Reshape"(%7, %13) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<4xi32>) -> tensor + // expected-remark@+1 {{Cost: 8}} + func.return %11, %14 : tensor<4xi32>, tensor +} + +// CHECK-LABEL: test_expensive_ops +func.func @test_expensive_ops(%arg: tensor) -> tensor { + // expected-remark@+1 {{Cost: 1}} + %0 = "tf.VarHandleOp"() {allowed_devices = [], container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", shared_name = "var"} : () -> tensor>> + // expected-remark@+1 {{Cost: 2}} + %1 = "tf.ReadVariableOp"(%0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor>>) -> tensor<512x512xf32> + // 262657 = 1 + 512 + 512 * 512 + // expected-remark@+1 {{Cost: 262657}} + %2 = "tf.MatMul"(%arg, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor, tensor<512x512xf32>) -> tensor + // expected-remark@+1 {{Cost: 512}} + func.return %2 : tensor +} + +// CHECK-LABEL: test_dynamic_shape +func.func @test_dynamic_shape(%key: tensor, %value: tensor<8xi64>) -> tensor<*xi1> { + // expected-remark@+1 {{Cost: 1}} + %default = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + // expected-remark@+1 {{Cost: 1}} + %0 = "tf.HashTableV2"() {container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", key_dtype = !tf_type.string, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + // expected-remark@+1 {{Cost: 1024}} + %1 = "tf.LookupTableFindV2"(%0, %key, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor<*xi64> + // 17 = 1 + 8 + 8 + // expected-remark@+1 {{Cost: 17}} + %2 = "tf.NotEqual"(%1, %value) {device = "/job:localhost/replica:0/task:0/device:CPU:0", incompatible_shape_error = true} : (tensor<*xi64>, tensor<8xi64>) -> tensor<*xi1> + // expected-remark@+1 {{Cost: 8}} + func.return %2 : tensor<*xi1> +} + +// CHECK-LABEL: test_gather +func.func @test_gather(%arg0 : tensor<1x2x20xf32>, %arg1 : tensor<3x5xi32>) -> (tensor<1x3x5x20xf32>){ + // expected-remark@+1 {{Cost: 1}} + %0 = "tf.Const"() { value = dense<[1]> : tensor<1xi32> } : () -> tensor<1xi32> + // expected-remark@+1 {{Cost: 300}} + %1 = "tf.GatherV2"(%arg0, %arg1, %0) : (tensor<1x2x20xf32>, tensor<3x5xi32>, tensor<1xi32>) -> tensor<1x3x5x20xf32> + // expected-remark@+1 {{Cost: 40}} + func.return %1 : tensor<1x3x5x20xf32> +} + +// CHECK-LABEL: test_sparse_segment_sum +func.func @test_sparse_segment_sum(%indices: tensor<3xi64>, %segment_ids: tensor<3xi64>) -> (tensor){ + // expected-remark@+1 {{Cost: 1}} + %data = "tf.Const"() { value = dense<0.1> : tensor<476x28xf32> } : () -> tensor<476x28xf32> + // expected-remark@+1 {{Cost: 28}} + %1 = "tf.SparseSegmentSum"(%data, %indices, %segment_ids) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<476x28xf32>, tensor<3xi64>, tensor<3xi64>) -> tensor + // expected-remark@+1 {{Cost: 3}} + func.return %1 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/tensor_array_side_effect_analysis.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/tensor_array_side_effect_analysis.mlir new file mode 100644 index 0000000000000000000000000000000000000000..37c92cd9547e080da6f119f09402a79a3cb65c10 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/tensor_array_side_effect_analysis.mlir @@ -0,0 +1,13 @@ +// RUN: tf-tfrt-opt -tfrt-test-tensor-array-effect -verify-diagnostics %s | FileCheck %s + +// CHECK-LABEL: @test_tensor_array_effect +// expected-remark@+1 {{HasAtMostTensorArrayEffect: 1}} +func.func @test_tensor_array_effect(%index: tensor, %size: tensor, %flow_0: tensor, %flow_1: tensor, %handle_0: tensor<2x!tf_type.resource>>, %handle_1: tensor<2x!tf_type.resource>>) -> (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) { + %cst = "tf.Const"() {value = dense<1.1> : tensor<100x512xf32>} : () -> tensor<100x512xf32> + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %x = "tf.TensorArrayReadV3"(%handle_0, %index, %flow_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + %y = "tf.MatMul"(%x, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<100x512xf32>) -> (tensor) + %flow_1_out = "tf.TensorArrayWriteV3"(%handle_1, %index, %y, %flow_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + %next_index = "tf.AddV2"(%index, %one) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %next_index, %size, %flow_0, %flow_1_out, %handle_0, %handle_1 : tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/testdata/test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/testdata/test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c8c583e31727e6c2afe2fcdc3083723716cab133 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/testdata/test.mlir @@ -0,0 +1,7 @@ +func.func @test(%ch: !tfrt.chain, %arg0: !corert.tensorhandle, %arg1_th: !corert.tensorhandle) { + %cpu = corert.get_op_handler %ch "cpu" + %0 = corert.executeop(%cpu) "tf.Relu"(%arg0) { T = f32 } : 1 + %arg1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %arg1_th {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + %1 = tfrt_fallback_async.executeop key(0) cost(100) device("/CPU:0") "tf.Relu"(%arg1) { T = f32 } : 1 + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/update_op_cost_in_tfrt_mlir_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/update_op_cost_in_tfrt_mlir_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..edf9c5843d41b90bbb66738888039f1e16028662 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/analysis/update_op_cost_in_tfrt_mlir_test.cc @@ -0,0 +1,89 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.h" + +#include +#include + +#include +#include +#include "absl/container/flat_hash_map.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tfrt/init_tfrt_dialects.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +constexpr char kCostAttrName[] = "_tfrt_cost"; +constexpr char kOpKeyAttrName[] = "op_key"; + +absl::flat_hash_map GetOpCostMap(mlir::ModuleOp op) { + absl::flat_hash_map op_cost_map; + op.walk([&](mlir::Operation* op) { + const auto cost_attr = op->getAttrOfType(kCostAttrName); + if (!cost_attr) return; + const auto op_key_attr = + op->getAttrOfType(kOpKeyAttrName); + if (!op_key_attr) return; + op_cost_map[op_key_attr.getInt()] = cost_attr.getInt(); + }); + return op_cost_map; +} + +TEST(CostUpdateTest, Basic) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/analysis/testdata/test.mlir"); + + mlir::DialectRegistry registry; + tfrt::RegisterTFRTDialects(registry); + registry.insert(); + registry.insert(); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + // Create a cost recorder with fake cost records. + auto expected_op_cost_map = GetOpCostMap(module.get()); + EXPECT_EQ(expected_op_cost_map.size(), 1); + unsigned int seed = 23579; + for (auto& [op_key, cost] : expected_op_cost_map) { + cost = rand_r(&seed) % 1000; + } + tensorflow::tfrt_stub::CostRecorder cost_recorder; + for (const auto& [op_key, cost] : expected_op_cost_map) { + cost_recorder.RecordCost(op_key, cost); + } + + // Update the TFRT MLIR with the cost recorder. + tfrt_compiler::UpdateOpCostInTfrtMlir(module.get(), cost_recorder); + + // Check the updated costs. + const auto got_op_cost_map = GetOpCostMap(module.get()); + EXPECT_THAT(got_op_cost_map, ::testing::ContainerEq(expected_op_cost_map)); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_fallback_resource_variable_as_captured_tensor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_fallback_resource_variable_as_captured_tensor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5fe31c24c4bfa1b97713659a03c587a33ea42b51 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_fallback_resource_variable_as_captured_tensor.mlir @@ -0,0 +1,26 @@ +// RUN: tf-tfrt-opt -pass-pipeline='builtin.module(tf-executor-to-tfrt-pipeline{target-tpurt=true})' %s | FileCheck %s + +module attributes {tf_saved_model.semantics} { + // CHECK-LABEL: func @main + func.func @main_func() -> (tensor<*xf32> {tf_saved_model.index_path = ["a"]}) attributes {tf_saved_model.exported_names = ["main_func"]} { + %0 = tf_executor.graph { + %outputs_0, %control_0 = tf_executor.island wraps "tf.VarHandleOp"() {container = "", shared_name = ""} : () -> tensor>> + %outputs_1, %control_1 = tf_executor.island wraps "tf.Cast"(%outputs_0) {Truncate = false} : (tensor>>) -> tensor<*x!tf_type.resource> + + // CHECK: tfrt_fallback_async.batch_function device([[DEVICE:.*]]) @batched_func ([[BATCHED_FUNC_ARG:%.*]]) + // CHECK-SAME: Tcaptured = [!corert.resource] + // CHECK-SAME: Tin = [] + // CHECK-SAME: Tout = [f32] + %outputs_2, %control_2 = tf_executor.island wraps "tf.BatchFunction"(%outputs_1) {batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", f = @batched_func, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + tf_executor.fetch %outputs_2 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> + } + func.func private @batched_func(%arg0: tensor<*x!tf_type.resource>) -> tensor { + %0 = tf_executor.graph { + %outputs_0, %control_0 = tf_executor.island wraps "tf.ReadVariableOp"(%arg0) : (tensor<*x!tf_type.resource>) -> tensor + tf_executor.fetch %outputs_0 : tensor + } + func.return %0 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_lowering.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_lowering.mlir new file mode 100644 index 0000000000000000000000000000000000000000..408ab7d507292c247913e013d3849dde112a64b2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/batch_function_lowering.mlir @@ -0,0 +1,27 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline %s | FileCheck %s --dump-input=always + +func.func private @batched_function(%arg0: tensor<1x3xf32> {tf._user_specified_name = "0"}, %arg1: tensor<*x!tf_type.resource>) -> tensor<1x3xf32> attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<1x3xf32> + %1 = "tf.AddV2"(%arg0, %0) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "batch/BatchFunction:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + // CHECK: tfrt_fallback_async.batch_function device("/device:CPU:0") @batched_function + // CHECK-SAME: Tin = [f32] + // CHECK-SAME: Tout = [f32] + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 10 : i64 + // CHECK-SAME: num_batch_threads = 1 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/convert_ref_variables.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/convert_ref_variables.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0b9f94b00340b115fb02161df7dd00677b9ba77f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/convert_ref_variables.mlir @@ -0,0 +1,106 @@ +// RUN: tf-tfrt-opt -tfrt-convert-ref-variables -split-input-file -verify-diagnostics %s | FileCheck %s + +// Test the basic cases where all uses of a ref variable can be converted. + +// CHECK-LABEL: @init +func.func @init() { + // CHECK-NOT: tf.VariableV2 + // CHECK-NOT: tf.Assign + + // CHECK: [[handle:%.*]] = "tf.VarHandleOp" + // CHECK-SAME: shared_name = "x" + // CHECK: "tf.AssignVariableOp"([[handle]], {{%.*}}) + %0 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor + %1 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %2 = "tf.Assign"(%0, %1) {T = i32, device = "", use_locking = true, validate_shape = true} : (tensor, tensor) -> tensor + func.return +} + +// CHECK-LABEL: @inference +func.func @inference() -> tensor { + // CHECK-NOT: tf.VariableV2 + + // CHECK: [[handle:%.*]] = "tf.VarHandleOp" + // CHECK-SAME: shared_name = "x" + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor + %1 = "tf.Identity"(%0) : (tensor) -> tensor + func.return %1 : tensor +} + +// ----- + +// Test the cases when there are both reads and writes, the order of the reads and writes are preserved. + +// CHECK-LABEL: @init +func.func @init() -> tensor { + // CHECK-NOT: tf.VariableV2 + + // CHECK: [[zero:%.*]] = "tf.Const" + // CHECK-SAME: dense<0> + %0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + + // CHECK: [[handle:%.*]] = "tf.VarHandleOp" + // CHECK-SAME: shared_name = "x" + // CHECK-NEXT: "tf.AssignVariableOp"([[handle]], [[zero]]) + // CHECK-NEXT: "tf.ReadVariableOp"([[handle]]) + %1 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor + %2 = "tf.Assign"(%1, %0) {T = i32, device = "", use_locking = true, validate_shape = true} : (tensor, tensor) -> tensor + %3 = "tf.Identity"(%1) : (tensor) -> tensor + + // CHECK: [[one:%.*]] = "tf.Const" + // CHECK-SAME: dense<1> + // CHECK-NEXT: "tf.AssignVariableOp"([[handle]], [[one]]) + // CHECK-NEXT: "tf.ReadVariableOp"([[handle]]) + %4 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %5 = "tf.Assign"(%1, %4) {T = i32, device = "", use_locking = true, validate_shape = true} : (tensor, tensor) -> tensor + %6 = "tf.Identity"(%1) : (tensor) -> tensor + + func.return %6 : tensor +} + +// CHECK-LABEL: @inference +func.func @inference() -> (tensor, tensor, tensor) { + // CHECK-NOT: tf.VariableV2 + + // CHECK: [[handle:%.*]] = "tf.VarHandleOp" + // CHECK-SAME: shared_name = "x" + // CHECK: "tf.ReadVariableOp"([[handle]]) + // CHECK: "tf.ReadVariableOp"([[handle]]) + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor + %1 = "tf.Identity"(%0) : (tensor) -> tensor + %2 = "tf.Identity"(%0) : (tensor) -> tensor + %3 = "tf.Identity"(%0) : (tensor) -> tensor + func.return %1, %2, %3 : tensor, tensor, tensor +} + +// ----- + +// Test report error when the shared_name of the tf.VariableV2 op is empty. + +// CHECK-LABEL: @inference +func.func @inference() -> tensor { + // expected-error @+1 {{unable to convert reference variables with empty shared_names.}} + %0 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = ""} : () -> tensor + %1 = "tf.Identity"(%0) : (tensor) -> tensor + func.return %1 : tensor +} + +// ----- + +// Test conversion when the user is a side-effect-free op. + +// CHECK-LABEL: @side_effect_free_user +func.func @side_effect_free_user() -> tensor<2xi32> { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp" + // CHECK-SAME: shared_name = "x" + // CHECK: [[value0:%.*]] = "tf.ReadVariableOp"([[handle]]) + // CHECK: [[value1:%.*]] = "tf.ReadVariableOp"([[handle]]) + // CHECK: "tf.ConcatV2"([[value1]], [[value0]] + // CHECK-SAME: (tensor, tensor, tensor) -> tensor<2xi32> + %axis = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %0 = "tf.VariableV2"() {container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor + %1 = "tf.ConcatV2"(%0, %0, %axis) : (tensor, tensor, tensor) -> tensor<2xi32> + func.return %1 : tensor<2xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/cross_device_transfer.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/cross_device_transfer.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7e099bfc953c3bc4cfbc255c6dbf5f09ae43e74f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/cross_device_transfer.mlir @@ -0,0 +1,31 @@ +// RUN: tf-tfrt-opt -tfrt-cross-device-transfer %s | FileCheck %s + +// CHECK-LABEL: test_transfer_op_result +func.func @test_transfer_op_result(%arg0: !tfrt.chain) -> () { + // CHECK-NEXT: %[[RESULT_0:.*]] = corert.get_op_handler %[[ARG_0:.*]] "cpu" + %0 = corert.get_op_handler %arg0 "cpu" + // CHECK-NEXT: %[[RESULT_1:.*]] = corert.get_op_handler %[[ARG_0]] "gpu" + %1 = corert.get_op_handler %arg0 "gpu" + // CHECK-NEXT: %[[RESULT_2:.*]] = corert.create_dense_tensor.i32 {shape = [0], value = []} + %2 = corert.create_dense_tensor.i32 {shape = [0], value = []} + // CHECK-NEXT: %[[RESULT_3:.*]] = corert.executeop(%[[RESULT_0]]) "tf.AddV2"(%[[RESULT_2]], %[[RESULT_2]]) + %3 = corert.executeop(%0) "tf.AddV2"(%2, %2) {T = f32, device = "/device:CPU:0"} : 1 + // CHECK-NEXT: %[[RESULT_4:.*]] = tfrt.get_device %[[ARG_0]] {device_name = "/device:GPU:0"} + // CHECK-NEXT: %[[RESULT_5:.*]] = corert.get_dst_tensor_type %[[RESULT_3]], %[[RESULT_4]] + // CHECK-NEXT: %[[RESULT_6:.*]] = corert.transfer %[[RESULT_3]], %[[RESULT_4]], %[[RESULT_5]] + // CHECK-NEXT: %[[RESULT_7:.*]] = corert.executeop(%[[RESULT_1]]) "tf.AddV2"(%[[RESULT_6]], %[[RESULT_6]]) + %4 = corert.executeop(%1) "tf.AddV2"(%3, %3) {T = f32, device = "/device:GPU:0"} : 1 + tfrt.return +} + +// CHECK: func @test_transfer_func_arg(%[[ARG_0:.*]]: !tfrt.chain, %[[ARG_1:.*]]: !corert.tensorhandle +func.func @test_transfer_func_arg(%arg0: !tfrt.chain, %arg1: !corert.tensorhandle {tfrt.device = "/device:CPU:0"}) -> () { + // CHECK-NEXT: %[[RESULT_0:.*]] = corert.get_op_handler %[[ARG_0]] "gpu" + %0 = corert.get_op_handler %arg0 "gpu" + // CHECK-NEXT: %[[RESULT_1:.*]] = tfrt.get_device %[[ARG_0]] {device_name = "/device:GPU:0"} + // CHECK-NEXT: %[[RESULT_2:.*]] = corert.get_dst_tensor_type %[[ARG_1]], %[[RESULT_1]] + // CHECK-NEXT: %[[RESULT_3:.*]] = corert.transfer %[[ARG_1]], %[[RESULT_1]], %[[RESULT_2]] + // CHECK-NEXT: %[[RESULT_4:.*]] = corert.executeop(%[[RESULT_0]]) "tf.AddV2"(%[[RESULT_3]], %[[RESULT_3]]) + %1 = corert.executeop(%0) "tf.AddV2"(%arg1, %arg1) {T = f32, device = "/device:GPU:0"} : 1 + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/deduplicate_if_results.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/deduplicate_if_results.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ff6b06f940680c702d9db52bf25e8fd6d7b53551 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/deduplicate_if_results.mlir @@ -0,0 +1,47 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-deduplicate-if-result %s | FileCheck %s -dump-input=fail + +func.func private @then(%x: tensor, %y: tensor) -> (tensor, tensor) { + func.return %x, %x : tensor, tensor +} + +func.func private @else(%x: tensor, %y: tensor) -> (tensor, tensor) { + func.return %y, %y : tensor, tensor +} + +// CHECK-LABEL: then/tfrt_dedup_results +// CHECK: return {{%.*}} : tensor + +// CHECK-LABEL: else/tfrt_dedup_results +// CHECK: return {{%.*}} : tensor + +// CHECK-LABEL: @basic +func.func @basic(%cond: tensor, %x: tensor, %y: tensor) -> (tensor, tensor) { + // CHECK-NEXT: [[r:%.*]] = "tf.If" + // CHECK-NEXT: return [[r]], [[r]] : tensor, tensor + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @else, then_branch = @then, is_stateless = true} : (tensor, tensor, tensor) -> (tensor, tensor) + return %0, %1 : tensor, tensor +} + +// ----- + +func.func private @unmatched_then(%x: tensor<*xi32>, %y: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + func.return %x, %x : tensor<*xi32>, tensor<*xi32> +} + +func.func private @unmatched_else(%x: tensor, %y: tensor) -> (tensor, tensor) { + func.return %y, %y : tensor, tensor +} + +// CHECK-LABEL: unmatched_then/tfrt_dedup_results +// CHECK: return {{%.*}} : tensor<*xi32> + +// CHECK-LABEL: unmatched_else/tfrt_dedup_results +// CHECK: return {{%.*}} : tensor + +// CHECK-LABEL: @unmatched_then_else_type +func.func @unmatched_then_else_type(%cond: tensor, %x: tensor<*xi32>, %y: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { + // CHECK-NEXT: [[r:%.*]] = "tf.If" + // CHECK-NEXT: return [[r]], [[r]] : tensor<*xi32>, tensor<*xi32> + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @unmatched_else, then_branch = @unmatched_then, is_stateless = true} : (tensor, tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) + return %0, %1 : tensor<*xi32>, tensor<*xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/fuse_tpu_compile_and_execute_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/fuse_tpu_compile_and_execute_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..42cac4f2697a53a1ee0e2a6af6ab8e190ea426ec --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/fuse_tpu_compile_and_execute_ops.mlir @@ -0,0 +1,176 @@ +// RUN: tf-tfrt-opt -verify-diagnostics -split-input-file -tfrt-fuse-tpu-compile-and-execute-ops -canonicalize %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +module attributes {tf_saved_model.semantics} { + +// Test fusing _TPUCompileMlirOp and TPUExecuteOp into TPUCompileMlirAndExecuteOp. + +// CHECK-LABEL: func private @test_fuse_tpu_ops +func.func private @test_fuse_tpu_ops(%arg0: tensor<*xi32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<*xi32> { + // CHECK-NOT: tf._TPUCompileMlirOp + // CHECK-NOT: tf.TPUCompileSucceededAssert + // CHECK-NOT: tf.TPUExecuteOp + + // CHECK-NEXT: %0 = "tf.ReadVariableOp"(%arg1) + // CHECK: [[key:%.*]], [[exec_result:%.*]] = "tf.TPUCompileMlirAndExecute"(%arg0, %0) <{metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, operands_with_static_shape = [], producer_name = "default"}> : (tensor<*xi32>, tensor<*xi32>) -> (tensor<3x!tf_type.string>, tensor<*xi32>) + // CHECK-NEXT: return [[exec_result]] : tensor<*xi32> + + %0 = "tf.ReadVariableOp"(%arg1) {device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %1 = "tf.Shape"(%arg0) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %2 = "tf.Shape"(%0) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %compilation_status, %program = "tf._TPUCompileMlir"(%1, %2) {device = "/CPU:0", metadata = "metadata", mlir_module = "mlir_module"} : (tensor, tensor) -> (tensor, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %3 = "tf.TPUExecute"(%arg0, %0, %program) {device = "/TPU:0"} : (tensor<*xi32>, tensor<*xi32>, tensor<3x!tf_type.string>) -> tensor<*xi32> + func.return %3 : tensor<*xi32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test models using Outside Compilation + +// CHECK-LABEL: func private @test_outside_compilation +func.func private @test_outside_compilation(%arg0: tensor<*xi32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<*xi32> { + // CHECK-NOT: tf._TPUCompileMlirOp + // CHECK-NOT: tf.TPUCompileSucceededAssert + // CHECK-NOT: tf.TPUExecuteOp + + // CHECK-NEXT: %0 = "tf.ReadVariableOp"(%arg1) + // CHECK: [[key:%.*]], [[exec_result:%.*]] = "tf.TPUCompileMlirAndExecute"(%arg0, %0) <{metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, operands_with_static_shape = [], producer_name = "default"}> : (tensor<*xi32>, tensor<*xi32>) -> (tensor<3x!tf_type.string>, tensor<*xi32>) + // CHECK-NEXT: "tf._XlaSendFromHost"(%arg0, %0, [[key]]) <{device_ordinal = 0 : i64, key = "host_compute_channel_0_retvals"}> {_xla_has_host_transfer = true, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*xi32>, tensor<*xi32>, tensor<3x!tf_type.string>) -> () + // CHECK-NEXT: return [[exec_result]] : tensor<*xi32> + %0 = "tf.ReadVariableOp"(%arg1) {device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %1 = "tf.Shape"(%arg0) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %2 = "tf.Shape"(%0) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %compilation_status, %program = "tf._TPUCompileMlir"(%1, %2) {device = "/CPU:0", metadata = "metadata", mlir_module = "mlir_module"} : (tensor, tensor) -> (tensor, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + "tf._XlaSendFromHost"(%arg0, %0, %program) {_xla_has_host_transfer = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", device_ordinal = 0 : i64, key = "host_compute_channel_0_retvals"} : (tensor<*xi32>, tensor<*xi32>, tensor<3x!tf_type.string>) -> () + %3 = "tf.TPUExecute"(%arg0, %0, %program) {device = "/TPU:0"} : (tensor<*xi32>, tensor<*xi32>, tensor<3x!tf_type.string>) -> tensor<*xi32> + func.return %3 : tensor<*xi32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test models with dynamic bounds ops. + +// CHECK-LABEL: func private @test_fuse_dynamic_dimension_ops +func.func private @test_fuse_dynamic_dimension_ops(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<2xi64>, %arg3: tensor, %arg4: tensor) -> tensor<*xi32> { + // CHECK-NOT: tf._TPUCompileMlirOp + // CHECK-NOT: tf.TPUCompileSucceededAssert + // CHECK-NOT: tf.TPUExecuteOp + // CHECK-NOT: tf.SetStaticDimensionBounds + + // CHECK: [[read_result:%.*]] = "tf.ReadVariableOp"(%arg1) + // CHECK: [[shape_result_1:%.*]] = "tf.Shape"(%arg0) {device = "/CPU:0"} : (tensor) -> tensor + // CHECK: [[shape_result_2:%.*]] = "tf.Shape"([[read_result]]) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + // CHECK: [[key:%.*]], [[exec_result:%.*]] = "tf.TPUCompileMlirAndExecute"(%arg0, [[shape_result_2]], %0, %0, %arg2, %arg4, %arg3) <{metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, operands_with_static_shape = [0 : i32, 1 : i32, 3 : i32], producer_name = "default"}> : (tensor, tensor, tensor<*xi32>, tensor<*xi32>, tensor<2xi64>, tensor, tensor) -> (tensor<3x!tf_type.string>, tensor<*xi32>) + // CHECK: [[key_1:%.*]], [[exec_result_1:%.*]] = "tf.TPUCompileMlirAndExecute"(%arg0, %2, %0, %1) <{metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, operands_with_static_shape = [], producer_name = "default"}> : (tensor, tensor, tensor<*xi32>, tensor) -> (tensor<3x!tf_type.string>, tensor<*xi32>) + // CHECK-NEXT: return [[exec_result]] : tensor<*xi32> + %0 = "tf.ReadVariableOp"(%arg1) {device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %dyn_arg0 = "tf.SetStaticDimensionBounds" (%arg0, %arg2) :(tensor, tensor<2xi64>) -> tensor + %dyn_0 = "tf.SetStaticDimensionBounds" (%0, %arg3) :(tensor<*xi32>, tensor) -> tensor + %1 = "tf.Shape"(%dyn_arg0) {device = "/CPU:0"} : (tensor) -> tensor + %2 = "tf.Shape"(%0) {device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %dyn_2 = "tf.SetStaticDimensionBounds" (%2, %arg4) :(tensor, tensor) -> tensor + %compilation_status, %program = "tf._TPUCompileMlir"(%1, %2) {device = "/CPU:0", metadata = "metadata", mlir_module = "mlir_module"} : (tensor, tensor) -> (tensor, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %3 = "tf.TPUExecute"(%dyn_arg0, %dyn_2, %0, %dyn_0, %program) {device = "/TPU:0"} : (tensor, tensor, tensor<*xi32>, tensor, tensor<3x!tf_type.string>) -> tensor<*xi32> + %compilation_status_2, %program_2 = "tf._TPUCompileMlir"(%1, %2) {device = "/CPU:0", metadata = "metadata", mlir_module = "mlir_module"} : (tensor, tensor) -> (tensor, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %4 = "tf.TPUExecute"(%arg0, %2, %0, %1, %program_2) {device = "/TPU:0"} : (tensor, tensor, tensor<*xi32>, tensor, tensor<3x!tf_type.string>) -> tensor<*xi32> + func.return %3 : tensor<*xi32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// CHECK-LABEL: func private @reorder_execute_arg_defining_ops +// CHECK: tf.VarHandleOp +// CHECK-NEXT: tf.ReadVariableOp +// CHECK-NEXT: tf.TPUCompileMlirAndExecute +func.func private @reorder_execute_arg_defining_ops(%arg0: tensor<1x3xf32> {tf.device = "/CPU:0"}) -> (tensor<1x1xf32> {tf.device = "/TPU:0"}) { + %compilation_status, %program = "tf._TPUCompileMlir"() {device = "/CPU:0", metadata = "metadata", mlir_module = "propgram"} : () -> (tensor, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %0 = "tf.VarHandleOp"() {_xla_inferred_shapes = [#tf_type.shape<>], allowed_devices = [], container = "", device = "/CPU:0", shared_name = "y"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/CPU:0"} : (tensor>>) -> tensor<3x1xf32> + %2 = "tf.TPUExecute"(%arg0, %1, %program) {_producer_name = "UNKNOWN", device = "/TPU:0"} : (tensor<1x3xf32>, tensor<3x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + return %2 : tensor<1x1xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { +// CHECK-LABEL: func private @spmd_fuse_mulitple_execute_ops +// CHECK-NEXT: %0 = "tf.VarHandleOp"() +// CHECK-NEXT: %1 = "tf.ReadVariableOp"(%0) +// CHECK-NEXT: %rendezvous_key_base, %results = "tf.TPUCompileMlirAndExecute"(%arg0, %1) +func.func private @spmd_fuse_mulitple_execute_ops(%arg0: tensor<1x4xf32> {tf.device = "/CPU:0"}) -> (tensor<1x1xf32> {tf.device = "/TPU:0"}) { + %cst = "tf.Const"() {device = "/CPU:0", value = dense<1> : tensor} : () -> tensor + %compilation_status, %program:2 = "tf._TPUCompileMlir"() {device = "/CPU:0", metadata = "metadata", mlir_module = "propgram"} : () -> (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %0 = "tf.VarHandleOp"() {_xla_inferred_shapes = [#tf_type.shape<>], allowed_devices = [], container = "", device = "/CPU:0", shared_name = "y"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/CPU:0"} : (tensor>>) -> tensor<2x1xf32> + %2:2 = "tf.Split"(%cst, %arg0) {device = "/CPU:0"} : (tensor, tensor<1x4xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) + %3 = "tf.TPUExecute"(%2#0, %1, %program#0) {_producer_name = "UNKNOWN", device = "/TPU:0"} : (tensor<1x2xf32>, tensor<2x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + %4 = "tf.TPUExecute"(%2#1, %1, %program#1) {_producer_name = "UNKNOWN", device = "/TPU:1"} : (tensor<1x2xf32>, tensor<2x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + return %3 : tensor<1x1xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { +// CHECK-LABEL: func private @spmd_fuse_mulitple_execute_ops_2 +// CHECK-NEXT: %0 = "tf.VarHandleOp"() +// CHECK-NEXT: %1 = "tf.ReadVariableOp"(%0) +// CHECK-NEXT: %rendezvous_key_base, %results = "tf.TPUCompileMlirAndExecute"(%arg0, %1) +func.func private @spmd_fuse_mulitple_execute_ops_2(%arg0: tensor<1x1xf32> {tf.device = "/CPU:0"}) -> (tensor<1x1xf32> {tf.device = "/TPU:0"}) { + %cst = "tf.Const"() {device = "/CPU:0", value = dense<0> : tensor} : () -> tensor + %compilation_status, %program:2 = "tf._TPUCompileMlir"() {device = "/CPU:0", metadata = "metadata", mlir_module = "propgram"} : () -> (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %0 = "tf.VarHandleOp"() {_xla_inferred_shapes = [#tf_type.shape<>], allowed_devices = [], container = "", device = "/CPU:0", shared_name = "y"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/CPU:0"} : (tensor>>) -> tensor<2x1xf32> + %2:2 = "tf.Split"(%cst, %1) {device = "/CPU:0"} : (tensor, tensor<2x1xf32>) -> (tensor<1x1xf32>, tensor<1x1xf32>) + %3 = "tf.TPUExecute"(%arg0, %2#0, %program#0) {_producer_name = "UNKNOWN", device = "/TPU:0"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + %4 = "tf.TPUExecute"(%arg0, %2#1, %program#1) {_producer_name = "UNKNOWN", device = "/TPU:1"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + return %3 : tensor<1x1xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { +// CHECK-LABEL: func private @spmd_fuse_split_nd_ops +// CHECK-NEXT: %0 = "tf.VarHandleOp"() +// CHECK-NEXT: %1 = "tf.ReadVariableOp"(%0) +// CHECK-NEXT: %rendezvous_key_base, %results = "tf.TPUCompileMlirAndExecute"(%arg0, %1) +func.func private @spmd_fuse_split_nd_ops(%arg0: tensor<1x4xf32> {tf.device = "/CPU:0"}) -> (tensor<1x1xf32> {tf.device = "/TPU:0"}) { + %cst = "tf.Const"() {device = "/CPU:0", value = dense<1> : tensor} : () -> tensor + %compilation_status, %program:4 = "tf._TPUCompileMlir"() {device = "/CPU:0", metadata = "metadata", mlir_module = "propgram"} : () -> (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) + "tf.TPUCompileSucceededAssert"(%compilation_status) {device = "/CPU:0"} : (tensor) -> () + %0 = "tf.VarHandleOp"() {_xla_inferred_shapes = [#tf_type.shape<>], allowed_devices = [], container = "", device = "/CPU:0", shared_name = "y"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/CPU:0"} : (tensor>>) -> tensor<1x1xf32> + %2:2 = "tf.Split"(%cst, %arg0) {device = "/CPU:0"} : (tensor, tensor<1x4xf32>) -> (tensor<1x2xf32>, tensor<1x2xf32>) + %3:2 = "tf.Split"(%cst, %2#0) {device = "/CPU:0"} : (tensor, tensor<1x2xf32>) -> (tensor<1x1xf32>, tensor<1x1xf32>) + %4:2 = "tf.Split"(%cst, %2#1) {device = "/CPU:0"} : (tensor, tensor<1x2xf32>) -> (tensor<1x1xf32>, tensor<1x1xf32>) + %5 = "tf.TPUExecute"(%3#0, %1, %program#0) {_producer_name = "UNKNOWN", device = "/TPU:0"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + %6 = "tf.TPUExecute"(%3#1, %1, %program#1) {_producer_name = "UNKNOWN", device = "/TPU:1"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + %7 = "tf.TPUExecute"(%4#0, %1, %program#2) {_producer_name = "UNKNOWN", device = "/TPU:2"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + %8 = "tf.TPUExecute"(%4#1, %1, %program#3) {_producer_name = "UNKNOWN", device = "/TPU:3"} : (tensor<1x1xf32>, tensor<1x1xf32>, tensor<3x!tf_type.string>) -> tensor<1x1xf32> + return %5 : tensor<1x1xf32> +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d1042f49fd4c27881a49bfc7b922987468aab16b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops.mlir @@ -0,0 +1,315 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-lower-tf-savedmodel=hoist-invariant-ops=true %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +module attributes {tf_saved_model.semantics} { + +// Test hoisting varhandle op. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[handle:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "x"}> : () -> tensor>> +// CHECK: [[x:%.*]] = "tf.ReadVariableOp"([[handle]]) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor +// CHECK: "tf._TfrtSetResource"([[x]]) <{index = 0 : i64}> {device = "/CPU:0"} : (tensor) -> () + +// CHECK-LABEL: func @test_hoist_varhandleop +func.func @hoist_varhandleop(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_varhandleop"]} { + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.ReadVariableOp + // CHECK: [[v:%.*]] = "tf._TfrtGetResource"() <{container = [""], indices = [0], shared_name = [""]}> {device = "/CPU:0"} : () -> tensor + // CHECK: [[r:%.*]] = "tf.AddV2"({{.*}}, [[v]]) {device = "/CPU:0"} : (tensor, tensor) -> tensor + // CHECK: return [[r]] + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %x = "tf.ReadVariableOp"(%handle) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test hoisting hash table op. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[handle:%.*]] = "tf.HashTableV2"() +// CHECK-SAME: shared_name = "x" +// CHECK: "tf._TfrtSetResource"([[handle]]) <{index = [[handle_idx:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} +// CHECK: [[x:%.*]] = "tf.LookupTableSizeV2"([[handle]]) +// CHECK: "tf._TfrtSetResource"([[x]]) <{index = [[size_idx:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> () + +// CHECK: func @test_hoist_hash_table +func.func @hoist_hash_table(%arg: tensor {tf_saved_model.index_path = ["input"]}, %default: tensor {tf_saved_model.index_path = ["default"]}) -> (tensor {tf_saved_model.index_path = ["r"]}, tensor<*xi64> {tf_saved_model.index_path = ["r1"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_hash_table"]} { + // CHECK-NOT: tf.HashTableV2 + // CHECK-NOT: tf.LookupTableSizeV2 + // CHECK: [[v:%.*]]:2 = "tf._TfrtGetResource"() <{container = ["", ""], indices = [0, 1], shared_name = [{{.*}}, {{.*}}]}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} + // CHECK: [[r:%.*]] = "tf.LookupTableFindV2"([[v]]#[[handle_idx]] + // CHECK: return [[v]]#[[size_idx]], [[r]] + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "x", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + %1 = "tf.LookupTableSizeV2"(%0) {device = ""} : (tensor) -> tensor + %2 = "tf.LookupTableFindV2"(%0, %arg, %default) {device = "/CPU:0"} : (tensor, tensor, tensor) -> tensor<*xi64> + func.return %1, %2 : tensor, tensor<*xi64> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test hoisting const op. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[const:%.*]] = "tf.Const"() <{value = dense<0> : tensor}> {device = "/CPU:0"} : () -> tensor +// CHECK: [[x:%.*]] = "tf.AddV2"([[const]], [[const]]) {device = "/CPU:0"} : (tensor, tensor) -> tensor +// CHECK: "tf._TfrtSetResource"([[x]]) <{index = 0 : i64}> {device = "/CPU:0"} : (tensor) -> () +// CHECK: [[const_1:%.*]] = "tf.Const"() <{value = dense<1> : tensor}> {device = "/CPU:0"} : () -> tensor +// CHECK: "tf._TfrtSetResource"([[const_1]]) <{index = 1 : i64}> {device = "/CPU:0"} : (tensor) -> () + +// CHECK-LABEL: func @test_hoist_const +func.func @hoist_const(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_const"]} { + // CHECK-NOT: tf.Const + // CHECK: [[v:%.*]] = "tf._TfrtGetResource"() <{container = [""], indices = [0], shared_name = [""]}> {device = "/CPU:0"} : () -> tensor + // CHECK-NEXT: "tf.AddV2"({{.*}}, [[v]]) {device = "/CPU:0"} : (tensor, tensor) -> tensor + // CHECK-NEXT: return + %const = "tf.Const"() {device = "/CPU:0", value = dense<0> : tensor} : () -> tensor + %x = "tf.AddV2"(%const, %const) {device = "/CPU:0"} : (tensor, tensor) -> tensor + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +// CHECK-LABEL: func @test_hoist_const_return +func.func @hoist_const_return(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_const_return"]} { + // CHECK-NOT: tf.Const + // CHECK: [[v:%.*]] = "tf._TfrtGetResource"() <{container = [""], indices = [1], shared_name = [""]}> {device = "/CPU:0"} : () -> tensor + // CHECK-NEXT: return [[v]] + %const = "tf.Const"() {device = "/CPU:0", value = dense<1> : tensor} : () -> tensor + func.return %const : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test not hoisting `tf.BatchFunction`. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[const:%.*]] = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> {device = "/CPU:0"} : () -> tensor<1xi32> +// CHECK: "tf._TfrtSetResource"([[const]]) <{index = 0 : i64}> {device = "/CPU:0"} : (tensor<1xi32>) -> () + +// CHECK-LABEL: func.func private @func_with_batch_function +func.func private @func_with_batch_function() -> tensor<*xi32> attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "StatefulPartitionedCall:0"}} { + // CHECK: "tf._TfrtGetResource"() + %cst = "tf.Const"() <{value = dense<1> : tensor<1xi32>}> {device = "/CPU:0"} : () -> tensor<1xi32> + // CHECK: "tf.BatchFunction" + %0 = "tf.BatchFunction"(%cst) <{allowed_batch_sizes = [1], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", enable_large_batch_splitting = true, f = @_batched, low_priority_allowed_batch_sizes = [], low_priority_batch_timeout_micros = 0 : i64, low_priority_max_batch_size = 0 : i64, low_priority_max_enqueued_batches = 0 : i64, max_batch_size = 1 : i64, max_enqueued_batches = 1 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch_function___inference_signature_wrapper_fn_with_defaults_36"}> {device = "/CPU:0"} : (tensor<1xi32>) -> tensor<*xi32> + return %0 : tensor<*xi32> +} +func.func private @_batched(%arg0: tensor<1xi32>) -> tensor<1xi32> { + return %arg0 : tensor<1xi32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test hoisting write side-effect ops. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[const:%.*]] = "tf.Const"() <{value = dense<0> : tensor}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor +// CHECK: "tf._TfrtSetResource"([[const]]) <{index = [[const_idx:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> () +// CHECK: [[handle:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "x"}> : () -> tensor>> +// CHECK: "tf._TfrtSetResource"([[handle]]) <{index = [[handle_idx:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor>>) -> () + +// CHECK: func @test_hoist_var_read_write +func.func @hoist_var_read_write() -> (tensor {tf_saved_model.index_path = ["x"]}, tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_var_read_write"]} { + // CHECK-NOT: tf.Const + // CHECK-NOT: tf.VarHandleOp + // CHECK: [[v:%.*]]:2 = "tf._TfrtGetResource"() <{container = ["", ""], indices = [0, 1], shared_name = [{{.*}}, {{.*}}]}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> ({{.*}}) + // CHECK: [[x:%.*]] = "tf.ReadVariableOp"([[v]]#[[handle_idx]]) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor + // CHECK-NEXT: "tf.AssignVariable"([[v]]#[[handle_idx]], [[v]]#[[const_idx]]) {device = "/CPU:0"} : (tensor>>, tensor) -> () + // CHECK-NEXT: [[r:%.*]] = "tf.ReadVariableOp"([[v]]#[[handle_idx]]) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor + // CHECK-NEXT: return [[x]], [[r]] + %const = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %x = "tf.ReadVariableOp"(%handle) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor + "tf.AssignVariable"(%handle, %const) {device = "/CPU:0"} : (tensor>>, tensor) -> () + %r = "tf.ReadVariableOp"(%handle) {device = "/CPU:0", dtype = i32} : (tensor>>) -> tensor + func.return %x, %r : tensor, tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test not hoisting read variable op that used by control flow ops if var handle op and read variable op are separated, but still hoists const ops and var handle ops. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[handle:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "x"}> : () -> tensor>> +// CHECK: "tf._TfrtSetResource"([[handle]]) +// CHECK-SAME: index = [[handle_index:.*]] +// CHECK: [[handle1:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "x"}> : () -> tensor>> +// CHECK: "tf._TfrtSetResource"([[handle1]]) +// CHECK-SAME: index = [[handle1_index:.*]] +// CHECK: [[const:%.*]] = "tf.Const"() <{value = dense : tensor}> {device = "/CPU:0"} : () -> tensor +// CHECK: "tf._TfrtSetResource"([[const]]) +// CHECK-SAME: index = [[const_index:.*]] +func.func private @some_func( + %arg: tensor>>) -> tensor { + // CHECK: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @test_not_hoist_stateful_call +func.func @not_hoist_stateful_call(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_not_hoist_stateful_call"]} { + // CHECK-NOT: tf.VarHandleOp + // CHECK: "tf._TfrtGetResource"() + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: tf.StatefulPartitionedCall + %x = "tf.StatefulPartitionedCall"(%handle) {device = "/CPU:0", config = "", config_proto = "", executor_type = "", f = @some_func} : (tensor>>) -> (tensor) + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +// CHECK-LABEL: func @test_not_hoist_if +func.func @not_hoist_if(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_not_hoist_if"]} { + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK-NOT: tf.Const + // CHECK: "tf._TfrtGetResource"() + %cond = "tf.Const"() {device = "/CPU:0", value = dense : tensor} : () -> tensor + // CHECK: tf.If + %x = "tf.If"(%cond, %handle) {then_branch = @some_func, else_branch = @some_func, is_stateless = false} : (tensor, tensor>>) -> tensor + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test hoist var handle op and read variable op in the batch function. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.ReadVariableOp + // CHECK: "tf._TfrtGetResource"() + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.AddV2"(%arg0, %1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %3 = "tf.Identity"(%2) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %3 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK-NOT: tf.VarHandleOp + // CHECK: "tf._TfrtGetResource"() + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + // CHECK: "tf.BatchFunction"(%arg0, %0) + // CHECK: operandSegmentSizes = array + %1 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test not hoisting callees in init functions. + +"tf_saved_model.session_initializer"() {initializers = [@init]} : () -> () + +func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_init"]} { + %var0 = "tf.VarHandleOp"() {container = "", shared_name = "var0"} : () -> tensor>> + %cond = "tf.ReadVariableOp"(%var0) {device = "/CPU:0"} : (tensor>>) -> tensor + %x = "tf.StatefulPartitionedCall"(%cond) {device = "/CPU:0", config = "", config_proto = "", executor_type = "", f = @some_func} : (tensor) -> (tensor) + %var1 = "tf.VarHandleOp"() {container = "", shared_name = "var1"} : () -> tensor>> + "tf.AssignVariable"(%var1, %x) {device = "/CPU:0"} : (tensor>>, tensor) -> () + func.return +} + + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK-NEXT: return + +// CHECK-LABEL: func private @some_func +func.func private @some_func(%arg: tensor) -> tensor { + // CHECK-NOT: tf._TfrtGetResource + %const = "tf.Const"() {device = "/CPU:0", value = dense<1> : tensor } : () -> tensor + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %0 = "tf.ReadVariableOp"(%handle) {device = "/CPU:0"} : (tensor>>) -> tensor + %r = "tf.SelectV2"(%arg, %const, %0) {device = "/CPU:0"} : (tensor, tensor, tensor) -> tensor + func.return %r : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test not hoisting callees in xla launch functions. + +// CHECK-LABEL: func private @xla_func +func.func private @xla_func(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK-NOT: tf._TfrtGetResource + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.AddV2"(%arg0, %1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %3 = "tf.Identity"(%2) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %3 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.XlaLaunch"(%arg0, %0) {device = "/device:GPU:0", function = @xla_func, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> + +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test not hoisting in TPU functions. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK-NEXT: return + +// CHECK-LABEL: func private @func2 +func.func private @func2(%arg: tensor) -> tensor { + // CHECK-NOT: tf._TfrtGetResource + "tf.TPUReplicateMetadata"() {_tpu_replicate = "0", allow_soft_placement = false, computation_shape = [], device = "", device_assignment = [], host_compute_core = [], num_cores_per_replica = 4 : i64, num_replicas = 1 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", tpu_compile_options_proto = "", use_spmd_for_xla_partitioning = true, use_tpu = true} : () -> () + %const = "tf.Const"() {device = "/CPU:0", value = dense<1> : tensor } : () -> tensor + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %0 = "tf.ReadVariableOp"(%handle) {device = "/CPU:0"} : (tensor>>) -> tensor + %r = "tf.SelectV2"(%arg, %const, %0) {device = "/CPU:0"} : (tensor, tensor, tensor) -> tensor + func.return %r : tensor +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops_mlrt.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops_mlrt.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c4358b8a24ad69256c7c69fe69f58aec7c0b5bf1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/hoist_invariant_ops_mlrt.mlir @@ -0,0 +1,31 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-lower-tf-savedmodel="hoist-invariant-ops=true fuse-get-resource-ops=false" %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +module attributes {tf_saved_model.semantics} { + +// Test hoisting hash table op. + +// CHECK-LABEL: func @_tfrt_resource_init +// CHECK: [[handle:%.*]] = "tf.HashTableV2"() +// CHECK-SAME: shared_name = "x" +// CHECK: "tf._TfrtSetResource"([[handle]]) <{index = [[handle_id:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} +// CHECK: [[x:%.*]] = "tf.LookupTableSizeV2"([[handle]]) +// CHECK: "tf._TfrtSetResource"([[x]]) <{index = [[size_id:.*]] : i64}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> () + +// CHECK: func @test_hoist_hash_table +func.func @hoist_hash_table(%arg: tensor {tf_saved_model.index_path = ["input"]}, %default: tensor {tf_saved_model.index_path = ["default"]}) -> (tensor {tf_saved_model.index_path = ["r"]}, tensor<*xi64> {tf_saved_model.index_path = ["r1"]}) + attributes {tf_saved_model.exported_names = ["test_hoist_hash_table"]} { + // CHECK-NOT: tf.HashTableV2 + // CHECK-NOT: tf.LookupTableSizeV2 + // CHECK-DAG: [[v0:%.*]] = "tf._TfrtGetResource"() <{container = [""], indices = [[[handle_id]]], shared_name = [{{.*}}]}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} + // CHECK-DAG: [[v1:%.*]] = "tf._TfrtGetResource"() <{container = [""], indices = [[[size_id]]], shared_name = [{{.*}}]}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} + // CHECK-DAG: [[r:%.*]] = "tf.LookupTableFindV2"([[v0]] + // CHECK-DAG: return [[v1]], [[r]] + %0 = "tf.HashTableV2"() {container = "", device = "", key_dtype = !tf_type.string, shared_name = "x", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + %1 = "tf.LookupTableSizeV2"(%0) {device = ""} : (tensor) -> tensor + %2 = "tf.LookupTableFindV2"(%0, %arg, %default) {device = "/CPU:0"} : (tensor, tensor, tensor) -> tensor<*xi64> + func.return %1, %2 : tensor, tensor<*xi64> +} + +} + +// ----- diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..c28bfba1525b71855bef5fac7feea6f3a15b48d8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/BUILD @@ -0,0 +1,21 @@ +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//mlir:run_lit.sh", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/lower_to_ifrt_restore_variable.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/lower_to_ifrt_restore_variable.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fe613e3614347b9d3923a8f93dfeebb1fe761dc2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/lower_to_ifrt_restore_variable.mlir @@ -0,0 +1,308 @@ +// RUN: tf-tfrt-opt -split-input-file -verify-diagnostics -lower-to-ifrt-restore-variable %s | FileCheck %s + + +// ----- +// single variable + +// CHECK-LABEL: func.func @restore_single() { +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLEY]]) +// CHECK-SAME: {restored_dtypes = [f32], returned_tensor_names = [], truncate_in_cast = array} +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + +module { + func.func @restore_single() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + +// ----- +// single variable: VarHandleOp is before RestoreV2 + +// CHECK-LABEL: func.func @varhandle_before_restore() { +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLEY]]) +// CHECK-SAME: {restored_dtypes = [f32], returned_tensor_names = [], truncate_in_cast = array} +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + +module { + func.func @varhandle_before_restore() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + + +// ----- +// multiple variables + +// CHECK-LABEL: func.func @restore_multiple() { +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() +// CHECK-NEXT: [[NAME:%.*]] = "tf.Const"() +// CHECK-NEXT: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "x", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: [[HANDLEZ:%.*]] = "tf.VarHandleOp"() <{container = "x", shared_name = "z"}> : () -> tensor>> +// CHECK-NEXT: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLEY]], [[HANDLEZ]]) +// CHECK-SAME: {restored_dtypes = [f32, f32], returned_tensor_names = [], truncate_in_cast = array} +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + +module { + func.func @restore_multiple() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<["", ""]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<["y", "z"]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + %0:2 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor<3x1xf32>, tensor<1x3xf32>) + %1 = "tf.VarHandleOp"() <{container = "x", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0#0) : (tensor>>, tensor<3x1xf32>) -> () + %2 = "tf.VarHandleOp"() <{container = "x", shared_name = "z"}> : () -> tensor>> + "tf.AssignVariableOp"(%2, %0#1) : (tensor>>, tensor<1x3xf32>) -> () + return + } +} + +// ----- +// Restored variable is not assigned with a name is an error. + +module { + func.func @unassigned_restore_return_error() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<["", ""]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<["y", "z"]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + //expected-error@below {{'tf.RestoreV2' op expects 2 valid users, but got 1}} + %0:2 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor<3x1xf32>, tensor<1x3xf32>) + %1 = "tf.VarHandleOp"() <{container = "x", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0#0) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + +// ----- +// Restored tensor is consumed by an op other than AssignVariableOp, it is returned as an output by IfrtRestoreVariableOp. + +// CHECK-LABEL: func.func @restore_with_consumer() { +// CHECK-NEXT: %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor +// CHECK-NEXT: %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: %1 = "tf.IfrtRestoreVariableOp"(%cst, %cst_1, %cst_0, %0) <{restored_dtypes = [f32], returned_tensor_names = ["y"], truncate_in_cast = array}> : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>, tensor>>) -> tensor<3x1xf32> +// CHECK-NEXT: %2 = "tf.ReluOp"(%1) : (tensor<3x1xf32>) -> tensor<3x1xf32> +// CHECK-NEXT: %3 = "tf.VarHandleOp"() <{container = "x", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: "tf.AssignVariableOp"(%3, %2) : (tensor>>, tensor<3x1xf32>) -> () +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + + +module { + func.func @restore_with_consumer() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %2 = "tf.ReluOp"(%0) : (tensor<3x1xf32>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "x", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %2) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + + + +// ----- +// variable with cast +// CHECK-LABEL: func.func @restore_with_cast() { +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> +// CHECK-NEXT: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-NEXT: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLEY]]) +// CHECK-SAME: {restored_dtypes = [f32], returned_tensor_names = [], truncate_in_cast = array} +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + +module { + func.func @restore_with_cast() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.Cast"(%0) <{Truncate = false}> : (tensor<3x1xf32>) -> tensor<3x1xbf16> + %2 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%2, %1) : (tensor>>, tensor<3x1xbf16>) -> () + return + } +} + +// ----- +// variable and table lookup +// CHECK-LABEL: func.func @restore_var_and_table() +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"model/foo"> : tensor}> : () -> tensor +// CHECK-NEXT: [[NAMES:%.*]] = "tf.Const"() <{value = dense<["var1", "table1_keys", "table1_vals"]> : tensor<3x!tf_type.string>}> : () -> tensor<3x!tf_type.string> +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<3x!tf_type.string>}> : () -> tensor<3x!tf_type.string> +// CHECK-NEXT: [[VAR1:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "var1"}> : () -> tensor>> +// CHECK-NEXT: [[TABLE1_KEYS:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "table1_keys"}> : () -> tensor>> +// CHECK-NEXT: [[TABLE1_VALS:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "table1_vals"}> : () -> tensor>> +// CHECK-NEXT: [[RETURNED_VALUES:%.*]]:2 = "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAMES]], [[SLICE]], [[VAR1]], [[TABLE1_KEYS]], [[TABLE1_VALS]]) +// CHECK-SAME: <{restored_dtypes = [i32, i64, f32], returned_tensor_names = ["table1_keys", "table1_vals"], truncate_in_cast = array}> : (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>, tensor>>, tensor>>, tensor>>) -> (tensor<*xi64>, tensor<*xf32>) +// CHECK-NEXT: [[TABLE1:%.*]] = "tf.HashTableV2"() <{container = "", key_dtype = i64, shared_name = "table1", value_dtype = f32}> : () -> tensor +// CHECK-NEXT: "tf.LookupTableImportV2"([[TABLE1]], [[RETURNED_VALUES]]#0, [[RETURNED_VALUES]]#1) : (tensor, tensor<*xi64>, tensor<*xf32>) -> () +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return +module { + func.func @restore_var_and_table() { + %cst = "tf.Const"() {value = dense<"model/foo"> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {value = dense<["var1", "table1_keys", "table1_vals"]> : tensor<3x!tf_type.string>} : () -> tensor<3x!tf_type.string> + %cst_1 = "tf.Const"() {value = dense<["", "", ""]> : tensor<3x!tf_type.string>} : () -> tensor<3x!tf_type.string> + %0:3 = "tf.RestoreV2"(%cst, %cst_0, %cst_1) {dtypes = [i32, i64, f32]} : (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) -> (tensor<*xi32>, tensor<*xi64>, tensor<*xf32>) + %1 = "tf.VarHandleOp"() {container = "", shared_name = "var1"} : () -> tensor>> + "tf.AssignVariableOp"(%1, %0#0) : (tensor>>, tensor<*xi32>) -> () + %2 = "tf.HashTableV2"() {container = "", key_dtype = i64, shared_name = "table1", value_dtype = f32} : () -> tensor + "tf.LookupTableImportV2"(%2, %0#1, %0#2) : (tensor, tensor<*xi64>, tensor<*xf32>) -> () + return + } +} + +// ----- +// variable and dense table lookup +// CHECK-LABEL: func.func @restore_var_and_dense_table() +// CHECK-NEXT: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"model/foo"> : tensor}> : () -> tensor +// CHECK-NEXT: [[NAMES:%.*]] = "tf.Const"() <{value = dense<["var1", "table1_keys", "table1_vals"]> : tensor<3x!tf_type.string>}> : () -> tensor<3x!tf_type.string> +// CHECK-NEXT: [[SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<3x!tf_type.string>}> : () -> tensor<3x!tf_type.string> +// CHECK-NEXT: [[VAR1:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "var1"}> : () -> tensor>> +// CHECK-NEXT: [[TABLE1_KEYS:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "table1_keys"}> : () -> tensor>> +// CHECK-NEXT: [[TABLE1_VALS:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "table1_vals"}> : () -> tensor>> +// CHECK-NEXT: [[RETURNED_VALUES:%.*]]:2 = "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NAMES]], [[SLICE]], [[VAR1]], [[TABLE1_KEYS]], [[TABLE1_VALS]]) +// CHECK-SAME: <{restored_dtypes = [i32, i64, f32], returned_tensor_names = ["table1_keys", "table1_vals"], truncate_in_cast = array}> : (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>, tensor>>, tensor>>, tensor>>) -> (tensor<*xi64>, tensor<*xf32>) +// CHECK-NEXT: [[EMPTY_KEY:%.*]] = "tf.Const"() <{value = dense<-1> : tensor}> : () -> tensor +// CHECK-NEXT: [[DEFAULT_VALUE:%.*]] = "tf.Const"() <{value = dense<0.000000e+00> : tensor}> : () -> tensor +// CHECK-NEXT: [[TABLE1:%.*]] = "tf.MutableDenseHashTableV2"(%cst_2, %cst_3) <{container = "", shared_name = "table1", value_dtype = f32}> {key_dtype = i64} : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf.LookupTableImportV2"([[TABLE1]], [[RETURNED_VALUES]]#0, [[RETURNED_VALUES]]#1) : (tensor, tensor<*xi64>, tensor<*xf32>) -> () +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return +module { + func.func @restore_var_and_dense_table() { + %cst = "tf.Const"() {value = dense<"model/foo"> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {value = dense<["var1", "table1_keys", "table1_vals"]> : tensor<3x!tf_type.string>} : () -> tensor<3x!tf_type.string> + %cst_1 = "tf.Const"() {value = dense<["", "", ""]> : tensor<3x!tf_type.string>} : () -> tensor<3x!tf_type.string> + %0:3 = "tf.RestoreV2"(%cst, %cst_0, %cst_1) {dtypes = [i32, i64, f32]} : (tensor, tensor<3x!tf_type.string>, tensor<3x!tf_type.string>) -> (tensor<*xi32>, tensor<*xi64>, tensor<*xf32>) + %1 = "tf.VarHandleOp"() {container = "", shared_name = "var1"} : () -> tensor>> + "tf.AssignVariableOp"(%1, %0#0) : (tensor>>, tensor<*xi32>) -> () + %empty_key = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %default_value = "tf.Const"() {value = dense<0.0> : tensor} : () -> tensor + %2 = "tf.MutableDenseHashTableV2"(%empty_key, %default_value) {container = "", key_dtype = i64, shared_name = "table1", value_dtype = f32} : (tensor, tensor) -> tensor + "tf.LookupTableImportV2"(%2, %0#1, %0#2) : (tensor, tensor<*xi64>, tensor<*xf32>) -> () + return + } +} + + +// ----- +// restored variable assigned to one var handle and then read and assigned to another +// CHECK-LABEL: func.func @restore_and_copy() { +// CHECK-DAG: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> +// CHECK-DAG: [[VAR_Y:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-DAG: [[VAR_Z:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> +// CHECK-DAG: [[NEW_NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<2x!tf_type.string>}> +// CHECK-DAG: [[NEW_SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<2x!tf_type.string>}> +// CHECK: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NEW_NAME]], [[NEW_SLICE]], [[VAR_Y]], [[VAR_Z]]) +// CHECK-SAME: <{restored_dtypes = [f32, f32], returned_tensor_names = [], truncate_in_cast = array}> +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NOT: "tf.ReadVariableOp" +// CHECK-NEXT: return + +module { + func.func @restore_and_copy() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + %read = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor<3x1xf32> + %2 = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> + "tf.AssignVariableOp"(%2, %read) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + +// ----- +// restored variable assigned to one var handle and then read and assigned to another in a different block/function +// CHECK-LABEL: func.func @restore_derived() { +// CHECK-DAG: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_variables"> : tensor}> +// CHECK-DAG: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-DAG: [[HANDLEZ:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> +// CHECK-DAG: [[NEW_NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<2x!tf_type.string>}> +// CHECK-DAG: [[NEW_SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<2x!tf_type.string>}> +// CHECK: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NEW_NAME]], [[NEW_SLICE]], [[HANDLEY]], [[HANDLEZ]]) +// CHECK-SAME: <{restored_dtypes = [f32, f32], returned_tensor_names = [], truncate_in_cast = array}> +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NEXT: return + +module { + func.func @restore_derived() { + %cst = "tf.Const"() <{value = dense<"restore_variables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + return + } + + func.func @derived_init() { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %2 = "tf.Identity"(%1) : (tensor<3x1xf32>) -> tensor<3x1xf32> + %3 = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> + "tf.AssignVariableOp"(%3, %2) : (tensor>>, tensor<3x1xf32>) -> () + return + } +} + +// ----- +// restored variable assigned to one var handle and then read, cast, and assigned to another +// CHECK-LABEL: func.func @restore_and_cast_copy() { +// CHECK-DAG: [[PREFIX:%.*]] = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> +// CHECK-DAG: [[HANDLEY:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> +// CHECK-DAG: [[HANDLEZ:%.*]] = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> +// CHECK-DAG: [[NEW_NAME:%.*]] = "tf.Const"() <{value = dense<"y"> : tensor<2x!tf_type.string>}> +// CHECK-DAG: [[NEW_SLICE:%.*]] = "tf.Const"() <{value = dense<""> : tensor<2x!tf_type.string>}> +// CHECK: "tf.IfrtRestoreVariableOp"([[PREFIX]], [[NEW_NAME]], [[NEW_SLICE]], [[HANDLEY]], [[HANDLEZ]]) +// CHECK-SAME: <{restored_dtypes = [f32, f32], returned_tensor_names = [], truncate_in_cast = array}> +// CHECK-NOT: "tf.RestoreV2" +// CHECK-NOT: "tf.ReadVariableOp" +// CHECK-NEXT: return +module { + func.func @restore_and_cast_copy() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + %read = "tf.ReadVariableOp"(%1) : (tensor>>) -> tensor<3x1xf32> + %cast = "tf.Cast"(%read) <{Truncate = true}> : (tensor<3x1xf32>) -> tensor<3x1xbf16> + %2 = "tf.VarHandleOp"() <{container = "", shared_name = "z"}> : () -> tensor>> + "tf.AssignVariableOp"(%2, %cast) : (tensor>>, tensor<3x1xbf16>) -> () + return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/propagate_static_shapes.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/propagate_static_shapes.mlir new file mode 100644 index 0000000000000000000000000000000000000000..91d1a653cceabe7e8b9c96d082958f32b1d2ee5f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/propagate_static_shapes.mlir @@ -0,0 +1,90 @@ +// RUN: tf-tfrt-opt -split-input-file -propagate-static-shapes %s | FileCheck %s + +// ----- +// CHECK-LABEL: func.func @callee(%arg0: tensor {tf._static_shape_arg_idx = 1 : i32}, %arg1: tensor<2xi64>) -> tensor attributes {tfrt_ifrt_serving.program_id = 123 : i64} +// CHECK: return %arg0 +// CHECK-LABEL: func.func @main +// CHECK-NEXT: %[[C0:.*]] = "tf.Const" +// CHECK-NEXT: %[[C1:.*]] = "tf.IfrtCall"(%arg0, %[[C0]]) <{operandSegmentSizes = array, program_id = 123 : i64, variable_arg_indices = []}> : (tensor, tensor<2xi64>) -> tensor +// CHECK-NEXT: return %[[C1]] + +module { + func.func @callee(%arg0: tensor) -> tensor attributes {tfrt_ifrt_serving.program_id = 123 : i64} { + func.return %arg0 : tensor + } + func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi64>} : () -> tensor<2xi64> + %1 = "tf.SetStaticDimensionBounds"(%arg0, %0) : (tensor, tensor<2xi64>) -> tensor + %2 = "tf.IfrtCall"(%1) {program_id = 123 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor) -> tensor + func.return %2 : tensor + } +} + +// ----- +// CHECK-LABEL: func.func @callee(%arg0: tensor {tf._static_shape_arg_idx = 3 : i32}, %arg1: tensor {tf._static_shape_arg_idx = 4 : i32}, %arg2: tensor, %arg3: tensor<2xi64>, %arg4: tensor<1xi64>) -> (tensor, tensor, tensor) attributes {tfrt_ifrt_serving.program_id = 456 : i64} +// CHECK: return %arg0, %arg1, %arg2 +// CHECK-LABEL: func.func @main +// CHECK-NEXT: %[[C0:.*]] = "tf.Const" +// CHECK-NEXT: %[[C1:.*]] = "tf.Const" +// CHECK-NEXT: %[[R:.*]]:3 = "tf.IfrtCall"(%arg0, %arg1, %arg2, %[[C0]], %[[C1]]) <{operandSegmentSizes = array, program_id = 456 : i64, variable_arg_indices = []}> : (tensor, tensor, tensor, tensor<2xi64>, tensor<1xi64>) -> (tensor, tensor, tensor) +// CHECK-NEXT: return %[[R]]#0, %[[R]]#1, %[[R]]#2 + +module { + func.func @callee(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> (tensor, tensor, tensor) attributes {tfrt_ifrt_serving.program_id = 456 : i64} { + func.return %arg0, %arg1, %arg2 : tensor, tensor, tensor + } + func.func @main(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> (tensor, tensor, tensor) { + %c0 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi64>} : () -> tensor<2xi64> + %c1 = "tf.Const"() {value = dense<4> : tensor<1xi64>} : () -> tensor<1xi64> + %0 = "tf.SetStaticDimensionBounds"(%arg0, %c0) : (tensor, tensor<2xi64>) -> tensor + %1 = "tf.SetStaticDimensionBounds"(%arg1, %c1) : (tensor, tensor<1xi64>) -> tensor + %2:3 = "tf.IfrtCall"(%0, %1, %arg2) {program_id = 456 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor, tensor, tensor) -> (tensor, tensor, tensor) + func.return %2#0, %2#1, %2#2 : tensor, tensor, tensor + } +} + +// ----- +// CHECK-LABEL: func.func @callee(%arg0: tensor {tf._static_shape_arg_idx = 1 : i32}, %arg1: tensor<2xi64>) -> tensor attributes {tfrt_ifrt_serving.program_id = 789 : i64} +// CHECK: return %arg0 +// CHECK-LABEL: func.func @main +// CHECK-NEXT: %[[C0:.*]] = "tf.Const" +// CHECK-NEXT: %[[C1:.*]] = "tf.IfrtCall"(%arg0, %[[C0]]) <{operandSegmentSizes = array, program_id = 789 : i64, variable_arg_indices = []}> : (tensor, tensor<2xi64>) -> tensor +// CHECK-NEXT: %[[C2:.*]] = "tf.IfrtCall"(%arg0, %[[C0]]) <{operandSegmentSizes = array, program_id = 789 : i64, variable_arg_indices = []}> : (tensor, tensor<2xi64>) -> tensor +// CHECK-NEXT: return %[[C2]] + +module { + func.func @callee(%arg0: tensor) -> tensor attributes {tfrt_ifrt_serving.program_id = 789 : i64} { + func.return %arg0 : tensor + } + func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi64>} : () -> tensor<2xi64> + %1 = "tf.SetStaticDimensionBounds"(%arg0, %0) : (tensor, tensor<2xi64>) -> tensor + %2 = "tf.IfrtCall"(%1) {program_id = 789 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor) -> tensor + %3 = "tf.SetStaticDimensionBounds"(%arg0, %0) : (tensor, tensor<2xi64>) -> tensor + %4 = "tf.IfrtCall"(%3) {program_id = 789 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor) -> tensor + func.return %4 : tensor + } +} + +// ----- +// CHECK-LABEL: func.func @callee(%arg0: tensor {tf._static_shape_arg_idx = 1 : i32}, %arg1: tensor<2xi64>) -> tensor attributes {tfrt_ifrt_serving.program_id = 999 : i64} +// CHECK: return %arg0 +// CHECK-LABEL: func.func @main +// CHECK-NEXT: %[[C0:.*]] = "tf.Const" +// CHECK-NEXT: %[[C1:.*]] = "tf.IfrtCall"(%arg0, %[[C0]]) <{operandSegmentSizes = array, program_id = 999 : i64, variable_arg_indices = []}> : (tensor, tensor<2xi64>) -> tensor +// CHECK-NEXT: %[[C2:.*]] = "tf.AsyncIfrtCall"(%arg0, %[[C0]]) <{operandSegmentSizes = array, program_id = 999 : i64, variable_arg_indices = []}> : (tensor, tensor<2xi64>) -> tensor +// CHECK-NEXT: return %[[C2]] + +module { + func.func @callee(%arg0: tensor) -> tensor attributes {tfrt_ifrt_serving.program_id = 999 : i64} { + func.return %arg0 : tensor + } + func.func @main(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<[1, 2]> : tensor<2xi64>} : () -> tensor<2xi64> + %1 = "tf.SetStaticDimensionBounds"(%arg0, %0) : (tensor, tensor<2xi64>) -> tensor + %2 = "tf.IfrtCall"(%1) {program_id = 999 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor) -> tensor + %3 = "tf.SetStaticDimensionBounds"(%arg0, %0) : (tensor, tensor<2xi64>) -> tensor + %4 = "tf.AsyncIfrtCall"(%3) {program_id = 999 : i64, variable_arg_indices = [], operandSegmentSizes = array} : (tensor) -> tensor + func.return %4 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/rewrite_cluster_to_ifrt_call.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/rewrite_cluster_to_ifrt_call.mlir new file mode 100644 index 0000000000000000000000000000000000000000..915c39eb3691fead036742e1cb851b33b2e93aea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/rewrite_cluster_to_ifrt_call.mlir @@ -0,0 +1,126 @@ +// RUN: tf-tfrt-opt -split-input-file -rewrite-cluster-to-ifrt-call %s | FileCheck %s +// TODO(b/316226111): the printer may not guarantee the same order of fields. Rewrite the checks to be less sensitive to proto serialization formats. +// ----- +// Non-SPMD: one input and one output +// +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { +// CHECK-NEXT: "tf.IfrtCall"(%arg0) +// CHECK-SAME: {operandSegmentSizes = array, program_id = [[PROGRAM_ID:.*]] : i64, variable_arg_indices = []} +// CHECK-SAME: (tensor<1x3xf32>) -> tensor<1x3xf32> +// CHECK: return +// +// CHECK: func.func @_ifrt_program__func(%arg0: tensor<1x3xf32>) +// CHECK-SAME: __tpu_compile_metadata_text = "args { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 3 } } kind: PARAMETER sharding { } is_bounded_dynamic_dim: false } retvals { sharding { } } num_replicas: 1 num_cores_per_replica: 1 " +// CHECK-SAME: device_assignment = [] +// CHECK-SAME: tfrt_ifrt_serving.program_id = [[PROGRAM_ID]] : i64 +// CHECK: return + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1704 : i32}} { +func.func @serving_default(%arg0: tensor<1x3xf32>) -> (tensor<1x3xf32>) { + %outputs = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster", device = ""} : () -> tensor + %0 = "tf_device.cluster_func"(%arg0) {_producer_name = "UNKNOWN", func = @_func, input_sharding_configuration = [""], num_cores_per_replica = 1 : i64, device_assignment = [], topology = "", output_sharding_configuration = [""], step_marker_location = "STEP_MARK_AT_ENTRY", use_spmd_for_xla_partitioning = false, use_tpu = true} : (tensor<1x3xf32>) -> (tensor<1x3xf32>) + return %0 : tensor<1x3xf32> +} + +// CHECK-LABEL: @_func +func.func private @_func(%arg0: tensor<1x3xf32>) -> (tensor<1x3xf32>) { + return %arg0 : tensor<1x3xf32> +} +} + + +// ----- +// SPMD: one input and no return +// +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<1x3xf32>) { +// CHECK-NEXT: "tf.IfrtCall"(%arg0) +// CHECK-SAME: {operandSegmentSizes = array, program_id = [[PROGRAM_ID:.*]] : i64, variable_arg_indices = []} +// CHECK-SAME: (tensor<1x3xf32>) -> () +// CHECK: return +// +// CHECK: func.func @_ifrt_program__func(%arg0: tensor<1x3xf32>) +// CHECK-SAME: __tpu_compile_metadata_text = "args { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 3 } } kind: PARAMETER sharding { type: OTHER tile_assignment_dimensions: 2 tile_assignment_dimensions: 1 tile_assignment_devices: 0 tile_assignment_devices: 1 } is_bounded_dynamic_dim: false } num_replicas: 1 num_cores_per_replica: 2 use_spmd_for_xla_partitioning: true " +// CHECK-SAME: device_assignment = [0, 0, 0, 0, 0, 0, 0, 1] +// CHECK-SAME: tfrt_ifrt_serving.program_id = [[PROGRAM_ID]] : i64 +// CHECK: return + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1704 : i32}} { +func.func @serving_default(%arg0: tensor<1x3xf32>) -> () { + %outputs = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster", device = ""} : () -> tensor + "tf_device.cluster_func"(%arg0) {_producer_name = "UNKNOWN", func = @_func, input_sharding_configuration = ["{devices=[2,1]0,1}"], num_cores_per_replica = 2 : i64, device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", output_sharding_configuration = [], step_marker_location = "STEP_MARK_AT_ENTRY", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<1x3xf32>) -> () + return +} + +// CHECK-LABEL: @_func +func.func private @_func(%arg0: tensor<1x3xf32>) -> () { + return +} +} + +// ----- +// Multiple ifrt calls and have two sharded arguments + +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x1xf32> { +// CHECK-NEXT: %0 = "tf.IfrtCall"(%arg1, %arg0) +// CHECK-SAME: {operandSegmentSizes = array, program_id = [[PROGRAM_ID:.*]] : i64, variable_arg_indices = [] +// CHECK-SAME: (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: %1 = "tf.Identity"(%arg1) {device = ""} : (tensor<1x3xf32>) -> tensor<1x3xf32> +// CHECK-NEXT: %2 = "tf.IfrtCall"(%1, %arg0) +// CHECK-SAME: {operandSegmentSizes = array, program_id = [[PROGRAM_ID]] : i64, variable_arg_indices = [] +// CHECK-SAME: (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: %3 = "tf.add"(%0, %2) : (tensor<1x1xf32>, tensor<1x1xf32>) -> tensor<1x1xf32> +// CHECK: return +// +// CHECK: func.func @_ifrt_program__func(%arg0: tensor<1x3xf32>, %arg1: tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK-SAME: device_assignment = [0, 0, 0, 0, 0, 0, 0, 1] +// CHECK-SAME: tfrt_ifrt_serving.program_id = [[PROGRAM_ID]] : i64 +// CHECK-NEXT: %0 = "tf.MatMul"(%arg0, %arg1) +// CHECK: return + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1704 : i32}} { +func.func @serving_default(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x1xf32>) { + %outputs = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster", device = ""} : () -> tensor + %outputs_0 = "tf_device.cluster_func"(%arg1, %arg0) {_producer_name = "UNKNOWN", func = @_func, input_sharding_configuration = ["{devices=[2,1]0,1}", ""], num_cores_per_replica = 2 : i64, device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", output_sharding_configuration = [""], step_marker_location = "STEP_MARK_AT_ENTRY", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %duplicate_arg = "tf.Identity"(%arg1) {device = ""} : (tensor<1x3xf32>) -> tensor<1x3xf32> + %outputs_1 = "tf_device.cluster_func"(%duplicate_arg, %arg0) {_producer_name = "UNKNOWN", func = @_func, input_sharding_configuration = ["{devices=[2,1]0,1}", ""], num_cores_per_replica = 2 : i64, device_assignment = [0, 0, 0, 0, 0, 0, 0, 1], topology = "\0A\04\01\01\01\02\10\01\18\02\22\08\00\00\00\00\00\00\00\01", output_sharding_configuration = [""], step_marker_location = "STEP_MARK_AT_ENTRY", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %outputs_2 = "tf.add"(%outputs_0, %outputs_1): (tensor<1x1xf32>, tensor<1x1xf32>) -> tensor<1x1xf32> + return %outputs_2 : tensor<1x1xf32> +} + +// CHECK-LABEL: @_func +func.func private @_func(%arg0: tensor<1x3xf32>, %arg1: tensor<3x1xf32>) -> (tensor<1x1xf32>) { + %outputs_0 = "tf.MatMul"(%arg0, %arg1) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + return %outputs_0 : tensor<1x1xf32> +} +} + + +// ----- +// Missing topology and device assignment attribute in spmd is ok + +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x1xf32> { +// CHECK-NEXT: %0 = "tf.IfrtCall"(%arg1, %arg0) +// CHECK-SAME: {operandSegmentSizes = array, program_id = [[PROGRAM_ID:.*]] : i64, variable_arg_indices = [] +// CHECK-SAME: (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK: return +// +// CHECK: func.func @_ifrt_program__func(%arg0: tensor<1x3xf32>, %arg1: tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK-SAME: device_assignment = [] +// CHECK-SAME: tfrt_ifrt_serving.program_id = [[PROGRAM_ID]] : i64 +// CHECK-NEXT: %0 = "tf.MatMul"(%arg0, %arg1) +// CHECK: return + +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0", "/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0", "/job:localhost/replica:0/task:0/device:TPU:0", "/job:localhost/replica:0/task:0/device:TPU:1"], tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1704 : i32}} { +func.func @serving_default(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x1xf32>) { + %outputs = "tf.TPUCompilationResult"() {_tpu_compilation_status = "cluster", device = ""} : () -> tensor + %outputs_0 = "tf_device.cluster_func"(%arg1, %arg0) {_producer_name = "UNKNOWN", func = @_func, input_sharding_configuration = ["{devices=[2,1]0,1}", ""], num_cores_per_replica = 2 : i64, output_sharding_configuration = [""], step_marker_location = "STEP_MARK_AT_ENTRY", use_spmd_for_xla_partitioning = true, use_tpu = true} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + return %outputs_0 : tensor<1x1xf32> +} + +// CHECK-LABEL: @_func +func.func private @_func(%arg0: tensor<1x3xf32>, %arg1: tensor<3x1xf32>) -> (tensor<1x1xf32>) { + %outputs_0 = "tf.MatMul"(%arg0, %arg1) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + return %outputs_0 : tensor<1x1xf32> +} +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/sink_variable_as_named_array.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/sink_variable_as_named_array.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d31e77ed8e98443f6c72978bd5ad9f5437daf8f8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/sink_variable_as_named_array.mlir @@ -0,0 +1,130 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-device-decompose-resource-ops -sink-variable-as-named-array %s | FileCheck %s + +// ----- +// Basic test: all variables tensors are for devices and sinked as named ifrt arrays +// +// +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { +// CHECK-NEXT: [[HANDLE2:%.*]] = "tf.VarHandleOp" +// CHECK-NEXT: [[KEY:%.*]], [[FUTURE:%.*]] = "tf.IfrtLoadVariable"([[HANDLE2]]) +// CHECK-SAME: used_by_host = false +// CHECK-NEXT: [[RES:%.*]] = "tf.IfrtCall"([[KEY]], %arg0) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [0 : i32]}> +// CHECK-SAME: : (tensor, tensor<1x3xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: return [[RES]] : tensor<1x1xf32> +// +module { + func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %result = "tf.IfrtCall"(%2, %arg0) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = []}> : (tensor<3x1xf32>, tensor<1x3xf32>) -> (tensor<1x1xf32>) + return %result : tensor<1x1xf32> + } +} + +// ----- +// Variable tensor for host can still be used. +// +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<1x3xf32>) -> (tensor<1x1xf32>, tensor<1x1xf32>) { +// CHECK: "tf.VarHandleOp" +// CHECK-NOT: [[VARIABLE:%.*]] = "tf.ReadVariableOp" +// CHECK-NEXT: [[KEY:%.*]], [[FUTURE:%.*]] = "tf.IfrtLoadVariable" +// CHECK-SAME: used_by_host = true +// CHECK-NEXT: [[MATRES:%.*]] = "tf.MatMul"(%arg0, [[FUTURE]]) +// CHECK-NEXT: [[RES:%.*]] = "tf.IfrtCall"(%arg0, [[KEY]]) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [1 : i32]}> +// CHECK-NEXT: return [[RES]], [[MATRES]] : tensor<1x1xf32>, tensor<1x1xf32> +// +module { + func.func @serving_default(%arg0: tensor<1x3xf32>) -> (tensor<1x1xf32>, tensor<1x1xf32>) { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %3 = "tf.MatMul"(%arg0, %2) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %result = "tf.IfrtCall"(%arg0, %2) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = []}> : (tensor<1x3xf32>, tensor<3x1xf32>) -> (tensor<1x1xf32>) + return %result, %3 : tensor<1x1xf32>, tensor<1x1xf32> + } +} + +// ----- +// Variable tensor is only for host +// +// CHECK-LABEL: func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { +// CHECK: "tf.VarHandleOp" +// CHECK-NOT: tf.ReadVariableOp +// CHECK-NEXT: [[KEY:%.*]], [[FUTURE:%.*]] = "tf.IfrtLoadVariable" +// CHECK-SAME: used_by_host = true +// CHECK-NEXT: [[RES:%.*]] = "tf.MatMul"(%arg0, [[FUTURE]]) +// CHECK-NEXT: return [[RES]] : tensor<1x1xf32> +// +module { + func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %3 = "tf.MatMul"(%arg0, %2) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + return %3: tensor<1x1xf32> + } +} + +// ----- +// Resources that are created in the same module are not sinked. +// +// CHECK-LABEL: func.func @serving_default +// CHECK-NOT: IfrtLoadVariable +// CHECK: "tf.VarHandleOp" +// CHECK-NEXT: "tf.AssignVariableOp" +// CHECK-NEXT: "tf.ReadVariableOp" +// CHECK-NEXT: "tf.StatefulPartitionedCall" +// CHECK-NEXT: return +// +module { + func.func @serving_default() -> tensor<*xi32> { + %cst = "tf.Const"() <{value = dense<"some_test.txt"> : tensor}> : () -> tensor + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "Variable"}> : () -> tensor>> + "tf.AssignVariableOp"(%0, %cst) <{validate_shape = false}> : (tensor>>, tensor) -> () + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<*x!tf_type.string> + %4 = "tf.StatefulPartitionedCall"(%2) <{config = "", config_proto = "", executor_type = "", f = @__initializer}> : (tensor<*x!tf_type.string>) -> tensor<*xi32> + return %4: tensor<*xi32> + } + func.func @__initializer(%arg0: tensor<*x!tf_type.string>) -> tensor { + %0 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + return %0 : tensor + } +} + + +// ----- +// Decomposable Resource Ops usage +// +// CHECK-LABEL: func.func @serving_default +// CHECK: "tf.VarHandleOp" +// CHECK-NEXT: "tf.IfrtLoadVariable" +// CHECK-NEXT: "tf.GatherV2" +// CHECK-NEXT: return +// +module { + func.func @serving_default() -> tensor<1x3xbf16> { + %cst = "tf.Const"() <{value = dense<[1]> : tensor<1xi32>}> : () -> tensor<1xi32> + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "Variable"}> : () -> tensor>> + %1 = "tf.ResourceGather"(%0, %cst) <{batch_dims = 0 : i64, validate_indices = true}> : (tensor>>, tensor<1xi32>) -> tensor<1x3xbf16> + return %1: tensor<1x3xbf16> + } +} + +// ----- +// AsyncIfrtCall test: all variables tensors are for devices and sinked as named ifrt arrays +// +// +// CHECK-LABEL: func.func @serving_default_async(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { +// CHECK-NEXT: [[HANDLE2:%.*]] = "tf.VarHandleOp" +// CHECK-NEXT: [[KEY:%.*]], [[FUTURE:%.*]] = "tf.IfrtLoadVariable"([[HANDLE2]]) +// CHECK-SAME: used_by_host = false +// CHECK-NEXT: [[RES:%.*]] = "tf.AsyncIfrtCall"([[KEY]], %arg0) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [0 : i32]}> +// CHECK-SAME: : (tensor, tensor<1x3xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: return [[RES]] : tensor<1x1xf32> +// +module { + func.func @serving_default_async(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %2 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %result = "tf.AsyncIfrtCall"(%2, %arg0) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = []}> : (tensor<3x1xf32>, tensor<1x3xf32>) -> (tensor<1x1xf32>) + return %result : tensor<1x1xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_device_cleanup.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_device_cleanup.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3c20f7578de4b9d4ed308d0a678a433b3f304c70 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_device_cleanup.mlir @@ -0,0 +1,8 @@ +// RUN: tf-tfrt-opt %s -tf-device-cleanup | FileCheck %s + +// CHECK-LABEL: func @ops_with_device +func.func @ops_with_device() { + %0 = "tf.VarHandleOp"() {container = "", shared_name = "var", device = "/device/..."} : () -> tensor>> + // CHECK-NOT: device = "/device/..." + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_identity_propagation.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_identity_propagation.mlir new file mode 100644 index 0000000000000000000000000000000000000000..1b66f05f575bd64816bd27a5cc20512f80ee9454 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_identity_propagation.mlir @@ -0,0 +1,38 @@ +// RUN: tf-tfrt-opt %s -tf-identity-propagation -canonicalize | FileCheck %s + +// CHECK-LABEL: func @identity +// CHECK-SAME: (%[[ARG0:.*]]: tensor) +func.func @identity(%arg0: tensor) -> tensor { + // CHECK-NOT: "tf.Identity" + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor + // CHECK: return %[[ARG0]] + func.return %0 : tensor +} + +// CHECK-LABEL: func @identity_terminator +// CHECK-SAME: (%[[ARG0:.*]]: tensor) +func.func @identity_terminator(%arg0: tensor) -> (tensor<*xi32>, tensor) { + // CHECK: %[[IDENTITY:.*]] = "tf.Identity" + %0 = "tf.Identity"(%arg0) : (tensor) -> tensor<*xi32> + // CHECK-NOT: "tf.Identity" + %1 = "tf.Identity"(%arg0) : (tensor) -> tensor + // CHECK: return %[[IDENTITY]], %[[ARG0]] + func.return %0, %1 : tensor<*xi32>, tensor +} + +// CHECK-LABEL: func @xla_sharding +func.func @xla_sharding(%arg0: tensor) -> tensor { + // CHECK: %[[OUTPUT:.*]] = "tf.Identity" + %0 = "tf.Identity"(%arg0) {_XlaSharding = ""} : (tensor) -> tensor + // CHECK: return %[[OUTPUT]] + func.return %0 : tensor +} + +// CHECK-LABEL: func @identity_n +// CHECK-SAME: (%[[ARG0:.*]]: tensor, %[[ARG1:.*]]: tensor) +func.func @identity_n(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + // CHECK-NOT: "tf.IdentityN" + %0:2 = "tf.IdentityN"(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + // CHECK: return %[[ARG0]], %[[ARG1]] + func.return %0#0, %0#1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_merging.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_merging.mlir new file mode 100644 index 0000000000000000000000000000000000000000..dd9525df74fc3708d5c8b4e5eecbe18b02b53276 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_merging.mlir @@ -0,0 +1,21 @@ +// RUN: tf-tfrt-opt %s -tf-restore-merging | FileCheck %s + +// CHECK-LABEL: func @single_restore_group +// CHECK-SAME: (%[[ARG0:.*]]: {{.*}}) +func.func @single_restore_group(%arg0: tensor) -> (tensor<*xf32>, tensor<*xi32>) { + %0 = "tf.Const"() {value = dense<"foo"> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %1 = "tf.Const"() {value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %2 = "tf.RestoreV2"(%arg0, %0, %1) : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<*xf32> + + %3 = "tf.Const"() {value = dense<"bar"> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %4 = "tf.Const"() {value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + %5 = "tf.RestoreV2"(%arg0, %3, %4) : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<*xi32> + + // CHECK: %[[NAMES:.*]] = "tf.Const"() <{value = dense<["foo", "bar"]> : tensor<2x!tf_type.string>}> + // CHECK-NEXT: %[[SHAPES:.*]] = "tf.Const"() <{value = dense<""> : tensor<2x!tf_type.string>}> + // CHECK-NEXT: %[[TENSORS:.*]]:2 = "tf.RestoreV2"(%[[ARG0]], %[[NAMES]], %[[SHAPES]]) + // CHECK-SAME: -> (tensor<*xf32>, tensor<*xi32>) + + // CHECK: return %[[TENSORS]]#0, %[[TENSORS]]#1 + func.return %2, %5 : tensor<*xf32>, tensor<*xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_pruning.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_pruning.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0537430e78da529be0bb537fb8279025379f3895 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_pruning.mlir @@ -0,0 +1,25 @@ +// RUN: tf-tfrt-opt -tf-restore-pruning %s | FileCheck %s + +// CHECK-LABEL: func.func @prune_unused_restore +func.func @prune_unused_restore() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + // CHECK-NOT: tf.RestoreV2 + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + return +} + + +// CHECK-LABEL: func.func @used_restore_remains +func.func @used_restore_remains() { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<""> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<"y"> : tensor<1x!tf_type.string>}> : () -> tensor<1x!tf_type.string> + // CHECK: tf.RestoreV2 + %0 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>) -> tensor<3x1xf32> + %1 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) : (tensor>>, tensor<3x1xf32>) -> () + return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_splitting.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_splitting.mlir new file mode 100644 index 0000000000000000000000000000000000000000..367f8081b328fce8e2ade419a8a72128953ab039 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt/tf_restore_splitting.mlir @@ -0,0 +1,18 @@ +// RUN: tf-tfrt-opt %s -tf-restore-splitting | FileCheck %s + +// CHECK-LABEL: func @single_restore +// CHECK-SAME: (%[[ARG0:.*]]: {{.*}}) +func.func @single_restore(%arg0: tensor) -> (tensor<*xf32>, tensor<*xi32>) { + %0 = "tf.Const"() {value = dense<["foo", "bar"]> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + %1 = "tf.Const"() {value = dense<""> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + %2:2 = "tf.RestoreV2"(%arg0, %0, %1) : (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor<*xf32>, tensor<*xi32>) + + // CHECK: %[[FOO_NAME:.*]] = "tf.Const"() <{value = dense<"foo"> : tensor<1x!tf_type.string>}> + // CHECK: %[[FOO:.*]] = "tf.RestoreV2"(%[[ARG0]], %[[FOO_NAME]], {{.*}}) + + // CHECK: %[[BAR_NAME:.*]] = "tf.Const"() <{value = dense<"bar"> : tensor<1x!tf_type.string>}> + // CHECK: %[[BAR:.*]] = "tf.RestoreV2"(%[[ARG0]], %[[BAR_NAME]], {{.*}}) + + // CHECK: return %[[FOO]], %[[BAR]] + func.return %2#0, %2#1 : tensor<*xf32>, tensor<*xi32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt_set_tpu_host_allocator.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt_set_tpu_host_allocator.mlir new file mode 100644 index 0000000000000000000000000000000000000000..97a8677ca0f8c2a7e255fbb32851ea3dbc85e6bc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ifrt_set_tpu_host_allocator.mlir @@ -0,0 +1,56 @@ +// RUN: tf-tfrt-opt -tf-mlrt-ifrt-set-tpu-host-allocator %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +// All arguments are non-variables + +// CHECK-LABEL: func.func @serving_default +// CHECK-NEXT: "tf.MatMul" +// CHECK-SAME: tf_mlrt.custom_device = "tpu_host_device" +// CHECK-NEXT: "tf.MatMul" +// CHECK-SAME: tf_mlrt.custom_device = "tpu_host_device" +// CHECK-NEXT: "tf.IfrtCall" +func.func @serving_default(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x1xf32>) { + %producer_0= "tf.MatMul"(%arg1, %arg0) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %producer_1= "tf.MatMul"(%arg1, %arg0) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %result = "tf.IfrtCall"(%producer_0, %producer_1) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = []}> : (tensor<1x1xf32>, tensor<1x1xf32>) -> (tensor<1x1xf32>) + return %result: tensor<1x1xf32> +} + +// Arguments to the IfrtCall are a mix of non-variables and variables + +// CHECK-LABEL: func.func @serving_default1 +// CHECK-NEXT: "tf.VarHandleOp" +// CHECK-NOT: tf_mlrt.custom_device +// CHECK-NEXT: "tf.ReadVariableOp" +// CHECK-NOT: tf_mlrt.custom_device +// CHECK-NEXT: "tf.MatMul" +// CHECK-SAME: tf_mlrt.custom_device = "tpu_host_device" +// CHECK-NEXT: "tf.MatMul" +// CHECK-NOT: tf_mlrt.custom_device +// CHECK-NEXT: "tf.IfrtCall" +func.func @serving_default1(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x1xf32>) { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) : (tensor>>) -> tensor<3x1xf32> + %2 = "tf.MatMul"(%arg1, %arg0) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %3 = "tf.MatMul"(%arg1, %arg0) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %result = "tf.IfrtCall"(%1, %2) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [0 : i32]}> : (tensor<3x1xf32>, tensor<1x1xf32>) -> (tensor<1x1xf32>) + return %result: tensor<1x1xf32> +} + +// ----- +// Async test: All arguments are non-variables +// +// CHECK-LABEL: func.func @serving_default_async +// CHECK-NEXT: "tf.MatMul" +// CHECK-SAME: tf_mlrt.custom_device = "tpu_host_device" +// CHECK-NEXT: "tf.MatMul" +// CHECK-SAME: tf_mlrt.custom_device = "tpu_host_device" +// CHECK-NEXT: "tf.AsyncIfrtCall" +func.func @serving_default_async(%arg0: tensor<3x1xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x1xf32>) { + %producer_0= "tf.MatMul"(%arg1, %arg0) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %producer_1= "tf.MatMul"(%arg1, %arg0) {transpose_a = false, transpose_b = false} : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %result = "tf.AsyncIfrtCall"(%producer_0, %producer_1) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = []}> : (tensor<1x1xf32>, tensor<1x1xf32>) -> (tensor<1x1xf32>) + return %result: tensor<1x1xf32> +} + + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..a125102991cba8d70bfb673a927eff1bc0972f42 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/BUILD @@ -0,0 +1,47 @@ +load("//tensorflow:tensorflow.bzl", "if_oss", "tf_cc_test") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + exclude = ["testdata/**"], + features = if_oss(["--path=org_tensorflow/tensorflow/compiler/mlir/tfrt"]), + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) + +tf_cc_test( + name = "tfrt_fallback_util_test", + srcs = ["tfrt_fallback_util_test.cc"], + data = [ + "testdata/test.mlir", + ], + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_sync_opdefs", + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_util", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/platform:resource_loader", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@tf_runtime//:init_tfrt_dialects", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/fallback_opt.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/fallback_opt.mlir new file mode 100644 index 0000000000000000000000000000000000000000..beb6251a29530fbe0d84839d4343abdce7bd1acc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/fallback_opt.mlir @@ -0,0 +1,84 @@ +// RUN: tf-tfrt-opt %s | tf-tfrt-opt | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @const_tensor_proto +func.func @const_tensor_proto() -> !tfrt_fallback.tf_tensor { + // CHECK: tfrt_fallback_async.const_tensor_proto "fake serialized proto" + %0 = tfrt_fallback_async.const_tensor_proto "fake serialized proto" + tfrt.return %0 : !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @const_dense_tensor +func.func @const_dense_tensor() -> !tfrt_fallback.tf_tensor { + // CHECK: tfrt_fallback_async.const_dense_tensor + %0 = tfrt_fallback_async.const_dense_tensor dense<0.0> : tensor {_tfrt_cost = 1 : i64} + tfrt.return %0 : !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @const_string_tensor +func.func @const_string_tensor() -> !tfrt_fallback.tf_tensor { + // CHECK: tfrt_fallback_async.const_string_tensor + %0 = tfrt_fallback_async.const_string_tensor {shape = [1, 2], value = ["const", "string"], _tfrt_cost = 1 : i64} + tfrt.return %0 : !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @convert +func.func @convert() -> !corert.tensorhandle { + %0 = corert.const_dense_tensor dense<0.0> : tensor + // CHECK: tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor + %1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %0 {_tfrt_cost = 1 : i64, device = "cpu"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + // CHECK: tfrt_fallback_async.executeop key(0) cost(100) device("cpu") "tf.Relu"(%{{.*}}) {T = f32} : 1 + %2 = tfrt_fallback_async.executeop key(0) cost(100) device("cpu") "tf.Relu"(%1) {T = f32} : 1 + // CHECK: tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle + %3 = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %2 {_tfrt_cost = 1 : i64, device="cpu"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + tfrt.return %3 : !corert.tensorhandle +} + +// CHECK-LABEL: func @predicate +func.func @predicate() -> i1 { + %0 = tfrt_fallback_async.const_dense_tensor dense<0.0> : tensor {_tfrt_cost = 1 : i64} + // CHECK: tfrt_fallback_async.predicate + %1 = tfrt_fallback_async.predicate %0 {_tfrt_cost = 1 : i64, device = "cpu"} + tfrt.return %1 : i1 +} + +// CHECK-LABEL: func @createop +func.func @createop(%in_ch: !tfrt.chain) -> !tfrt.chain { + + // CHECK: [[ch:%.*]] = tfrt_fallback_async.createop(%{{.*}}) key(100) device("cpu") "tf.AddV2"() {T = i32} num_args(2) + %out_ch = tfrt_fallback_async.createop(%in_ch) key(100) device("cpu") "tf.AddV2"() {T = i32} num_args(2) + + // CHECK: tfrt.return [[ch]] + tfrt.return %out_ch: !tfrt.chain +} + +// CHECK-LABEL: func @fallback_resource +func.func @fallback_resource(%ch0: !tfrt.chain) -> !tfrt.chain { + + %ra = tfrt_fallback_async.const_dense_tensor dense<0.0> : tensor {_tfrt_cost = 1 : i64} + %rb = tfrt_fallback_async.const_dense_tensor dense<0.5> : tensor {_tfrt_cost = 1 : i64} + + // CHECK: tfrt_fallback_async.set_resource {{%.*}}, {{%.*}} {device = "cpu", index = 0 : i64} + // CHECK: tfrt_fallback_async.set_resource {{%.*}}, {{%.*}} {device = "cpu", index = 1 : i64} + // CHECK: tfrt_fallback_async.get_resource {{%.*}} {_tfrt_cost = 1 : i64, device = "cpu", indices = [0, 1]} + %ch1 = tfrt_fallback_async.set_resource %ch0, %ra {device = "cpu", index = 0 : i64} + %ch2 = tfrt_fallback_async.set_resource %ch1, %rb {device = "cpu", index = 1 : i64} + %ch3, %a, %b = tfrt_fallback_async.get_resource %ch2 {_tfrt_cost = 1 : i64, device = "cpu", indices = [0 : i64, 1 : i64]} : (!tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + tfrt.return %ch3: !tfrt.chain +} + +// CHECK-LABEL: func @copy_if_small +func.func @copy_if_small(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + // CHECK: tfrt_fallback_async.copy_if_small {{%.*}} {_tfrt_cost = 1 : i64} : (!tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + %0:2 = tfrt_fallback_async.copy_if_small %arg {_tfrt_cost = 1 : i64} : (!tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + tfrt.return %0#0, %0#1 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @custom_allocator +func.func @custom_allocator(%ch: !tfrt.chain, %arg: !tfrt_fallback.tf_tensor, %allocator: !tfrt_fallback.tf_allocator) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) { + // CHECK: tfrt_fallback_async.executeop.allocator(%{{.*}}) key(200) cost(100) device("cpu") "tf.Cast"(%{{.*}}) {Truncate = false, T = f32} : 1 + %0 = tfrt_fallback_async.executeop.allocator(%allocator) key(200) cost(100) device("cpu") "tf.Cast"(%arg) {Truncate = false, T = f32} : 1 + // CHECK: tfrt_fallback_async.executeop.seq.allocator(%{{.*}}, %{{.*}}) key(201) cost(100) device("cpu") "tf.Cast"(%{{.*}}) {Truncate = false, T = i32} : 1 + %out_ch, %1 = tfrt_fallback_async.executeop.seq.allocator(%ch, %allocator) key(201) cost(100) device("cpu") "tf.Cast"(%0) {Truncate = false, T = i32} : 1 + tfrt.return %out_ch, %1 : !tfrt.chain, !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/testdata/test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/testdata/test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c8c583e31727e6c2afe2fcdc3083723716cab133 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/testdata/test.mlir @@ -0,0 +1,7 @@ +func.func @test(%ch: !tfrt.chain, %arg0: !corert.tensorhandle, %arg1_th: !corert.tensorhandle) { + %cpu = corert.get_op_handler %ch "cpu" + %0 = corert.executeop(%cpu) "tf.Relu"(%arg0) { T = f32 } : 1 + %arg1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %arg1_th {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + %1 = tfrt_fallback_async.executeop key(0) cost(100) device("/CPU:0") "tf.Relu"(%arg1) { T = f32 } : 1 + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/tfrt_fallback_util_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/tfrt_fallback_util_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..472363aa321850ce3b5403b53f98b4562abb24d2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/ir/tfrt_fallback_util_test.cc @@ -0,0 +1,63 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_util.h" + +#include +#include +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tfrt/init_tfrt_dialects.h" // from @tf_runtime + +namespace tfrt { +namespace fallback_async { +namespace { + +TEST(SavedModelTest, MapFallbackArgs) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/ir/testdata/test.mlir"); + + mlir::DialectRegistry registry; + RegisterTFRTDialects(registry); + registry.insert(); + registry.insert(); + + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + std::vector> func_and_index; + ForEachArgConsumedByFallback( + module.get(), + [&func_and_index](llvm::StringRef func_name, int arg_index) { + func_and_index.push_back({func_name.str(), arg_index}); + }); + + ASSERT_EQ(func_and_index.size(), 1); + EXPECT_EQ(func_and_index[0].first, "test"); + EXPECT_EQ(func_and_index[0].second, 2); +} + +} // namespace +} // namespace fallback_async +} // namespace tfrt diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..c28bfba1525b71855bef5fac7feea6f3a15b48d8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/BUILD @@ -0,0 +1,21 @@ +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + driver = "//tensorflow/compiler/mlir:run_lit.sh", + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//mlir:run_lit.sh", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/assign_op_key.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/assign_op_key.mlir new file mode 100644 index 0000000000000000000000000000000000000000..76ecf5400bae0211d2f8e7b9adf40c05a7fcfc52 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/assign_op_key.mlir @@ -0,0 +1,49 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-assign-op-key %s | FileCheck %s + +// CHECK-LABEL: func @main +// CHECK: tf.AddV2 +// CHECK-SAME: {__op_key = 0 : i32} + +// CHECK: tf.AddV2 +// CHECK-SAME: {__op_key = 1 : i32} + +// CHECK: tf.AddV2 +// CHECK-SAME: {__op_key = 2 : i32} + +// CHECK: tf.AddV2 +// CHECK-SAME: {__op_key = 3 : i32} + +// CHECK: tf.Sub +// CHECK-SAME: {__op_key = 4 : i32} + +// CHECK: tf.Sub +// CHECK-SAME: {__op_key = 5 : i32} + +// CHECK: tf.Sub +// CHECK-SAME: {__op_key = 6 : i32} + +// CHECK: tf.Sub +// CHECK-SAME: {__op_key = 7 : i32} + + +// CHECK: [[x:%.*]] = "tf.AddV2" +// CHECK-SAME: {__op_key = 8 : i32} + +// CHECK: return [[x]] + +func.func @main(%a: tensor, %b: tensor) -> tensor { + + %a0 = "tf.AddV2"(%a, %a) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %a) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %a) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %a) : (tensor, tensor) -> tensor + + %b0 = "tf.Sub"(%b, %b) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %b) : (tensor, tensor) -> tensor + %b2 = "tf.Sub"(%b1, %b) : (tensor, tensor) -> tensor + %b3 = "tf.Sub"(%b2, %b) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%a3, %b3) : (tensor, tensor) -> tensor + + func.return %c : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/async_while.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/async_while.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a98f8b432ad2ad3c0943c0540904f06ab9e3fa70 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/async_while.mlir @@ -0,0 +1,230 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-async-while %s | FileCheck %s + +// This is a simple case that should be pipelined. + +// CHECK-LABEL: func.func private @"map/while_cond" +func.func private @"map/while_cond"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> tensor { + %0 = "tf.Less"(%loop_count, %max_iterations) : (tensor, tensor) -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func.func private @"map/while_cond/TfMlrtAsyncWhilePredicate"(%arg0: tensor, %arg1: tensor) -> tensor { +// CHECK-NEXT: %0 = "tf.Less"(%arg0, %arg1) : (tensor, tensor) -> tensor +// CHECK-NEXT: return %0 : tensor + +// CHECK-LABEL: func.func private @"map/while_body" +func.func private @"map/while_body"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) { + %cst_1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%loop_count, %cst_1) : (tensor, tensor) -> tensor + %1 = "tf.TensorArrayReadV3"(%handle, %loop_count, %flow_in) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> + %2 = "tf.MatMul"(%1, %matrix) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + return %0, %max_iterations, %handle, %flow_in, %2: tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32> +} + +// CHECK-LABEL: func.func private @"map/while_body/TfMlrtAsyncWhileBody"(%arg0: !mlrt.promise, %arg1: !mlrt.future, %arg2: !mlrt.promise, %arg3: !mlrt.future, %arg4: !mlrt.promise, %arg5: tensor, %arg6: tensor, %arg7: tensor<*xf32>) { +// CHECK-NEXT: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor +// CHECK-NEXT: %0 = "tf_mlrt.tf_await"(%arg1) : (!mlrt.future) -> tensor +// CHECK-NEXT: %1 = "tf.AddV2"(%0, %cst) : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg2, %1) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %2 = "tf.PartitionedCall"(%1, %arg5) <{config = "", config_proto = "", executor_type = "", f = @"map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg0, %2) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %3 = "tf.TensorArrayReadV3"(%arg6, %0, %arg7) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %4 = "tf_mlrt.tf_await"(%arg3) : (!mlrt.future) -> tensor<3x3xf32> +// CHECK-NEXT: %5 = "tf.MatMul"(%3, %4) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg4, %5) : (!mlrt.promise, tensor<3x3xf32>) -> () +// CHECK-NEXT: return + +//CHECK-LABEL: func.func @serving_default +func.func @serving_default(%max_iterations: tensor, %array_handle: tensor, %array_flow: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> (tensor<3x3xf32>, tensor<*xf32>) { + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: %0 = "tf.PartitionedCall"(%cst, %arg0) <{config = "", config_proto = "", executor_type = "", f = @"map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor + // CHECK-NEXT: %1:6 = tf_mlrt.tf_async_while @"map/while_body/TfMlrtAsyncWhileBody"(%0, %cst, %arg3, %arg0, %arg1, %arg2) {invariant_size = 3 : i32} : (tensor, tensor, tensor<3x3xf32>, tensor, tensor, tensor<*xf32>) -> (!mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future) + %1:5 = "tf.While"(%cst_0, %max_iterations, %array_handle, %array_flow, %matrix) {body= @"map/while_body", cond = @"map/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) + // CHECK-NEXT: %2 = "tf_mlrt.tf_await"(%1#5) : (!mlrt.future) -> tensor<*xf32> + // CHECK-NEXT: %3 = "tf_mlrt.tf_await"(%1#2) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: return %3, %2 : tensor<3x3xf32>, tensor<*xf32> + return %1#4, %1#3 : tensor<3x3xf32>, tensor<*xf32> +} + + +//CHECK-LABEL: func.func @multi_while_test +func.func @multi_while_test(%max_iterations: tensor, %array_handle: tensor, %array_flow: tensor<*xf32>, %matrix: tensor<3x3xf32>, %array_handle_2: tensor, %array_flow_2: tensor<*xf32>, %matrix_2: tensor<3x3xf32>) -> (tensor<3x3xf32>, tensor<3x3xf32>) { + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: %0 = "tf.PartitionedCall"(%cst, %arg0) <{config = "", config_proto = "", executor_type = "", f = @"map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor + // CHECK-NEXT: %1:6 = tf_mlrt.tf_async_while @"map/while_body/TfMlrtAsyncWhileBody"(%0, %cst, %arg3, %arg0, %arg1, %arg2) {invariant_size = 3 : i32} : (tensor, tensor, tensor<3x3xf32>, tensor, tensor, tensor<*xf32>) -> (!mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future) + %1:5 = "tf.While"(%cst_0, %max_iterations, %array_handle, %array_flow, %matrix) {body= @"map/while_body", cond = @"map/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) + // CHECK: %2 = "tf.PartitionedCall"(%cst, %arg0) <{config = "", config_proto = "", executor_type = "", f = @"map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor + // CHECK-NEXT: %3:6 = tf_mlrt.tf_async_while @"map/while_body/TfMlrtAsyncWhileBody"(%2, %cst, %arg6, %arg0, %arg4, %arg5) {invariant_size = 3 : i32} : (tensor, tensor, tensor<3x3xf32>, tensor, tensor, tensor<*xf32>) -> (!mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future) + %2:5 = "tf.While"(%cst_0, %max_iterations, %array_handle_2, %array_flow_2, %matrix_2) {body= @"map/while_body", cond = @"map/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) + // CHECK-NEXT: %4 = "tf_mlrt.tf_await"(%1#2) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: %5 = "tf_mlrt.tf_await"(%3#2) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: return %4, %5 : tensor<3x3xf32>, tensor<3x3xf32> + return %1#4, %2#4 : tensor<3x3xf32>, tensor<3x3xf32> +} + +// ----- + +// Test a case in which predicate is updated after mutables and shall not be converted to AsyncWhile. + +// CHECK-LABEL: func.func private @"map/while_cond" +func.func private @"map/while_cond"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> tensor { + %0 = "tf.Less"(%loop_count, %max_iterations) : (tensor, tensor) -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func.func private @"map/while_body" +func.func private @"map/while_body"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) { + %cst_1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %1 = "tf.TensorArrayReadV3"(%handle, %loop_count, %flow_in) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> + %2 = "tf.MatMul"(%1, %matrix) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + // Predicate is update at the last stage. + %0 = "tf.AddV2"(%loop_count, %cst_1) : (tensor, tensor) -> tensor + return %0, %max_iterations, %handle, %flow_in, %2: tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32> +} + +//CHECK-LABEL: func.func @serving_default +func.func @serving_default(%max_iterations: tensor, %array_handle: tensor, %array_flow: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> tensor<3x3xf32> { + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: tf.While + // CHECK-NOT: AsyncWhile + %1:5 = "tf.While"(%cst_0, %max_iterations, %array_handle, %array_flow, %matrix) {body= @"map/while_body", cond = @"map/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) + return %1#4 : tensor<3x3xf32> +} + +// ----- + +// The newly create function name may have conflict with existing functions (very rare). + +// CHECK-LABEL: func.func private @"random/while_cond/TfMlrtAsyncWhilePredicate"(%arg0: tensor, %arg1: tensor) -> tensor { +func.func private @"random/while_cond/TfMlrtAsyncWhilePredicate"(%arg0: tensor, %arg1: tensor) -> tensor { + + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor, tensor) -> tensor + return %0: tensor +} + +// CHECK-LABEL: func.func private @"random/while_body/TfMlrtAsyncWhileBody"(%arg0: tensor, %arg1: tensor) -> tensor { +func.func private @"random/while_body/TfMlrtAsyncWhileBody"(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.AddV2"(%arg0, %arg1) : (tensor, tensor) -> tensor + return %0: tensor +} + +// CHECK-LABEL: func.func private @"random/while_cond" +func.func private @"random/while_cond"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> tensor { + %0 = "tf.Less"(%loop_count, %max_iterations) : (tensor, tensor) -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func.func private @"random/while_cond/TfMlrtAsyncWhilePredicate_0"(%arg0: tensor, %arg1: tensor) -> tensor { +// CHECK-NEXT: %0 = "tf.Less"(%arg0, %arg1) : (tensor, tensor) -> tensor +// CHECK-NEXT: return %0 : tensor + +// CHECK-LABEL: func.func private @"random/while_body" +func.func private @"random/while_body"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) { + %cst_1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%loop_count, %cst_1) : (tensor, tensor) -> tensor + %1 = "tf.TensorArrayReadV3"(%handle, %loop_count, %flow_in) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> + %2 = "tf.MatMul"(%1, %matrix) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + return %0, %max_iterations, %handle, %flow_in, %2: tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32> +} + +// CHECK-LABEL: func.func private @"random/while_body/TfMlrtAsyncWhileBody_1"(%arg0: !mlrt.promise, %arg1: !mlrt.future, %arg2: !mlrt.promise, %arg3: !mlrt.future, %arg4: !mlrt.promise, %arg5: tensor, %arg6: tensor, %arg7: tensor<*xf32>) { +// CHECK-NEXT: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor +// CHECK-NEXT: %0 = "tf_mlrt.tf_await"(%arg1) : (!mlrt.future) -> tensor +// CHECK-NEXT: %1 = "tf.AddV2"(%0, %cst) : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg2, %1) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %2 = "tf.PartitionedCall"(%1, %arg5) <{config = "", config_proto = "", executor_type = "", f = @"random/while_cond/TfMlrtAsyncWhilePredicate_0"}> : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg0, %2) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %3 = "tf.TensorArrayReadV3"(%arg6, %0, %arg7) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %4 = "tf_mlrt.tf_await"(%arg3) : (!mlrt.future) -> tensor<3x3xf32> +// CHECK-NEXT: %5 = "tf.MatMul"(%3, %4) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg4, %5) : (!mlrt.promise, tensor<3x3xf32>) -> () +// CHECK-NEXT: return + +//CHECK-LABEL: func.func @random_serving_default +func.func @random_serving_default(%max_iterations: tensor, %array_handle: tensor, %array_flow: tensor<*xf32>, %matrix: tensor<3x3xf32>) -> (tensor<3x3xf32>, tensor<*xf32>) { + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: %0 = "tf.PartitionedCall"(%cst, %arg0) <{config = "", config_proto = "", executor_type = "", f = @"random/while_cond/TfMlrtAsyncWhilePredicate_0"}> : (tensor, tensor) -> tensor + // CHECK-NEXT: %1:6 = tf_mlrt.tf_async_while @"random/while_body/TfMlrtAsyncWhileBody_1"(%0, %cst, %arg3, %arg0, %arg1, %arg2) {invariant_size = 3 : i32} : (tensor, tensor, tensor<3x3xf32>, tensor, tensor, tensor<*xf32>) -> (!mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future) + %1:5 = "tf.While"(%cst_0, %max_iterations, %array_handle, %array_flow, %matrix) {body= @"random/while_body", cond = @"random/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>) + // CHECK-NEXT: %2 = "tf_mlrt.tf_await"(%1#5) : (!mlrt.future) -> tensor<*xf32> + // CHECK-NEXT: %3 = "tf_mlrt.tf_await"(%1#2) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: return %3, %2 : tensor<3x3xf32>, tensor<*xf32> + return %1#4, %1#3 : tensor<3x3xf32>, tensor<*xf32> +} + +// ----- +// This case test the re-ordering of the while body function to maximize pipelining between iterations. + +// CHECK-LABEL: func.func private @"sort_map/while_cond" +func.func private @"sort_map/while_cond"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>, %handle_2: tensor, %flow_in_2: tensor<*xf32>, %matrix_2: tensor<3x3xf32>, %bound: tensor) -> tensor { + %0 = "tf.Less"(%loop_count, %max_iterations) : (tensor, tensor) -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func.func private @"sort_map/while_cond/TfMlrtAsyncWhilePredicate"(%arg0: tensor, %arg1: tensor) -> tensor { +// CHECK-NEXT: %0 = "tf.Less"(%arg0, %arg1) : (tensor, tensor) -> tensor +// CHECK-NEXT: return %0 : tensor + +// CHECK-LABEL: func.func private @"sort_map/while_body" +// CHECK-NEXT: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor +// CHECK-NEXT: %0 = "tf.AddV2"(%arg0, %cst) : (tensor, tensor) -> tensor +// CHECK-NEXT: %1 = "tf.TensorArrayReadV3"(%arg2, %arg0, %arg3) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %2 = "tf.TensorArrayReadV3"(%arg5, %arg0, %arg6) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %3 = "tf.MatMul"(%1, %arg4) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %4 = "tf.GreaterEqual"(%0, %arg8) : (tensor, tensor) -> tensor +// CHECK-NEXT: %5 = "tf.MatMul"(%3, %arg7) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %6 = "tf.Select"(%4, %3, %arg4) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %7 = "tf.Identity"(%4) : (tensor) -> tensor +// CHECK-NEXT: %8 = "tf.MatMul"(%5, %arg7) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %9 = "tf.Select"(%7, %8, %arg7) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: return %0, %arg1, %arg2, %arg3, %6, %arg5, %arg6, %9, %arg8 +func.func private @"sort_map/while_body"(%loop_count: tensor, %max_iterations: tensor, %handle: tensor, %flow_in: tensor<*xf32>, %matrix: tensor<3x3xf32>, %handle_2: tensor, %flow_in_2: tensor<*xf32>, %matrix_2: tensor<3x3xf32>, %bound: tensor) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor) { + %cst_1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %updated_loop_count = "tf.AddV2"(%loop_count, %cst_1) : (tensor, tensor) -> tensor + %in_matrix = "tf.TensorArrayReadV3"(%handle, %loop_count, %flow_in) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> + %out_matrix = "tf.MatMul"(%in_matrix, %matrix) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %in_matrix1 = "tf.TensorArrayReadV3"(%handle_2, %loop_count, %flow_in_2) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> + %out_matrix1 = "tf.MatMul"(%out_matrix, %matrix_2) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %out_matrix2 = "tf.MatMul"(%out_matrix1, %matrix_2): (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %cond = "tf.GreaterEqual"(%updated_loop_count, %bound) : (tensor, tensor) -> tensor + %result = "tf.Select"(%cond, %out_matrix, %matrix) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %cond2 = "tf.Identity"(%cond) : (tensor) -> tensor + %result2 = "tf.Select"(%cond2, %out_matrix2, %matrix_2) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + return %updated_loop_count, %max_iterations, %handle, %flow_in, %result, %handle_2, %flow_in_2, %result2, %bound: tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor +} + +// CHECK-LABEL: func.func private @"sort_map/while_body/TfMlrtAsyncWhileBody" +// CHECK-NEXT: %cst = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor +// CHECK-NEXT: %0 = "tf_mlrt.tf_await"(%arg1) : (!mlrt.future) -> tensor +// CHECK-NEXT: %1 = "tf.AddV2"(%0, %cst) : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg2, %1) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %2 = "tf.PartitionedCall"(%1, %arg7) <{config = "", config_proto = "", executor_type = "", f = @"sort_map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg0, %2) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: %3 = "tf.TensorArrayReadV3"(%arg8, %0, %arg9) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %4 = "tf.TensorArrayReadV3"(%arg10, %0, %arg11) : (tensor, tensor, tensor<*xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %5 = "tf_mlrt.tf_await"(%arg3) : (!mlrt.future) -> tensor<3x3xf32> +// CHECK-NEXT: %6 = "tf.MatMul"(%3, %5) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %7 = "tf.GreaterEqual"(%1, %arg12) : (tensor, tensor) -> tensor +// CHECK-NEXT: %8 = "tf_mlrt.tf_await"(%arg5) : (!mlrt.future) -> tensor<3x3xf32> +// CHECK-NEXT: %9 = "tf.MatMul"(%6, %8) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %10 = "tf.Select"(%7, %6, %5) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg4, %10) : (!mlrt.promise, tensor<3x3xf32>) -> () +// CHECK-NEXT: %11 = "tf.Identity"(%7) : (tensor) -> tensor +// CHECK-NEXT: %12 = "tf.MatMul"(%9, %8) : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: %13 = "tf.Select"(%11, %12, %8) : (tensor, tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg6, %13) : (!mlrt.promise, tensor<3x3xf32>) -> () +// CHECK-NEXT: return + +//CHECK-LABEL: func.func @sort_serving_default +func.func @sort_serving_default(%max_iterations: tensor, %array_handle: tensor, %array_flow: tensor<*xf32>, %matrix: tensor<3x3xf32>, %bound: tensor) -> (tensor<3x3xf32>, tensor<3x3xf32>, tensor<*xf32>) { + %cst_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + // CHECK: %0 = "tf.PartitionedCall"(%cst, %arg0) <{config = "", config_proto = "", executor_type = "", f = @"sort_map/while_cond/TfMlrtAsyncWhilePredicate"}> : (tensor, tensor) -> tensor + // CHECK-NEXT: %1:10 = tf_mlrt.tf_async_while @"sort_map/while_body/TfMlrtAsyncWhileBody"(%0, %cst, %arg3, %arg3, %arg0, %arg1, %arg2, %arg1, %arg2, %arg4) {invariant_size = 6 : i32} : (tensor, tensor, tensor<3x3xf32>, tensor<3x3xf32>, tensor, tensor, tensor<*xf32>, tensor, tensor<*xf32>, tensor) -> (!mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future, !mlrt.future) + %1:9 = "tf.While"(%cst_0, %max_iterations, %array_handle, %array_flow, %matrix , %array_handle, %array_flow, %matrix, %bound) {body= @"sort_map/while_body", cond = @"sort_map/while_cond", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor, tensor<*xf32>, tensor<3x3xf32>,tensor) -> (tensor, tensor, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor, tensor<*xf32>, tensor<3x3xf32>, tensor) + // CHECK-NEXT: %2 = "tf_mlrt.tf_await"(%1#6) : (!mlrt.future) -> tensor<*xf32> + // CHECK-NEXT: %3 = "tf_mlrt.tf_await"(%1#2) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: %4 = "tf_mlrt.tf_await"(%1#3) : (!mlrt.future) -> tensor<3x3xf32> + // CHECK-NEXT: return %3, %4, %2 : + return %1#4, %1#7, %1#3 : tensor<3x3xf32>, tensor<3x3xf32>, tensor<*xf32> +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/fuse_mlrt_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/fuse_mlrt_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fc4f0c91434473b385c61ab52a8f00f0d3742511 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/fuse_mlrt_ops.mlir @@ -0,0 +1,78 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-fuse %s | FileCheck %s + +// CHECK-LABEL: @main +// CHECK-SAME: ([[f0:%.*]]: !mlrt.future, [[f1:%.*]]: !mlrt.future, [[f2:%.*]]: !mlrt.future) +func.func @main(%f0: !mlrt.future, %f1: !mlrt.future, %f2: !mlrt.future) -> (!tf_mlrt.tensor, !tf_mlrt.tensor, !tf_mlrt.tensor) { + // CHECK-NEXT: [[t:%.*]]:3 = tf_mlrt.await_all [[f0]], [[f1]], [[f2]] + // CHECK-NOT: tf_mlrt.await + // CHECK-NEXT: return [[t]]#0, [[t]]#1, [[t]]#2 + %t0 = tf_mlrt.await %f0 + %t1 = tf_mlrt.await %f1 + %t2 = tf_mlrt.await %f2 + func.return %t0, %t1, %t2 : !tf_mlrt.tensor, !tf_mlrt.tensor, !tf_mlrt.tensor +} + +// ----- + +// CHECK-LABEL: @main +// CHECK-SAME: ([[f0:%.*]]: !mlrt.future, [[f1:%.*]]: !mlrt.future, [[f2:%.*]]: !mlrt.future) +func.func @main(%f0: !mlrt.future, %f1: !mlrt.future, %f2: !mlrt.future) -> (!tf_mlrt.tensor, !tf_mlrt.tensor) { + // CHECK-NEXT: [[t:%.*]]:2 = tf_mlrt.await_all [[f0]], [[f1]] + // CHECK-NOT: tf_mlrt.await + // CHECK-NEXT: [[t2:%.*]] = tf_mlrt.executeop([[t]]#0, [[t]]#1) + // CHECK-NEXT: [[t3:%.*]] = tf_mlrt.await [[f2]] + // CHECK-NEXT: return [[t2]], [[t3]] + %t0 = tf_mlrt.await %f0 + %t1 = tf_mlrt.await %f1 + %t2 = tf_mlrt.executeop(%t0, %t1) {node_def = "AddV2", op_key = 0 : i32} : (!tf_mlrt.tensor, !tf_mlrt.tensor) -> (!tf_mlrt.tensor) + %t3 = tf_mlrt.await %f2 + func.return %t2, %t3 : !tf_mlrt.tensor, !tf_mlrt.tensor +} + +// ----- + +// CHECK-LABEL: @main +// CHECK-SAME: ([[f0:%.*]]: !mlrt.async_handle, [[f1:%.*]]: !mlrt.async_handle, [[f2:%.*]]: !mlrt.async_handle) +func.func @main(%f0: !mlrt.async_handle, %f1: !mlrt.async_handle, %f2: !mlrt.async_handle) -> () { + // CHECK-NEXT: mlrt.await_all_handle [[f0]], [[f1]], [[f2]] + // CHECK-NOT: mlrt.await_handle + // CHECK-NEXT: return + mlrt.await_handle %f0 + mlrt.await_handle %f1 + mlrt.await_handle %f2 + func.return +} + +// ----- + +// CHECK-LABEL: @main +func.func @main() -> (!tf_mlrt.tensor, !tf_mlrt.tensor) { + // CHECK-NEXT: [[r:%.*]]:3 = tf_mlrt.get_resource {indices = [2, 0, 1]} + // CHECK-NEXT: [[v:%.*]] = tf_mlrt.executeop([[r]]#0, [[r]]#1) + // CHECK-NEXT: return [[v]], [[r]]#2 + %0 = tf_mlrt.get_resource {indices = [2]} : !tf_mlrt.tensor + %1 = tf_mlrt.get_resource {indices = [0]} : !tf_mlrt.tensor + %r = tf_mlrt.executeop(%0, %1) {node_def = "AddV2", op_key = 0 : i32} : (!tf_mlrt.tensor, !tf_mlrt.tensor) -> (!tf_mlrt.tensor) + %2 = tf_mlrt.get_resource {indices = [1]} : !tf_mlrt.tensor + func.return %r, %2 : !tf_mlrt.tensor, !tf_mlrt.tensor +} + +// ----- + +// CHECK-LABEL: @fuse_promise_return +// CHECK-SAME: ([[p:%.*]]: !mlrt.promise, [[v:%.*]]: !tf_mlrt.tensor) +func.func @fuse_promise_return(%p: !mlrt.promise, %v: !tf_mlrt.tensor) -> () { + // CHECK: tf_mlrt.promise_return [[p]], [[v]] + tf_mlrt.promise %p, %v + func.return +} + +// ----- + +// CHECK-LABEL: @not_fuse_promise_return +// CHECK-SAME: ([[p:%.*]]: !mlrt.promise, [[v:%.*]]: !tf_mlrt.tensor) +func.func @not_fuse_promise_return(%p: !mlrt.promise, %v: !tf_mlrt.tensor) -> (!tf_mlrt.tensor) { + // CHECK-NOT: tf_mlrt.promise_return + tf_mlrt.promise %p, %v + func.return %v : !tf_mlrt.tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/inline.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/inline.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5d5805a32f52c209892907033a427926e4bc2eb7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/inline.mlir @@ -0,0 +1,50 @@ +// RUN: tf-tfrt-opt -split-input-file -pass-pipeline='builtin.module(tf-to-mlrt, inline)' %s | FileCheck %s -dump-input=fail + +// Test generated tf_mlrt while body and predicate is inlined. + +func.func @then(%x: tensor, %y: tensor, %z: tensor) -> tensor { + return %x: tensor +} + +func.func @else(%x: tensor, %y: tensor, %z: tensor) -> tensor { + return %y: tensor +} + +// CHECK-LABEL: func @while_cond_if +// CHECK: [[cond:%.*]] = tf_mlrt.predicate +// CHECK: [[z:%.*]] = mlrt.cond [[cond]] @then @else +// CHECK: return [[z]] +func.func @while_cond_if(%cond: tensor, %x: tensor, %y: tensor, %z: tensor) -> (tensor) { + %r = "tf.If"(%cond, %x, %y, %z) {then_branch = @then, else_branch = @else, is_stateless = true} : (tensor, tensor, tensor, tensor) -> tensor + return %r : tensor +} + +// CHECK-LABEL: func @while_body_if +func.func @while_body_if(%cond: tensor, %x: tensor, %y: tensor, %z: tensor) -> (tensor, tensor, tensor, tensor) { + %0 = "tf.Const"() {__op_key = 0: i32, device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Add"(%z, %0) {__op_key = 1: i32, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %cond, %x, %y, %1 : tensor, tensor, tensor, tensor +} + +// CHECK-LABEL: func @while_test_if +// CHECK-SAME: -> !tf_mlrt.tensor +func.func @while_test_if(%cond: tensor, %x: tensor, %y: tensor) -> (tensor) { + // CHECK: [[CONST:%.*]] = tf_mlrt.constop {tensor_proto = "\08\03\12\00"} + %cst = "tf.Const"() {__op_key = 2: i32, device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + // Predicate should be inlined. + // CHECK-NEXT: tf_mlrt.predicate + // CHECK-NEXT: mlrt.cond + // CHECK-NEXT: tf_mlrt.predicate + + // CHECK-NEXT: mlrt.while + %0:4 = "tf.While"(%cond, %x, %y, %cst) { cond = @while_cond_if, body = @while_body_if, is_stateless = false, parallel_iterations = 1} : (tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor, tensor) + // CHECK: return + // CHECK-SAME: !tf_mlrt.tensor + func.return %0#3 : tensor +} + +// CHECK-LABEL: func @"while_body_if/tf_mlrt_body" +// CHECK-NOT: call + +// CHECK-LABEL: func @"while_cond_if/tf_mlrt_predicate" +// CHECK-NOT: call diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/parallelization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/parallelization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..257334a47d2af60d1e4a09031f9a7545e58b75e6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/parallelization.mlir @@ -0,0 +1,378 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-parallelization %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +// CHECK-LABEL: func private @main_stream_{{[0-9]*}} +// CHECK-SAME: ({{%.*}}: tensor, [[PROMISE:%.*]]: !mlrt.promise) +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: [[RES:%.*]] = "tf.Sub" +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]], [[RES]]) +// CHECK: return + +// CHECK-LABEL: func @main +// CHECK: [[PROMISE:%.*]], [[FUTURE:%.*]] = "tf_mlrt.allocate_futures" +// CHECK: [[HANDLE:%.*]] = mlrt.async({{%.*}}, [[PROMISE]]) +// CHECK-SAME: callee = @main_stream_{{[0-9]*}} +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: [[x:%.*]] = "tf.AddV2" +// CHECK: [[y:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]) +// CHECK: [[RES:%.*]] = "tf.AddV2"([[x]], [[y]]) +// CHECK: mlrt.await_handle [[HANDLE]] +// CHECK: return [[RES]] + +func.func @main(%a: tensor, %b: tensor) -> tensor { + + %a0 = "tf.AddV2"(%a, %a) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %a) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %a) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %a) : (tensor, tensor) -> tensor + + %b0 = "tf.Sub"(%b, %b) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %b) : (tensor, tensor) -> tensor + %b2 = "tf.Sub"(%b1, %b) : (tensor, tensor) -> tensor + %b3 = "tf.Sub"(%b2, %b) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%a3, %b3) : (tensor, tensor) -> tensor + + func.return %c : tensor +} + +// ----- + +// Test merging child streams + +// CHECK-LABEL: func private @main_stream_{{[0-9]*}} +// CHECK-SAME: ({{%.*}}: tensor, {{%.*}}: tensor, [[PROMISE:%.*]]: !mlrt.promise) +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: [[RES:%.*]] = "tf.Sub" +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]], [[RES]]) +// CHECK: return + +// CHECK-LABEL: func @main +// CHECK: [[PROMISE:%.*]], [[FUTURE:%.*]] = "tf_mlrt.allocate_futures" +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: [[VALUE:%.*]] = "tf.AddV2" +// CHECK: [[HANDLE:%.*]] = mlrt.async([[VALUE]], {{%.*}}, [[PROMISE]]) +// CHECK-SAME: callee = @main_stream_{{[0-9]*}} +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: [[x:%.*]] = "tf.AddV2" +// CHECK: [[y:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]) +// CHECK: [[RES:%.*]] = "tf.AddV2"([[x]], [[y]]) +// CHECK: mlrt.await_handle [[HANDLE]] +// CHECK: return [[RES]] + +func.func @main(%a: tensor, %b: tensor) -> tensor { + + %a0 = "tf.AddV2"(%a, %a) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %a) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %a) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %a) : (tensor, tensor) -> tensor + %a4 = "tf.AddV2"(%a3, %a) : (tensor, tensor) -> tensor + %a5 = "tf.AddV2"(%a4, %a) : (tensor, tensor) -> tensor + %a6 = "tf.AddV2"(%a5, %a) : (tensor, tensor) -> tensor + %a7 = "tf.AddV2"(%a6, %a) : (tensor, tensor) -> tensor + + %b0 = "tf.Sub"(%a3, %b) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %b) : (tensor, tensor) -> tensor + %b2 = "tf.Sub"(%b1, %b) : (tensor, tensor) -> tensor + %b3 = "tf.Sub"(%b2, %b) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%a7, %b3) : (tensor, tensor) -> tensor + + func.return %c : tensor +} + +// ----- + +// Test side-effecting ops + +// CHECK-LABEL: func private @main_stream_{{[0-9]*}} +// CHECK-SAME: ([[ARG:%.*]]: tensor, [[FUTURE:%.*]]: !mlrt.future, [[CONTROL_PROMISE:%.*]]: !mlrt.promise) +// CHECK: [[HANDLE:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]) +// CHECK: "tf.AssignVariableOp"([[HANDLE]], [[ARG]]) +// CHECK-NEXT: mlrt.promise_control [[CONTROL_PROMISE]] + +// CHECK-LABEL: func private @main_stream_{{[0-9]*}} +// CHECK-SAME: ({{%.*}}: tensor, {{%.*}}: tensor, [[FUTURE:%.*]]: !mlrt.future, [[PROMISE:%.*]]: !mlrt.promise) +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: [[V:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]) +// CHECK-NEXT: [[RES:%.*]] = "tf.Sub"({{%.*}}, [[V]]) +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]], [[RES]]) +// CHECK: return + +// CHECK-LABEL: func private @main_stream_{{[0-9]*}} +// CHECK-SAME: ([[CONTROL_FUTURE:%.*]]: !mlrt.future, [[PROMISE:%.*]]: !mlrt.promise, [[PROMISE_HANDLE:%.*]]: !mlrt.promise) +// CHECK: [[HANDLE:%.*]] = "tf.VarHandleOp" +// CHECK-NEXT: "tf_mlrt.tf_promise"([[PROMISE_HANDLE]], [[HANDLE]]) +// CHECK: mlrt.await_control [[CONTROL_FUTURE]] +// CHECK-NEXT: [[V:%.*]] = "tf.ReadVariableOp"([[HANDLE]]) +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]], [[V]]) + +// CHECK-LABEL: func @main +// CHECK: [[PROMISE:%.*]]:3, [[FUTURE:%.*]]:3 = "tf_mlrt.allocate_futures" +// CHECK: [[CONTROL_PROMISE:%.*]], [[CONTROL_FUTURE:%.*]] = "mlrt.allocate_control_futures" +// CHECK: [[ASYNC_HANDLE_0:%.*]] = mlrt.async([[CONTROL_FUTURE]], [[PROMISE]]#0, [[PROMISE]]#1) +// CHECK-SAME: callee = @main_stream_{{[0-9]*}} +// CHECK: [[ASYNC_HANDLE_1:%.*]] = mlrt.async({{%.*}}, {{%.*}}, [[FUTURE]]#0, [[PROMISE]]#2) +// CHECK-SAME: callee = @main_stream_{{[0-9]*}} +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: [[x:%.*]] = "tf.AddV2" +// CHECK: [[ASYNC_HANDLE_2:%.*]] = mlrt.async([[x]], [[FUTURE]]#1, [[CONTROL_PROMISE]]) +// CHECK-SAME: callee = @main_stream_{{[0-9]*}} +// CHECK: [[y:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]#2) +// CHECK: [[RES:%.*]] = "tf.AddV2"([[x]], [[y]]) +// CHECK: mlrt.await_handle [[ASYNC_HANDLE_0]] +// CHECK-NEXT: mlrt.await_handle [[ASYNC_HANDLE_1]] +// CHECK-NEXT: mlrt.await_handle [[ASYNC_HANDLE_2]] +// CHECK-NEXT: return [[RES]] + +func.func @main(%a: tensor, %b: tensor) -> tensor { + %handle = "tf.VarHandleOp"() {container = "", shared_name = "var"} : () -> tensor>> + + %a0 = "tf.AddV2"(%a, %a) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %a) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %a) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %a) : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%handle, %a3) : (tensor>>, tensor) -> () + + %b0 = "tf.Sub"(%a, %b) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %b) : (tensor, tensor) -> tensor + %b2 = "tf.Sub"(%b1, %b) : (tensor, tensor) -> tensor + %var = "tf.ReadVariableOp"(%handle) : (tensor>>) -> tensor + %b3 = "tf.Sub"(%b2, %var) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%a3, %b3) : (tensor, tensor) -> tensor + + func.return %c : tensor +} + +// ----- + +// Test multiple promises and futures + +// CHECK-LABEL: func private @main_stream_1 +// CHECK: mlrt.await_control +// CHECK: "tf.DummySideEffecting"() {id = 4 +// CHECK: return + +// CHECK-LABEL: func private @main_stream_2 +// CHECK: mlrt.await_control +// CHECK: "tf.DummySideEffecting"() {id = 3 +// CHECK: mlrt.promise_control +// CHECK: return + +// CHECK-LABEL: func private @main_stream_3 +// CHECK: mlrt.await_control +// CHECK: "tf.DummySideEffecting"() {id = 2 +// CHECK: mlrt.promise_control +// CHECK: return + +// CHECK-LABEL: func private @main_stream_4 +// CHECK: "tf.DummySideEffecting"() {id = 1 +// CHECK: mlrt.promise_control +// CHECK: return + +// CHECK-LABEL: func @main() +// CHECK: [[PROMISES:%.*]]:3, [[FUTURES:%.*]]:3 = "mlrt.allocate_control_futures" +// CHECK: mlrt.async([[PROMISES]]#2) {callee = @main_stream_4 +// CHECK: mlrt.async([[FUTURES]]#2, [[PROMISES]]#1) {callee = @main_stream_3 +// CHECK: mlrt.async([[FUTURES]]#1, [[PROMISES]]#0) {callee = @main_stream_2 +// CHECK: mlrt.async([[FUTURES]]#0) {callee = @main_stream_1 +// CHECK: mlrt.await_handle +// CHECK: mlrt.await_handle +// CHECK: mlrt.await_handle +// CHECK: mlrt.await_handle + +func.func @main() { + "tf.DummySideEffecting"() {id = 1} : () -> () + "tf.DummySideEffecting"() {id = 2} : () -> () + "tf.DummySideEffecting"() {id = 3} : () -> () + "tf.DummySideEffecting"() {id = 4} : () -> () + func.return +} + +// ----- + +// Test correctness when there are both data and control promises in a stream function. + +// CHECK-LABEL: func private @main_stream_1 +// CHECK-SAME: ([[PROMISE:%.*]]: !mlrt.promise, [[CONTROL_PROMISE:%.*]]: !mlrt.promise) +// CHECK: tf.DummySideEffecting +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]] +// CHECK: mlrt.promise_control [[CONTROL_PROMISE]] + +func.func @main() -> tensor { + %v = "tf.DummySideEffecting"() {id = 1} : () -> tensor + + %w = "tf.DummySideEffecting"() {id = 2} : () -> tensor + %r = "tf.AddV2"(%w, %v) : (tensor, tensor) -> tensor + func.return %r : tensor +} + +// ----- + +// Test inputs to the child streams are merged to the parent streams + +// CHECK-LABEL: func private @main_stream_1 +// CHECK-SAME: ([[INPUT0:%.*]]: tensor, [[INPUT1:%.*]]: tensor +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: mlrt.async({{%.*}}, [[INPUT1]] + +// CHECK-LABEL: func @main +func.func @main(%a: tensor, %b: tensor) -> tensor { + + %a0 = "tf.AddV2"(%a, %a) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %a) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %a) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %a) : (tensor, tensor) -> tensor + + %b0 = "tf.Sub"(%b, %b) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %b) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%b1, %a) : (tensor, tensor) -> tensor + + %b2 = "tf.Sub"(%b1, %b) : (tensor, tensor) -> tensor + %b3 = "tf.Sub"(%b2, %b) : (tensor, tensor) -> tensor + + %d = "tf.AddN"(%a3, %b3, %c) : (tensor, tensor, tensor) -> tensor + func.return %d : tensor +} + +// ----- + +// Test that constants are copied instead of using promise/await. + +// CHECK-LABEL: func private @main_stream_1 +// CHECK-SAME: ({{%.*}}: tensor, [[PROMISE:%.*]]: !mlrt.promise) +// CHECK: tf._TfrtGetResource +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: tf.Sub +// CHECK: [[RES:%.*]] = "tf.Sub" +// CHECK: "tf_mlrt.tf_promise"([[PROMISE]], [[RES]]) +// CHECK: return + +// CHECK-NOT: func private @main_stream + +// CHECK-LABEL: func @main +// CHECK: [[PROMISE:%.*]], [[FUTURE:%.*]] = "tf_mlrt.allocate_futures" +// CHECK-NEXT: [[HANDLE:%.*]] = mlrt.async({{%.*}}, [[PROMISE]]) +// CHECK-SAME: callee = @main_stream_1 +// CHECK: tf._TfrtGetResource +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: tf.AddV2 +// CHECK: [[x:%.*]] = "tf.AddV2" +// CHECK: [[y:%.*]] = "tf_mlrt.tf_await"([[FUTURE]]) +// CHECK: [[RES:%.*]] = "tf.AddV2"([[x]], [[y]]) +// CHECK: mlrt.await_handle [[HANDLE]] +// CHECK: return [[RES]] + +func.func @main(%a: tensor, %b: tensor) -> tensor { + + %c0 = "tf._TfrtGetResource"() {indices = [0], shared_name = [""], container = [""]} : () -> (tensor) + + %a0 = "tf.AddV2"(%a, %c0) : (tensor, tensor) -> tensor + %a1 = "tf.AddV2"(%a0, %c0) : (tensor, tensor) -> tensor + %a2 = "tf.AddV2"(%a1, %c0) : (tensor, tensor) -> tensor + %a3 = "tf.AddV2"(%a2, %c0) : (tensor, tensor) -> tensor + + %b0 = "tf.Sub"(%b, %c0) : (tensor, tensor) -> tensor + %b1 = "tf.Sub"(%b0, %c0) : (tensor, tensor) -> tensor + %b2 = "tf.Sub"(%b1, %c0) : (tensor, tensor) -> tensor + %b3 = "tf.Sub"(%b2, %c0) : (tensor, tensor) -> tensor + + %c = "tf.AddV2"(%a3, %b3) : (tensor, tensor) -> tensor + + func.return %c : tensor +} + +// ----- + +// Test that constants private to a stream are still handled properly when we are copying shared constants. + +// CHECK-LABEL: func private @main_stream_1 +// CHECK: [[r:%.*]] = "tf._TfrtGetResource" +// CHECK-SAME: indices = [1] +// CHECK: "tf.DummySideEffecting"([[r]]) + +// CHECK-LABEL: func private @main_stream_2 +// CHECK: [[r:%.*]] = "tf._TfrtGetResource" +// CHECK-SAME: indices = [0] +// CHECK: "tf.DummySideEffecting"([[r]]) + +// CHECK-LABEL: func @main + +func.func @main(%a: tensor, %b: tensor) -> () { + + %c0 = "tf._TfrtGetResource"() {indices = [0], shared_name = [""], container = [""]} : () -> (tensor) + "tf.DummySideEffecting"(%c0) : (tensor) -> () + + %c1 = "tf._TfrtGetResource"() {indices = [1], shared_name = [""], container = [""]} : () -> (tensor) + "tf.DummySideEffecting"(%c1) : (tensor) -> () + + func.return +} + +// ----- + +// Test that streams with no args but side-effecting ops are still created properly + +// CHECK-LABEL: func private @main_stream_1() +// CHECK: [[r:%.*]] = "tf._TfrtGetResource" +// CHECK-SAME: indices = [0] +// CHECK: "tf.DummySideEffecting"([[r]]) + +// CHECK-LABEL: func @main + +func.func @main(%a: tensor, %b: tensor) -> () { + %c0 = "tf._TfrtGetResource"() {indices = [0], shared_name = [""], container = [""]} : () -> (tensor) + "tf.DummySideEffecting"(%c0) : (tensor) -> () + func.return +} + +// ----- + +// Test control deps of tf.Assert is skipped. + +// CHECK-LABEL: func.func private @skip_assert_stream_3( +// CHECK-NOT: mlrt.await_control +// CHECK: tf.Assert +// CHECK-NOT: mlrt.promise_control +// CHECK: return + +// CHECK-LABEL: func.func private @skip_assert_stream_2( +// CHECK-NOT: mlrt.await_control +// CHECK: tf.Assert +// CHECK-NOT: mlrt.promise_control +// CHECK: return + +func.func @skip_assert(%key: tensor) -> (tensor, tensor) { + %error_message = "tf.Const"() {value = dense<"error"> : tensor} : () -> tensor + %default = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %handle = "tf.HashTableV2"() {container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", key_dtype = !tf_type.string, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + + + %keys = "tf.Const"() {value = dense<["a", "b", "c", "d"]> : tensor<4x!tf_type.string>} : () -> tensor<4x!tf_type.string> + %values = "tf.Const"() {value = dense<[1, 2, 3, 4]> : tensor<4xi64>} : () -> tensor<4xi64> + "tf.LookupTableImportV2"(%handle, %keys, %values) {device = ""} : (tensor, tensor<4x!tf_type.string>, tensor<4xi64>) -> () + %value0 = "tf.LookupTableFindV2"(%handle, %key, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + %cond = "tf.Equal"(%value0, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0", incompatible_shape_error = true} : (tensor, tensor) -> tensor + "tf.Assert"(%cond, %error_message) {device = "/job:localhost/replica:0/task:0/device:CPU:0", summarize = 3 : i64} : (tensor, tensor) -> () + "tf.Assert"(%cond, %error_message) {device = "/job:localhost/replica:0/task:0/device:CPU:0", summarize = 3 : i64} : (tensor, tensor) -> () + %value1 = "tf.LookupTableFindV2"(%handle, %key, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + func.return %value0, %value1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/rewrite_ifrt_load_variable.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/rewrite_ifrt_load_variable.mlir new file mode 100644 index 0000000000000000000000000000000000000000..620586f0db45192f9abf9db5f5ae525b854331cb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/rewrite_ifrt_load_variable.mlir @@ -0,0 +1,43 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-rewrite-ifrt-load-variable %s | FileCheck %s + +// Variable is used by both CPU and TPU +// +// CHECK-LABEL: func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: [[HANDLE:%.*]] = "tf.VarHandleOp"() +// CHECK-NEXT: [[ARRAYKEY:%.*]], [[FURTURE:%.*]] = "tf_mlrt.tf_ifrt_load_variable"([[HANDLE]]) +// CHECK-SAME: <{used_by_host = true}> : (tensor>>) -> (tensor, !mlrt.future) +// CHECK-NEXT: [[TENSOR:%.*]] = "tf_mlrt.tf_await"([[FURTURE]]) : (!mlrt.future) -> tensor<3x1xf32> +// CHECK-NEXT: "tf.MatMul"(%arg0, [[TENSOR]]) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> +// CHECK-NEXT: "tf.IfrtCall"(%arg0, [[ARRAYKEY]]) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [1 : i32]}> {__tpu_compile_metadata_text = "retvals { sharding { } }"} : (tensor<1x3xf32>, tensor) -> tensor<1x1xf32> +// CHECK-NEXT: return +// + func.func @serving_default(%arg0: tensor<1x3xf32>) -> tensor<1x1xf32> { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %array_key, %tensor = "tf.IfrtLoadVariable"(%0) <{used_by_host = true}> : (tensor>>) -> (tensor, tensor<3x1xf32>) + %1 = "tf.MatMul"(%arg0, %tensor) : (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + %2 = "tf.IfrtCall"(%arg0, %array_key) <{operandSegmentSizes = array, program_id = 6515870160938153680 : i64, variable_arg_indices = [1 : i32]}> {__tpu_compile_metadata_text = "retvals { sharding { } }"} : (tensor<1x3xf32>, tensor) -> tensor<1x1xf32> + return %2 : tensor<1x1xf32> + } + + +// ----- +// Variable is used by two CPU ops +// +// CHECK-LABEL: func @serving_default +// CHECK-NEXT: [[HANDLE:%.*]] = "tf.VarHandleOp"() +// CHECK-NEXT: [[ARRAYKEY:%.*]], [[FURTURE:%.*]] = "tf_mlrt.tf_ifrt_load_variable"([[HANDLE]]) +// CHECK-SAME: <{used_by_host = true}> : (tensor>>) -> (tensor, !mlrt.future) +// CHECK: [[TENSOR:%.*]] = "tf_mlrt.tf_await"([[FURTURE]]) : (!mlrt.future) -> tensor<3x1xf32> +// CHECK-NEXT: "tf.AddV2"([[TENSOR]], %cst) : (tensor<3x1xf32>, tensor<3x1xf32>) -> tensor<3x1xf32> +// CHECK-NEXT: "tf.Sub"([[TENSOR]], %cst) : (tensor<3x1xf32>, tensor<3x1xf32>) -> tensor<3x1xf32> +// CHECK-NEXT: return +// + func.func @serving_default() { + %0 = "tf.VarHandleOp"() <{container = "", shared_name = "y"}> : () -> tensor>> + %array_key, %tensor = "tf.IfrtLoadVariable"(%0) <{used_by_host = true}> : (tensor>>) -> (tensor, tensor<3x1xf32>) + %cst_24 = "tf.Const"() <{value = dense<[[0.0], [1.0], [2.0]]> : tensor<3x1xf32>}> : () -> tensor<3x1xf32> + %1 = "tf.AddV2"(%tensor, %cst_24) : (tensor<3x1xf32>, tensor<3x1xf32>) -> tensor<3x1xf32> + %2 = "tf.Sub"(%tensor, %cst_24) : (tensor<3x1xf32>, tensor<3x1xf32>) -> tensor<3x1xf32> + + return + } diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tf_to_mlrt.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tf_to_mlrt.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6517a85aea1e3ed5e262113bb0bdbb6c1eb6145e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tf_to_mlrt.mlir @@ -0,0 +1,602 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-to-mlrt %s | FileCheck %s + +// CHECK-LABEL: @main_stream_0 +// CHECK-SAME: ([[input0:%.*]]: !tf_mlrt.tensor, [[promise_b:%.*]]: !mlrt.promise) +func.func @main_stream_0(%input0: tensor, %promise_b: !mlrt.promise) { + %const = "tf.Const"() {__op_key = 0 : i32, value = dense<1> : tensor} : () -> tensor + // CHECK: [[a:%.*]] = tf_mlrt.executeop([[input0]], + // CHECK-SAME: AddV2 + %a = "tf.AddV2"(%input0, %const) {__op_key = 1: i32}: (tensor, tensor) -> tensor + // CHECK: [[b:%.*]] = tf_mlrt.executeop([[a]]) + // CHECK-SAME: Abs + %b = "tf.Abs"(%a) {__op_key = 2 : i32}: (tensor) -> tensor + // CHECK: tf_mlrt.promise [[promise_b]], [[b]] + "tf_mlrt.tf_promise"(%promise_b, %b) : (!mlrt.promise, tensor) -> () + // CHECK: return + return +} + +// CHECK-LABEL: @main_stream_1 +// CHECK-SAME: ([[input1:%.*]]: !tf_mlrt.tensor, [[promise_c:%.*]]: !mlrt.promise, [[promise_d:%.*]]: !mlrt.promise) +func.func @main_stream_1(%input1: tensor, %promise_c: !mlrt.promise, %promise_d: !mlrt.promise) { + %const = "tf.Const"() {__op_key = 3 : i32, value = dense<1> : tensor} : () -> tensor + // CHECK: [[c:%.*]] = tf_mlrt.executeop([[input1]], + // CHECK-SAME: Sub + %c = "tf.Sub"(%input1, %const) {__op_key = 4: i32} : (tensor, tensor) -> tensor + // CHECK: tf_mlrt.promise [[promise_c]], [[c]] + "tf_mlrt.tf_promise"(%promise_c, %c) : (!mlrt.promise, tensor) -> () + // CHECK: [[d:%.*]] = tf_mlrt.executeop([[c]]) + // CHECK-SAME: Abs + %d = "tf.Abs"(%c) {__op_key = 5: i32}: (tensor) -> tensor + // CHECK: tf_mlrt.promise [[promise_d]], [[d]] + "tf_mlrt.tf_promise"(%promise_d, %d) : (!mlrt.promise, tensor) -> () + // CHECK: return + return +} + +// CHECK-LABEL: @main +// CHECK-SAME: ([[input0:%.*]]: !tf_mlrt.tensor, [[input1:%.*]]: !tf_mlrt.tensor) +func.func @main(%input0: tensor, %input1: tensor) -> tensor { + // CHECK: [[promises:%.*]]:3, [[futures:%.*]]:3 = "tf_mlrt.allocate_futures" + // CHECK-SAME: num_futures = 3 + %promise_b, %promise_c, %promise_d, %future_b, %future_c, %future_d = + "tf_mlrt.allocate_futures"() + {num_futures = 3 : i32, resultSegmentSizes = array} : () -> + (!mlrt.promise, !mlrt.promise, !mlrt.promise, + !mlrt.future, !mlrt.future, !mlrt.future) + + // CHECK: [[handle_0:%.*]] = mlrt.async([[input0]], [[promises]]#0) + // CHECK-SAME: callee = @main_stream_0 + %handle_0 = mlrt.async(%input0, %promise_b) + {callee = @main_stream_0} : + (tensor, !mlrt.promise) -> !mlrt.async_handle + // CHECK: [[handle_1:%.*]] = mlrt.async([[input1]], [[promises]]#1, [[promises]]#2) + // CHECK-SAME: callee = @main_stream_1 + %handle_1 = mlrt.async(%input1, %promise_c, %promise_d) + {callee = @main_stream_1} : + (tensor, !mlrt.promise, !mlrt.promise) -> !mlrt.async_handle + + %const = "tf.Const"() {__op_key = 6: i32, value = dense<2> : tensor} : () -> tensor + // CHECK: [[e:%.*]] = tf_mlrt.executeop([[input1]], + // CHECK-SAME: Mul + %e = "tf.Mul"(%input1, %const) {__op_key = 7: i32} : (tensor, tensor) -> tensor + // CHECK: [[c:%.*]] = tf_mlrt.await [[futures]]#1 + %c = "tf_mlrt.tf_await"(%future_c) : (!mlrt.future) ->tensor + // CHECK: [[f:%.*]] = tf_mlrt.executeop([[e]], [[c]]) + // CHECK-SAME: Div + %f = "tf.Div"(%e, %c) {__op_key = 8: i32}: (tensor, tensor) -> tensor + + // CHECK: [[b:%.*]] = tf_mlrt.await [[futures]]#0 + %b = "tf_mlrt.tf_await"(%future_b) : (!mlrt.future) ->tensor + // CHECK: [[d:%.*]] = tf_mlrt.await [[futures]]#2 + %d = "tf_mlrt.tf_await"(%future_d) : (!mlrt.future) ->tensor + + // CHECK: [[result:%.*]] = tf_mlrt.executeop([[b]], [[d]], [[f]]) + // CHECK-SAME: AddN + %result = "tf.AddN"(%b, %d, %f) {__op_key = 9: i32}: (tensor, tensor, tensor) -> tensor + + // CHECK: mlrt.await_handle [[handle_0]] + // CHECK: mlrt.await_handle [[handle_1]] + mlrt.await_handle %handle_0 + mlrt.await_handle %handle_1 + + // CHECK: return [[result]] + return %result : tensor +} + +// ----- + +// Test lowering tf.If + +func.func @then(%x: tensor, %y: tensor) -> tensor { + return %x: tensor +} + +func.func @else(%x: tensor, %y: tensor) -> tensor { + return %y: tensor +} + +// CHECK-LABEL: func @main +// CHECK-SAME: ([[cond_tensor:%.*]]: !tf_mlrt.tensor, [[x:%.*]]: !tf_mlrt.tensor, [[y:%.*]]: !tf_mlrt.tensor) +// CHECK: [[cond:%.*]] = tf_mlrt.predicate [[cond_tensor]] +// CHECK: [[z:%.*]] = mlrt.cond [[cond]] @then @else([[x]], [[y]]) +// CHECK: return [[z]] +func.func @main(%cond: tensor, %x: tensor, %y: tensor) -> tensor { + %z = "tf.If"(%cond, %x, %y) {then_branch = @then, else_branch = @else, is_stateless = true} : (tensor, tensor, tensor) -> tensor + return %z: tensor +} + +// ----- + +// Test lowering AsyncOpKernel + +// CHECK-LABEL: func @main +func.func @main(%x: tensor) -> (tensor, tensor, tensor) { + // CHECK: [[y_future:%.*]] = tf_mlrt.async_executeop + %y = "tf.TestAsyncIdentity"(%x) {__op_key = 0: i32, T = i32} : (tensor) -> tensor + // CHECK: [[z:%.*]] = tf_mlrt.executeop + %z = "tf.Identity"(%x) {__op_key = 1: i32}: (tensor) -> tensor + // CHECK: [[y:%.*]] = tf_mlrt.await [[y_future]] + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %w = "tf.AddV2"(%y, %z) {__op_key = 2: i32}: (tensor, tensor) -> tensor + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %u = "tf.AddV2"(%y, %z) {__op_key = 3: i32} : (tensor, tensor) -> tensor + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %v = "tf.AddV2"(%y, %z) {__op_key = 4: i32}: (tensor, tensor) -> tensor + return %w, %u, %v : tensor, tensor, tensor +} + +// ----- + +// Test lowering BatchFunction op. + +func.func @batched_function(%x: tensor) -> tensor { + return %x : tensor +} + +// CHECK-LABEL: func @main +func.func @main(%x: tensor<1xi32>) -> (tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) { + // CHECK: [[y_future:%.*]] = tf_mlrt.batch_function + // CHECK-SAME: f = @batched_function + // CHECK-SAME: \22batch_function\22 + %y = "tf.BatchFunction"(%x) { + allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, + batching_queue = "", container = "", device = "/device:CPU:0", + enable_large_batch_splitting = false, f = @batched_function, + max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, + num_batch_threads = 1 : i64, operandSegmentSizes = array, + shared_name = "batch_function" + } : (tensor<1xi32>) -> tensor<1xi32> + + // CHECK: [[z:%.*]] = tf_mlrt.executeop + %z = "tf.Identity"(%x) {__op_key = 0: i32} : (tensor<1xi32>) -> tensor<1xi32> + // CHECK: [[y:%.*]] = tf_mlrt.await [[y_future]] + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %w = "tf.AddV2"(%y, %z) {__op_key = 1: i32}: (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %u = "tf.AddV2"(%y, %z) {__op_key = 2: i32}: (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + // CHECK-NEXT: tf_mlrt.executeop([[y]] + %v = "tf.AddV2"(%y, %z) {__op_key = 3: i32}: (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + return %w, %u, %v : tensor<1xi32>, tensor<1xi32>, tensor<1xi32> +} + +// ----- + +// Test node names are preserved. + +// CHECK-LABEL: func @main +func.func @main(%x: tensor) -> tensor { + // CHECK: tf_mlrt.executeop + // CHECK-SAME: name: \22name_loc/AddV2_0\22 + %y = "tf.AddV2"(%x, %x) {__op_key = 0: i32} : (tensor, tensor) -> tensor loc("name_loc:AddV2") + // CHECK: tf_mlrt.executeop + // CHECK-SAME: name: \22fused_loc/AddV2_1\22 + %z = "tf.AddV2"(%y, %x) {__op_key = 1: i32}: (tensor, tensor) -> tensor loc(fused["fused_loc:", "AddV2"]) + // CHECK: tf_mlrt.executeop + // CHECK-SAME: name: \22AddV2_2\22 + %w = "tf.AddV2"(%z, %x) {__op_key = 2: i32}: (tensor, tensor) -> tensor + return %z : tensor +} + +// ----- + +// Test function name canonicalization + +// CHECK-LABEL: func @__inference_pruned_35 +func.func @__inference_pruned_35() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "flatmapdataset__4_RetVal"}} { + %0 = "tf.Const"() {__op_key = 0: i32, device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {__op_key = 1: i32, device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %2 = "tf.Const"() {__op_key = 2: i32, device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %1, %2) {__op_key = 3: i32, device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tf_mlrt.executeop{{.*}}op: \22FlatMapDataset\22 + // CHECK-SAME: \22__inference_Dataset_flat_map_lambda_19\22 + %4 = "tf.FlatMapDataset"(%3) {__op_key = 4: i32, Targuments = [], device = "/device:CPU:0", f = @__inference_Dataset_flat_map_lambda_190, output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor) -> tensor + func.return %4 : tensor +} +// CHECK-LABEL: __inference_Dataset_flat_map_lambda_190 +func.func private @__inference_Dataset_flat_map_lambda_190(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._original_func_name = "__inference_Dataset_flat_map_lambda_19", tf._tf_data_function = true, tf.signature.is_stateful} { + %0 = "tf.Const"() {__op_key = 5: i32, device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {__op_key = 6: i32,device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %2 = "tf.Const"() {__op_key = 7: i32,device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %2, %1) {__op_key = 8: i32, device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tf_mlrt.executeop{{.*}}op: \22MapDataset\22 + // CHECK-SAME: \22__inference_Dataset_map_lambda_16\22 + %4 = "tf.MapDataset"(%3) {__op_key = 9: i32, device = "/device:CPU:0", f = @__inference_Dataset_map_lambda_160, f._tf_data_function = true, output_shapes = [#tf_type.shape<>], output_types = [i64], preserve_cardinality = true, use_inter_op_parallelism = true, metadata = ""} : (tensor) -> tensor + %5 = "tf.Identity"(%4) {__op_key = 10: i32, device = "/device:CPU:0"} : (tensor) -> tensor + func.return %5 : tensor +} +// CHECK-LABEL: __inference_Dataset_map_lambda_160 +func.func private @__inference_Dataset_map_lambda_160(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._tf_data_function = true} { + %0 = "tf.Const"() {__op_key = 11: i32, device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Mul"(%arg0, %0) {__op_key = 12: i32, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.Identity"(%1) {__op_key = 13: i32, device = "/device:CPU:0"} : (tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// Test while conversion + +// CHECK-LABEL: func @while_cond_lt9 +// CHECK-SAME: ([[arg0:%.*]]: !tf_mlrt.tensor) -> !tf_mlrt.tensor +func.func @while_cond_lt9(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {__op_key = 0: i32, device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {__op_key = 1: i32, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_body_add2 +// CHECK-SAME: ([[arg0:%.*]]: !tf_mlrt.tensor) -> !tf_mlrt.tensor +func.func @while_body_add2(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {__op_key = 2: i32, device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {__op_key = 3: i32, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_test() +// CHECK-SAME: -> !tf_mlrt.tensor +func.func @while_test() -> (tensor) { + // CHECK: [[CONST:%.*]] = tf_mlrt.constop + %0 = "tf.Const"() {__op_key = 4: i32, device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: [[pred_res:%.*]] = call @"while_cond_lt9/tf_mlrt_predicate"([[CONST]]) : (!tf_mlrt.tensor) -> i1 + // CHECK: [[while_res:%.*]]:2 = mlrt.while + // CHECK-SAME: @"while_body_add2/tf_mlrt_body"([[CONST]]) + // CHECK-SAME: (!tf_mlrt.tensor) -> (!tf_mlrt.tensor, i1) + %1 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + // CHECK: return [[while_res]]#0 : !tf_mlrt.tensor + func.return %1 : tensor +} +// CHECK: func @"while_body_add2/tf_mlrt_body"([[arg:%.*]]: !tf_mlrt.tensor) -> (!tf_mlrt.tensor, i1) +// CHECK: [[body_res:%.*]] = call @while_body_add2([[arg]]) : (!tf_mlrt.tensor) -> !tf_mlrt.tensor +// CHECK: [[pred_res:%.*]] = call @"while_cond_lt9/tf_mlrt_predicate"([[body_res]]) : (!tf_mlrt.tensor) -> i1 +// CHECK: return [[body_res]], [[pred_res]] : !tf_mlrt.tensor, i1 + +// CHECK: func @"while_cond_lt9/tf_mlrt_predicate"([[arg:%.*]]: !tf_mlrt.tensor) -> i1 +// CHECK: [[cond_res:%.*]] = call @while_cond_lt9([[arg]]) : (!tf_mlrt.tensor) -> !tf_mlrt.tensor +// CHECK: [[bool_res:%.*]] = tf_mlrt.predicate [[cond_res]] +// CHECK: return [[bool_res]] : i1 + +// CHECK-LABEL: func @multi_while_test +func.func @multi_while_test() -> (tensor, tensor) { + %0 = "tf.Const"() {__op_key = 5: i32, device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {__op_key = 6: i32, device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + // CHECK: [[pred_0:%.*]] = call @"while_cond_lt9/tf_mlrt_predicate" + // CHECK: mlrt.while [[pred_0]] @"while_body_add2/tf_mlrt_body" + // CHECK: [[pred_1:%.*]] = call @"while_cond_lt9/tf_mlrt_predicate" + // CHECK: mlrt.while [[pred_1]] @"while_body_add2/tf_mlrt_body" + %2 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + %3 = "tf.While"(%1) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + func.return %2, %3 : tensor, tensor +} + +// ----- + +// Test async output to function is converted + +// CHECK-LABEL: @serving_default_stream_1 +// CHECK-SAME: !mlrt.future +func.func private @serving_default_stream_1(%arg0: tensor) { + // CHECK: [[tensor:%.*]] = tf_mlrt.await + // CHECK: tf_mlrt.executeop([[tensor]]) + %0 = "tf.StringFormat"(%arg0) {__op_key = 0: i32, device = "/job:localhost/replica:0/task:0/device:CPU:0", placeholder = "{}", strtemplate = "%s", summarize = 3 : i64, template = "Outside compiled {}"} : (tensor) -> tensor + "tf.PrintV2"(%0) {__op_key = 1: i32, device = "/job:localhost/replica:0/task:0/device:CPU:0", end = "\0A", output_stream = "stderr"} : (tensor) -> () + return +} + +func.func @callee(%arg: tensor) -> (tensor) { + func.return %arg: tensor +} + +// CHECK-LABEL: @executeop_input +func.func @executeop_input(%arg0: tensor) -> (tensor) { + // CHECK: [[async_out:%.*]] = tf_mlrt.batch_function + %2 = "tf.BatchFunction"(%arg0) {device = "/device:CPU:0", allowed_batch_sizes = [64], batch_timeout_micros = 1 : i64, batching_queue = "", container = "", f = @callee, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor) -> tensor + // CHECK-NEXT: mlrt.async([[async_out]]) {{.*}} : (!mlrt.future) + %3 = mlrt.async(%2) {callee = @serving_default_stream_1} : (tensor) -> !mlrt.async_handle + // CHECK: mlrt.await_handle + mlrt.await_handle %3 + // CHECK: return + // CHECK-SAME: !tf_mlrt.tensor + func.return %2 : tensor +} + +// ----- + +// Support pre-assigned op_key + +// CHECK-LABEL: @main +// CHECK-SAME: ([[input0:%.*]]: !tf_mlrt.tensor, [[promise_b:%.*]]: !mlrt.promise) +func.func @main(%input0: tensor, %promise_b: !mlrt.promise) { + %const = "tf.Const"() {__op_key = 0 : i32, value = dense<1> : tensor} : () -> tensor + // CHECK: [[a:%.*]] = tf_mlrt.executeop([[input0]], + // CHECK-SAME: AddV2 + // CHECK-SAME: op_key = 1 + // CHECK-NOT: __op_key + %a = "tf.AddV2"(%input0, %const) {__op_key = 1: i32}: (tensor, tensor) -> tensor + // CHECK: [[b:%.*]] = tf_mlrt.executeop([[a]]) + // CHECK-SAME: Abs + // CHECK-SAME: op_key = 2 + // CHECK-NOT: __op_key + %b = "tf.Abs"(%a) {__op_key = 2: i32 }: (tensor) -> tensor + // CHECK: tf_mlrt.promise [[promise_b]], [[b]] + "tf_mlrt.tf_promise"(%promise_b, %b) : (!mlrt.promise, tensor) -> () + // CHECK: return + return +} + +// ----- + +// Test future as input to promise + +// CHECK-LABEL: func @main_stream_0 +func.func @main_stream_0(%x: tensor, %p: !mlrt.promise) -> () { + // CHECK: [[y_future:%.*]] = tf_mlrt.async_executeop + %y = "tf.TestAsyncIdentity"(%x) {__op_key = 0: i32, T = i32} : (tensor) -> tensor + // CHECK: tf_mlrt.promise_future + // CHECK-SAME: [[y_future]] + "tf_mlrt.tf_promise"(%p, %y): (!mlrt.promise, tensor) -> () + return +} + +// CHECK-LABEL: @main +// CHECK-SAME: ([[input0:%.*]]: !tf_mlrt.tensor) +func.func @main(%input0: tensor) -> tensor { + // CHECK: [[promises:%.*]], [[futures:%.*]] = "tf_mlrt.allocate_futures" + // CHECK-SAME: num_futures = 1 + %promise_b, %future_b = "tf_mlrt.allocate_futures"() + {num_futures = 1 : i32, resultSegmentSizes = array} : () -> + (!mlrt.promise, !mlrt.future) + + // CHECK: [[handle_0:%.*]] = mlrt.async([[input0]], [[promises]]) + // CHECK-SAME: callee = @main_stream_0 + %handle_0 = mlrt.async(%input0, %promise_b) + {callee = @main_stream_0} : + (tensor, !mlrt.promise) -> !mlrt.async_handle + + // CHECK: [[const:%.*]] = tf_mlrt.const + %const = "tf.Const"() {__op_key = 1: i32, value = dense<2> : tensor} : () -> tensor + + // CHECK: [[b:%.*]] = tf_mlrt.await [[futures]] + %b = "tf_mlrt.tf_await"(%future_b) : (!mlrt.future) ->tensor + + // CHECK: [[result:%.*]] = tf_mlrt.executeop([[b]], [[const]]) + // CHECK-SAME: AddV2 + %result = "tf.AddV2"(%b, %const) {__op_key = 2: i32}: (tensor, tensor) -> tensor + + // CHECK: mlrt.await_handle [[handle_0]] + mlrt.await_handle %handle_0 + + // CHECK: return [[result]] + return %result : tensor +} + +// ----- + +// Test lowering of tf call ops + +// CHECK-LABEL: @callee +func.func @callee(%arg0: tensor) -> (tensor) { + func.return %arg0: tensor +} + +// CHECK-LABEL: func @call_test +func.func @call_test(%arg0: tensor) -> (tensor, tensor, tensor) { + %0 = "tf.Add"(%arg0, %arg0) {__op_key = 0, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + // CHECK: [[results_0:%.*]] = call @callee( + // CHECK-SAME: (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %1 = "tf.StatefulPartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> (tensor) + // CHECK-NEXT: [[results_1:%.*]] = call @callee( + // CHECK-SAME: (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %2 = "tf.PartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> (tensor) + // CHECK-NEXT: [[results_2:%.*]] = call @callee( + // CHECK-SAME: (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %3 = "tf.LegacyCall"(%0) {f = @callee} : (tensor) -> (tensor) + // CHECK: [[results_0]], [[results_1]], [[results_2]] + func.return %1, %2, %3 : tensor, tensor, tensor +} + +// CHECK-LABEL: @branch0 +func.func @branch0(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.Add" (%arg0, %arg1) {__op_key = 1, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: @branch1 +func.func @branch1(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.Add" (%arg0, %arg1) {__op_key = 2, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Add" (%arg0, %0) {__op_key = 3, device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @case_test +// CHECK-SAME: ([[tf_idx:%.*]]: !tf_mlrt.tensor, [[branch_arg0:%.*]]: !tf_mlrt.tensor, [[branch_arg1:%.*]]: !tf_mlrt.tensor) +func.func @case_test(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[idx:%.*]] = tf_mlrt.tensor_to_int32 [[tf_idx]] + // CHECK-NEXT: [[out:%.*]] = mlrt.case [[idx]] [@branch0, @branch1]([[branch_arg0]], [[branch_arg1]]) + %0 = "tf.Case"(%arg0, %arg1, %arg2) {_lower_using_switch_merge = true, branches = [@branch0, @branch1], is_stateless = true} : (tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// Test await is added for unused futures + +// CHECK-LABEL: func @unused_future_arg +// CHECK-SAME: ({{%.*}}: !tf_mlrt.tensor, [[unused:%.*]]: !mlrt.future) +func.func @unused_future_arg(%x: tensor, %unused: !mlrt.future) -> tensor { + // CHECK: mlrt.await_all_control [[unused]] + return %x : tensor +} + +// CHECK-LABEL: func @unused_future +func.func @unused_future(%x: tensor) -> tensor { + // CHECK: [[unused:%.*]] = tf_mlrt.async_executeop + %unused = "tf.TestAsyncIdentity"(%x) {__op_key = 0: i32, T = i32} : (tensor) -> tensor + // CHECK: mlrt.await_all_control [[unused]] + return %x : tensor +} + +// ----- + +// Test for XlaLaunch + +func.func private @xla_func_0(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) {__op_key = 0: i32} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @xla_func +func.func @xla_func(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {__op_key = 1: i32, device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {__op_key = 2: i32, device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: tf_mlrt.executeop + // CHECK: tf_mlrt.async_executeop{{.*}}op: \22XlaLaunch\22\0A + // CHECK: tf_mlrt.await + // CHECK: return + // CHECK-SAME: !tf_mlrt.tensor + %2 = "tf.XlaLaunch"(%arg0, %1) {__op_key = 3: i32, _noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_0, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +// ----- + +// Test lowering of IfrtLoadVariableOp + +// CHECK-LABEL: func @ifrt_load_variable_test +func.func @ifrt_load_variable_test() -> () { + // CHECK: [[HANDLE:%.*]] = tf_mlrt.executeop() + // CHECK-SAME: VarHandleOp + %0 = "tf.VarHandleOp"() {__op_key = 1: i32, device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + // CHECK-NEXT: "tf_mlrt.ifrt_load_variable"([[HANDLE]]) + // CHECK-SAME: used_by_host = true + %1, %2 = "tf_mlrt.tf_ifrt_load_variable"(%0) {used_by_host = true, __op_key = 2: i32, device = "/device:CPU:0"} : (tensor>>) -> (tensor, !mlrt.future) + // CHECK-NEXT: mlrt.await_all_control + // CHECK-NEXT: return + func.return +} + +// ----- + +// Test lowering of IfrtRestoreVariableOp + +// CHECK-LABEL: func @ifrt_restore_variable_test +func.func @ifrt_restore_variable_test() -> () { + // CHECK-NEXT: [[PREFIX:%.*]] = tf_mlrt.constop + %cst = "tf.Const"() {__op_key = 0: i32, value = dense<"restore_ariables"> : tensor} : () -> tensor + // CHECK-NEXT: [[SLICE:%.*]] = tf_mlrt.constop + %cst_0 = "tf.Const"() {__op_key = 1: i32, value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + // CHECK-NEXT: [[NAME:%.*]] = tf_mlrt.constop + %cst_1 = "tf.Const"() {__op_key = 2: i32, value = dense<["y"]> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + // CHECK-NEXT: [[HANDLE:%.*]] = tf_mlrt.executeop + %handle = "tf.VarHandleOp"() {__op_key = 3: i32, container = "x", shared_name = "y"} : () -> tensor>> + // CHECK-NEXT: "tf_mlrt.ifrt_restore_variable"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLE]]) <{restored_dtypes = [f32], returned_tensor_names = [], truncate_in_cast = array}> + "tf.IfrtRestoreVariableOp"(%cst, %cst_1, %cst_0, %handle) {restored_dtypes = [f32], returned_tensor_names = [], truncate_in_cast = array} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>, tensor>>) -> () + // CHECK-NEXT: return + func.return +} + +// ----- + +// Test lowering of tf.IfrtResourceDeserializeOp to tf_mlrt.ifrt_resource_deserialize + +// CHECK-LABEL: func @ifrt_resource_deserialize_test +func.func @ifrt_resource_deserialize_test(%arg0: tensor>>) { + %input_dir = "tf.Const"() { value = dense<"some/path"> : tensor } : () -> tensor + // CHECK: "tf_mlrt.ifrt_resource_deserialize"(%arg0, %{{.*}}) <{tensor_name = "my_tensor"}> + "tf.IfrtResourceDeserialize"(%arg0, %input_dir) {require_matching_crc = false, tensor_name = "my_tensor"} : (tensor>>, tensor) -> () + func.return +} + + +// ----- + +// Test lowering of tf.IfrtRestoreVariableOp with outputs to tf_mlrt.ifrt_restore_variable + +// CHECK-LABEL: func @ifrt_restore_variable_with_output_test +func.func @ifrt_restore_variable_with_output_test() -> (tensor<3x1xf32>) { + // CHECK-NEXT: [[PREFIX:%.*]] = tf_mlrt.constop + %cst = "tf.Const"() {__op_key = 0: i32, value = dense<"restore_ariables"> : tensor} : () -> tensor + // CHECK-NEXT: [[SLICE:%.*]] = tf_mlrt.constop + %cst_0 = "tf.Const"() {__op_key = 1: i32, value = dense<""> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + // CHECK-NEXT: [[NAME:%.*]] = tf_mlrt.constop + %cst_1 = "tf.Const"() {__op_key = 2: i32, value = dense<["y"]> : tensor<1x!tf_type.string>} : () -> tensor<1x!tf_type.string> + // CHECK-NEXT: [[HANDLE:%.*]] = tf_mlrt.executeop + %handle = "tf.VarHandleOp"() {__op_key = 3: i32, container = "x", shared_name = "y"} : () -> tensor>> + // CHECK-NEXT: [[RESFUTURE:%.*]] = "tf_mlrt.ifrt_restore_variable"([[PREFIX]], [[NAME]], [[SLICE]], [[HANDLE]]) <{restored_dtypes = [f32], returned_tensor_names = ["y"], truncate_in_cast = array}> : (!tf_mlrt.tensor, !tf_mlrt.tensor, !tf_mlrt.tensor, !tf_mlrt.tensor) -> !mlrt.future + %result = "tf.IfrtRestoreVariableOp"(%cst, %cst_1, %cst_0, %handle) {restored_dtypes = [f32], returned_tensor_names = ["y"], truncate_in_cast = array} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>, tensor>>) -> tensor<3x1xf32> + // CHECK-NEXT: [[RESULT:%.*]] = tf_mlrt.await [[RESFUTURE]] + // CHECK-NEXT: return [[RESULT]] : !tf_mlrt.tensor + func.return %result : tensor<3x1xf32> +} + +// ----- + +// Test lowering of tf.IfrtCall (should use fallback) + +// CHECK-LABEL: func @ifrt_call_fallback_test +func.func @ifrt_call_fallback_test(%arg0: tensor) -> tensor { + // CHECK: tf_mlrt.executeop + // CHECK-SAME: IfrtCall + %0 = "tf.IfrtCall"(%arg0) {program_id = 123 : i64, variable_arg_indices = [], __op_key = 0 : i32, operandSegmentSizes = array} : (tensor) -> tensor + return %0 : tensor +} + +// ----- + +// Test lowering of tf.AsyncIfrtCall (should split into async call and await) + +// CHECK-LABEL: func @async_ifrt_call_test +func.func @async_ifrt_call_test(%arg0: tensor) -> tensor { + // CHECK: [[FUTURE:%.*]] = tf_mlrt.async_ifrt_call + // CHECK: tf_mlrt.await [[FUTURE]] + %0 = "tf.AsyncIfrtCall"(%arg0) {program_id = 123 : i64, variable_arg_indices = [], __op_key = 0 : i32, operandSegmentSizes = array} : (tensor) -> tensor + return %0 : tensor +} + +// ----- + +// Test lowering of tf.PartitionedCall and tf.StatefulPartitionedCall to func.call + +// CHECK-LABEL: func private @callee +// CHECK-SAME: (%arg0: !tf_mlrt.tensor) -> !tf_mlrt.tensor +func.func private @callee(%arg0: tensor) -> tensor { + return %arg0 : tensor +} + +// CHECK-LABEL: func @test_call_ops +// CHECK-SAME: ([[ARG:%.*]]: !tf_mlrt.tensor) -> (!tf_mlrt.tensor, !tf_mlrt.tensor) +func.func @test_call_ops(%arg0: tensor) -> (tensor, tensor) { + // CHECK: [[RES1:%.*]] = call @callee([[ARG]]) : (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %0 = "tf.PartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> tensor + + // CHECK: [[RES2:%.*]] = call @callee([[ARG]]) : (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %1 = "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> tensor + + // CHECK: return [[RES1]], [[RES2]] : !tf_mlrt.tensor, !tf_mlrt.tensor + return %0, %1 : tensor, tensor +} + +// ----- + +// Test lowering of tf.PartitionedCall with a future input (should insert await) + +// CHECK-LABEL: func private @callee_future +// CHECK-SAME: (%arg0: !tf_mlrt.tensor) -> !tf_mlrt.tensor +func.func private @callee_future(%arg0: tensor<3x1xf32>) -> tensor<3x1xf32> { + return %arg0 : tensor<3x1xf32> +} + +// CHECK-LABEL: func @test_call_with_future +// CHECK-SAME: ([[ARG0:%.*]]: !tf_mlrt.tensor, [[ARG1:%.*]]: !tf_mlrt.tensor, [[ARG2:%.*]]: !tf_mlrt.tensor, [[ARG3:%.*]]: !tf_mlrt.tensor) -> !tf_mlrt.tensor +func.func @test_call_with_future(%arg0: tensor, %arg1: tensor<1x!tf_type.string>, %arg2: tensor<1x!tf_type.string>, %arg3: tensor>>) -> tensor<3x1xf32> { + // CHECK: [[FUTURE:%.*]] = "tf_mlrt.ifrt_restore_variable"([[ARG0]], [[ARG1]], [[ARG2]], [[ARG3]]) + %result = "tf.IfrtRestoreVariableOp"(%arg0, %arg1, %arg2, %arg3) {restored_dtypes = [f32], returned_tensor_names = ["y"], truncate_in_cast = array} : (tensor, tensor<1x!tf_type.string>, tensor<1x!tf_type.string>, tensor>>) -> tensor<3x1xf32> + // CHECK-NEXT: [[AWAITED:%.*]] = tf_mlrt.await [[FUTURE]] + // CHECK-NEXT: [[RESULT:%.*]] = call @callee_future([[AWAITED]]) : (!tf_mlrt.tensor) -> !tf_mlrt.tensor + %0 = "tf.PartitionedCall"(%result) {config = "", config_proto = "", executor_type = "", f = @callee_future} : (tensor<3x1xf32>) -> tensor<3x1xf32> + // CHECK: return [[RESULT]] : !tf_mlrt.tensor + return %0 : tensor<3x1xf32> +} + + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tpu_conversions.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tpu_conversions.mlir new file mode 100644 index 0000000000000000000000000000000000000000..5902c84cf0f56a1a87abf83f5e20a5bfe13ef690 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/tpu_conversions.mlir @@ -0,0 +1,229 @@ +// RUN: tf-tfrt-opt --split-input-file -pass-pipeline='builtin.module(pre-parallel-tf-to-mlrt{use-tpu-host-allocator-for-inputs=true},tf-mlrt-parallelization{tfrt-cost-threshold=4},tf-to-mlrt)' %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +func.func @callee(%arg0: tensor, %arg1: tensor) -> (tensor) { + func.return %arg0: tensor +} + +// CHECK-LABEL: func @batch_function +func.func @batch_function(%arg0: tensor) -> (tensor) { + // CHECK: [[batch_result_future:%.*]] = tf_mlrt.batch_function + // CHECK: [[batch_result:%.*]] = tf_mlrt.await [[batch_result_future]] + // CHECK-NEXT: [[rendezvous_key_base:%.*]] = tf_mlrt_tpu.compile_and_execute([[batch_result]]) + // CHECK-NEXT: return [[rendezvous_key_base]] + %0 = "tf.BatchFunction"(%arg0, %arg0) {device = "/device:CPU:0", allowed_batch_sizes = [64], batch_timeout_micros = 1 : i64, batching_queue = "", container = "", f = @callee, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor, tensor) -> tensor + %1 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + func.return %1 : tensor +} + +// ----- + +func.func @executeop_input(%arg0: tensor) -> (tensor, tensor) { + // CHECK-NOT: tf_mlrt.executeop( + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[cast:%.*]] = tf_mlrt.executeop.device([[device]]){{.*}}op: \22Cast\22 + // CHECK: [[rendezvous_key_base:%.*]], [[result_future:%.*]] = tf_mlrt_tpu.compile_and_execute([[cast]]) + // CHECK: tf_mlrt.await [[result_future]] + %0 = "tf.Cast"(%arg0) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor) -> tensor + %1, %2 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> (tensor, tensor) + func.return %1, %2 : tensor, tensor +} + +// ----- + +func.func @executeop_side_effecting_input(%arg0: tensor>>, %indices: tensor) -> (tensor) { + // CHECK-NOT: tf_mlrt.executeop( + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[var:%.*]] = tf_mlrt.executeop.device([[device]]){{.*}}op: \22ResourceGather\22 + // CHECK: [[rendezvous_key_base:%.*]] = tf_mlrt_tpu.compile_and_execute([[var]]) + %0 = "tf.ResourceGather"(%arg0, %indices) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor>>, tensor) -> tensor + %1 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + func.return %1 : tensor +} + +// ----- + +func.func @executeop_input_same_execute_op(%arg0: tensor, %arg1: tensor<2xf32>) -> (tensor) { + // CHECK-NOT: tf_mlrt.executeop( + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[split:%.*]]:2 = tf_mlrt.executeop.device([[device]]) + // CHECK: tf_mlrt_tpu.compile_and_execute([[split]]#0, [[split]]#1) + %0, %1 = "tf.Split"(%arg0, %arg1) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor, tensor<2xf32>) -> (tensor, tensor) + %2 = "tf.TPUCompileMlirAndExecute"(%0, %1) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +// Test that inputs are lowered correctly when they form a DAG. + +// CHECK-LABEL: executeop_dag +func.func @executeop_dag(%arg0: tensor) -> (tensor) { + // CHECK-NEXT: tf_mlrt_tpu.get_tpu_host_device + // CHECK-NEXT: tf_mlrt.executeop.device{{.*}}op: \22Cast\22 + // CHECK-NEXT: tf_mlrt_tpu.get_tpu_host_device + // CHECK-NEXT: tf_mlrt.executeop.device{{.*}}op: \22Relu\22 + // CHECK-NEXT: tf_mlrt_tpu.compile_and_execute + %0 = "tf.Cast"(%arg0) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor) -> tensor + %1 = "tf.Relu"(%0) {__op_key = 1: i32, device = "/device:CPU:0"} : (tensor) -> (tensor) + %2 = "tf.TPUCompileMlirAndExecute"(%1, %0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// ----- + +func.func @test_fuse_dynamic_dimension_ops(%arg0: tensor<*xi32>, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<*xi32>, %arg3: tensor<*xi32>, %arg4: tensor<*xi32>, %arg5: tensor, %arg6: tensor, %arg7: tensor) -> tensor<*xi32> { + %0 = "tf.ReadVariableOp"(%arg1) {__op_key = 0: i32, device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<*xi32> + %1 = "tf.Shape"(%arg0) {__op_key = 1: i32, device = "/CPU:0"} : (tensor<*xi32>) -> tensor + %2 = "tf.Shape"(%0) {__op_key = 2: i32, device = "/CPU:0"} : (tensor<*xi32>) -> tensor + // CHECK: [[rendezvous_key_base:%.*]], [[result_future:%.*]] = tf_mlrt_tpu.compile_and_execute + // CHECK-SAME: constant_operand_indices = array + // CHECK-SAME: num_operands = 4 + // CHECK-SAME: operands_with_static_shape = array + %rendezvous_key_base, %results = "tf.TPUCompileMlirAndExecute"(%arg0, %2, %0, %1, %arg5, %arg6, %arg7) {operands_with_static_shape = [0 : i32, 1 : i32, 3 : i32], metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor<*xi32>, tensor, tensor<*xi32>, tensor, tensor, tensor, tensor) -> (tensor<3x!tf_type.string>, tensor<*xi32>) + func.return %results : tensor<*xi32> +} + +// ----- + +// Test async output of tf.TPUCompileMlirAndExecute to function is converted + +// CHECK-LABEL: @executeop_input_stream_1 +// CHECK-SAME: ([[future:%.*]]: !mlrt.future +// CHECK: [[tensor:%.*]] = tf_mlrt.await [[future]] +// CHECK: tf_mlrt.executeop([[tensor]]) +// CHECK-SAME: StringFormat + +// CHECK-LABEL: @executeop_input +func.func @executeop_input(%arg0: tensor) -> (tensor) { + // CHECK: tf_mlrt.executeop + %0 = "tf.Cast"(%arg0) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor) -> tensor + // CHECK: [[rendezvous_key_base:%.*]], [[result:%.*]] = tf_mlrt_tpu.compile_and_execute + %1, %2 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> (tensor, tensor) + %3 = "tf.StringFormat"(%2) {__op_key = 1: i32, device = "/job:localhost/replica:0/task:0/device:CPU:0", placeholder = "{}", strtemplate = "%s", summarize = 3 : i64, template = "Outside compiled {}"} : (tensor) -> tensor + "tf.PrintV2"(%3) {__op_key = 2: i32, device = "/job:localhost/replica:0/task:0/device:CPU:0", end = "\0A", output_stream = "stderr"} : (tensor) -> () + // CHECK: [[handle:%.*]] = mlrt.async([[result]]) + // CHECK-SAME: (!mlrt.future) + // CHECK: mlrt.await_handle [[handle]] + // CHECK: return [[rendezvous_key_base]] + // CHECK-SAME: !tf_mlrt.tensor + func.return %1 : tensor +} + +// ----- + +// Test constant arguments to tf.TPUCompileMlirAndExecute are preserved during parallelization. + +// CHECK-LABEL: @preserve_constant_args( +func.func @preserve_constant_args(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<*x!tf_type.resource>, %arg3: tensor<*x!tf_type.resource>) -> (tensor) { + // CHECK-NOT: ReadVariableOp + // CHECK: mlrt.async( + %v0 = "tf.ReadVariableOp"(%arg1) {__op_key = 0: i32, device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor + %v1 = "tf.ReadVariableOp"(%arg2) {__op_key = 1: i32, device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor + // CHECK: [[cast:%.*]] = tf_mlrt.executeop( + // CHECK-SAME: ReadVariableOp + %v2 = "tf.ReadVariableOp"(%arg3) {__op_key = 2: i32, device = "/CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor + // CHECK: [[cast:%.*]] = tf_mlrt.executeop.device + // CHECK-SAME: Cast + %0 = "tf.Cast"(%arg0) {__op_key = 3: i32, device = "/device:CPU:0"} : (tensor) -> tensor + // CHECK: tf_mlrt_tpu.compile_and_execute({{%.*}}, [[cast]] + // CHECK-SAME: constant_operand_indices = array + %1, %2 = "tf.TPUCompileMlirAndExecute"(%0, %v1, %0, %v2, %v0, %arg0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor, tensor, tensor, tensor, tensor, tensor) -> (tensor, tensor) + func.return %2 : tensor +} + +// ----- + +func.func @executeop_input_async() -> (tensor, tensor) { + // CHECK-NOT: tf_mlrt.executeop( + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[recv_future:%.*]] = tf_mlrt.async_executeop.device([[device]]){{.*}}op: \22Recv\22 + // CHECK: [[recv:%.*]] = tf_mlrt.await [[recv_future]] + // CHECK: [[rendezvous_key_base:%.*]], [[result_future:%.*]] = tf_mlrt_tpu.compile_and_execute([[recv]]) + // CHECK: tf_mlrt.await [[result_future]] + %0 = "tf.Recv"() {__op_key = 0: i32, device = "/device:CPU:0", tensor_name = "tensor", send_device = "/device:CPU:0", send_device_incarnation = 0, recv_device = "/device:CPU:0"} : () -> tensor + %1, %2 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> (tensor, tensor) + func.return %1, %2 : tensor, tensor +} + +// ----- + +// Test the output from TPU op is properly awaited before its use by map_fn. +// CHECK-LABEL: @main +// CHECK-SAME: ([[input0:%.*]]: !tf_mlrt.tensor, [[input1:%.*]]: !tf_mlrt.tensor) +func.func @main(%input0: tensor, %input1: tensor, %input2: tensor>> ) -> tensor { + %0 = "tf.Cast"(%input0) {__op_key = 0: i32, device = "/device:CPU:0"} : (tensor) -> tensor + // CHECK: tf_mlrt_tpu.compile_and_execute + %1, %2 = "tf.TPUCompileMlirAndExecute"(%0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> (tensor, tensor) + %max_iter = "tf.Const"() {__op_key = 1, value = dense<2> : tensor} : () -> tensor + // CHECK: tf_mlrt.map_fn + %result = "tf_mlrt.tf_map_fn"(%max_iter, %input2, %2) { operandSegmentSizes = array, body_fn = @NopMapFnBody, num_tensor_list_or_flow_in = 1 : i32} : (tensor, tensor>>, tensor) -> tensor + return %result : tensor +} + +// CHECK-LABEL: @NopMapFnBody +func.func private @NopMapFnBody(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor>>) -> () { + %const = "tf.Const"() {__op_key = 2 : i32, value = dense<1> : tensor} : () -> tensor + %a = "tf.AddV2"(%arg2, %const) {__op_key = 3: i32}: (tensor, tensor) -> tensor + return +} + +// ----- + +// If not all in_tensors of the batched op are used by TPUCompileMlirAndExecute, +// should not lower batch_function to tf_mlrt.batch_function.device. + +func.func @callee(%arg0: tensor, %arg1: tensor) -> (tensor) { + %1 = "tf.TPUCompileMlirAndExecute"(%arg0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + %const = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %a = "tf.AddV2"(%arg1, %const) {__op_key = 3: i32}: (tensor, tensor) -> tensor + func.return %1: tensor +} + +// CHECK-LABEL: func @batch_function +func.func @batch_function(%arg0: tensor, %arg1: tensor) -> (tensor) { + // CHECK: tf_mlrt.batch_function(%arg0, %arg1) + // CHECK-NOT: batch_function.device + %0 = "tf.BatchFunction"(%arg0, %arg1) {device = "/device:CPU:0", allowed_batch_sizes = [64], batch_timeout_micros = 1 : i64, batching_queue = "", container = "", f = @callee, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- + +// captured tensors not used by TPUCompileMlirAndExecute is ok. + +func.func @callee(%arg0: tensor, %arg1: tensor) -> (tensor) { + %1 = "tf.TPUCompileMlirAndExecute"(%arg0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + %const = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %a = "tf.AddV2"(%arg1, %const) {__op_key = 3: i32}: (tensor, tensor) -> tensor + func.return %1: tensor +} + +// CHECK-LABEL: func @batch_function +func.func @batch_function(%arg0: tensor, %arg1: tensor) -> (tensor) { + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[batch_result_future:%.*]] = tf_mlrt.batch_function.device([[device]]) (%arg0, %arg1) + // CHECK: [[batch_result:%.*]] = tf_mlrt.await [[batch_result_future]] + // CHECK: return [[batch_result]] + %0 = "tf.BatchFunction"(%arg0, %arg1) {device = "/device:CPU:0", allowed_batch_sizes = [64], batch_timeout_micros = 1 : i64, batching_queue = "", container = "", f = @callee, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +// ----- +func.func @batched_func(%arg0: tensor, %arg1: tensor) -> (tensor) { + %0 = "tf.TPUCompileMlirAndExecute"(%arg0) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + %2 = "tf.TPUCompileMlirAndExecute"(%arg1) {metadata = "metadata", mlir_module = "mlir_module", operandSegmentSizes = array, producer_name = "producer_name"} : (tensor) -> tensor + func.return %2: tensor +} + +// CHECK-LABEL: func @batch_function +func.func @batch_function(%arg0: tensor, %arg1: tensor) -> (tensor) { + // CHECK: [[device:%.*]] = tf_mlrt_tpu.get_tpu_host_device + // CHECK: [[batch_result_future:%.*]] = tf_mlrt.batch_function.device([[device]]) (%arg0, %arg1) + // CHECK: [[batch_result:%.*]] = tf_mlrt.await [[batch_result_future]] + // CHECK: return [[batch_result]] + %0 = "tf.BatchFunction"(%arg0, %arg1) {device = "/device:CPU:0", allowed_batch_sizes = [64], batch_timeout_micros = 1 : i64, batching_queue = "", container = "", f = @batched_func, max_batch_size = 256 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = ""} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/while_to_map_fn.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/while_to_map_fn.mlir new file mode 100644 index 0000000000000000000000000000000000000000..260cf41077a0b24a7c9ff9d3a339b38e9d008498 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/mlrt/while_to_map_fn.mlir @@ -0,0 +1,815 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-mlrt-while-to-map-fn %s | FileCheck %s + +// Test a while to map_fn conversion in which the max iteration is hard coded inside the predicate body. + +// CHECK-LABEL: map/while_cond +func.func private @"map/while_cond"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> tensor { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor} : () -> tensor + %0 = "tf.Less"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg1, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + return %2 : tensor +} + +// CHECK-LABEL: map/while_body +func.func private @"map/while_body"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> (tensor, tensor, tensor>>, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00, 7.000000e+00, 8.000000e+00, 9.000000e+00]> : tensor<9xf32>} : () -> tensor<9xf32> + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0, 1, 2]> : tensor<3xi32>} : () -> tensor<3xi32> + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_3 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00, 7.000000e+00, 8.000000e+00]> : tensor<9xf32>} : () -> tensor<9xf32> + %cst_4 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%arg0, %cst_4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Mul"(%arg3, %cst_3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<9xf32>) -> tensor<9xf32> + %2 = "tf.Reshape"(%1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<9xf32>, tensor<2xi32>) -> tensor<3x3xf32> + %3 = "tf.AddV2"(%arg1, %cst_4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %4 = "tf.GatherV2"(%cst_1, %arg1, %cst_0) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<3xi32>, tensor, tensor) -> tensor + %5 = "tf.Cast"(%4) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %6 = "tf.Mul"(%5, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<9xf32>) -> tensor<9xf32> + %7 = "tf.Reshape"(%6, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<9xf32>, tensor<2xi32>) -> tensor<3x3xf32> + %8 = "tf.MatMul"(%2, %7) {device = "/job:localhost/replica:0/task:0/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %9 = "tf.MatrixDeterminant"(%8) {T = f32, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<3x3xf32>) -> tensor + %10 = "tf.TensorListSetItem"(%arg2, %arg1, %9) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor) -> tensor>> + return %0, %3, %10, %arg3 : tensor, tensor, tensor>>, tensor +} + +// CHECK-LABEL: map/while_body/MapFnBody +// CHECK-SAME: (%arg0: !mlrt.future, %arg1: !mlrt.promise, %arg2: tensor, %arg3: tensor, %arg4: tensor) +// CHECK: [[det:%.*]] = "tf.MatrixDeterminant" +// CHECK-NEXT: [[ta_0:%.*]] = "tf_mlrt.tf_await"(%arg0) : (!mlrt.future) -> tensor>> +// CHECK-NEXT: [[ta_1:%.*]] = "tf.TensorListSetItem"([[ta_0]], %arg3, [[det]]) <{ +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg1, [[ta_1]]) : (!mlrt.promise, tensor>>) -> () +// CHECK-NEXT: return + +//CHECK-LABEL: @serving_default +func.func @serving_default(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) -> tensor<3xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "serving_default_input:0", outputs = "PartitionedCall:0"}} { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor} : () -> tensor + // CHECK: [[tensor_list:%.*]] = "tf.TensorListReserve"([[shape:%.*]], [[reserve_size:%.*]]) { + %0 = "tf.TensorListReserve"(%cst_1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor>> + // CHECK: [[map_fn_result:%.*]] = tf_mlrt.tf_map_fn([[reserve_size]], [[tensor_list]], %arg0) + // CHECK-SAME: {body_fn = @"map/while_body/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} + // CHECK-NOT: tf.While + %1:4 = "tf.While"(%cst, %cst, %0, %arg0) {_lower_using_switch_merge = true, _num_original_outputs = 6 : i64, _read_only_resource_inputs = [], _xla_propagate_compile_time_consts = true, body = @"map/while_body", cond = @"map/while_cond", device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = true, parallel_iterations = 4 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor) + // CHECK-NEXT: "tf.TensorListStack"([[map_fn_result]], %cst_0) <{ + %2 = "tf.TensorListStack"(%1#2, %cst_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 3 : i64} : (tensor>>, tensor<0xi32>) -> tensor<3xf32> + return %2 : tensor<3xf32> +} + +// ----- + +// Test a while to map_fn conversion in which max_iterations are passed +// into the predicate function. + +// CHECK-LABEL: @"map/while_cond" +func.func private @"map/while_cond"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor, %arg4: tensor>>, %arg5: tensor, %arg6: tensor) -> tensor { + %outputs = "tf.Less"(%arg0, %arg3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %outputs_0 = "tf.Less"(%arg1, %arg3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %outputs_2 = "tf.LogicalAnd"(%outputs_0, %outputs) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + + return %outputs_2 : tensor +} + +// CHECK-LABEL: @"map/while_body" +func.func private @"map/while_body"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor, %arg4: tensor>>, %arg5: tensor, %arg6: tensor) -> (tensor, tensor, tensor>>, tensor, tensor>>, tensor, tensor) { + %outputs = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %outputs_2 = "tf.AddV2"(%arg0, %outputs_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %outputs_4 = "tf.ReadVariableOp"(%arg4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor>>) -> tensor<3x1xf32> + %outputs_6 = "tf.Identity"(%outputs_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %outputs_8 = "tf.MatMul"(%arg5, %outputs_4) {device = "/job:localhost/replica:0/task:0/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor, tensor<3x1xf32>) -> tensor + %outputs_10 = "tf.AddV2"(%arg1, %outputs_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %outputs_12 = "tf.Identity"(%outputs_10) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %outputs_14 = "tf.GatherV2"(%arg6, %arg1, %outputs) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor<4xf32> + %outputs_16 = "tf.AddV2"(%outputs_8, %outputs_14) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<4xf32>) -> tensor + %outputs_18 = "tf.TensorListSetItem"(%arg2, %arg1, %outputs_16) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor) -> tensor>> + return %outputs_6, %outputs_12, %outputs_18, %arg3, %arg4, %arg5, %arg6 : tensor, tensor, tensor>>, tensor, tensor>>, tensor, tensor +} + +// CHECK-LABEL: @"map/while_body/MapFnBody" +// CHECK-SAME (%arg0: !mlrt.Future, %arg1: !mlrt.Promise, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor>>, %arg6: tensor, %arg7: tensor) +// CHECK-NEXT: [[cst_0:%.*]] = "tf.Const" +// CHECK-NEXT: [[cst_1:%.*]] = "tf.Const" +// CHECK-NEXT: [[loop_counter:%.*]] = "tf.AddV2"(%arg2, [[cst_1]]) +// CHECK-NEXT: [[weight:%.*]] = "tf.ReadVariableOp"(%arg5) +// CHECK-NEXT: [[mpy:%.*]] = "tf.MatMul"(%arg6, [[weight]]) +// CHECK-NEXT: [[element_index:%.*]] = "tf.AddV2"(%arg3, [[cst_1]]) +// CHECK-NEXT: [[bias:%.*]] = "tf.GatherV2"(%arg7, %arg3, [[cst_0]]) +// CHECK-NEXT: [[res:%.*]] = "tf.AddV2"([[mpy]], [[bias]]) +// CHECK-NEXT: [[ta_0:%.*]] = "tf_mlrt.tf_await"(%arg0) +// CHECK-NEXT: [[ta_1:%.*]] = "tf.TensorListSetItem"([[ta_0]], %arg3, [[res]]) +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg1, [[ta_1]]) +// CHECK-NEXT: return + +// CHECK-LABEL: func @main_while +func.func @main_while(%arg0: tensor, %arg1: tensor) -> tensor { + %outputs = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[-1, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %outputs_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_4 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_6 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %outputs_8 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + // CHECK: [[elems:%.*]] = "tf.VarHandleOp" + %outputs_10 = "tf.VarHandleOp"() {_xla_inferred_shapes = [#tf_type.shape<>], allowed_devices = [], container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", shared_name = "w"} : () -> tensor>> + %outputs_12 = "tf.Shape"(%arg1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor<2xi32> + // CHECK: [[max_iter:%.*]] = "tf.StridedSlice" + %outputs_14 = "tf.StridedSlice"(%outputs_12, %outputs_2, %outputs_4, %outputs_4) {begin_mask = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<2xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // CHECK: [[tensor_list:%.*]] = "tf.TensorListReserve" + %outputs_16 = "tf.TensorListReserve"(%outputs_0, %outputs_14) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor>> + // CHECK: tf_mlrt.tf_map_fn + // CHECK-SAME: ([[max_iter]], [[tensor_list]], [[max_iter]], [[elems]], %arg0, %arg1) + // CHECK-SAME: {body_fn = @"map/while_body/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} + // CHECK-NOT: tf.while + %outputs_18:7 = "tf.While"(%outputs_6, %outputs_6, %outputs_16, %outputs_14, %outputs_10, %arg0, %arg1) {_lower_using_switch_merge = true, _num_original_outputs = 8 : i64, _read_only_resource_inputs = [6], _xla_propagate_compile_time_consts = true, body = @"map/while_body", cond = @"map/while_cond", device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor, tensor>>, tensor, tensor) -> (tensor, tensor, tensor>>, tensor, tensor>>, tensor, tensor) + %outputs_20 = "tf.TensorListStack"(%outputs_18#2, %outputs) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = -1 : i64} : (tensor>>, tensor<2xi32>) -> tensor + return %outputs_20 : tensor +} + +// ----- + +// Test a while to map_fn conversion in which the passed in max_iterations +// is not in typical location of %arg3 and there are identify chains in function bodies. + +// CHECK-LABEL: @map_while_cond_170 +func.func private @map_while_cond_170(%arg0: tensor {tf._user_specified_name = "map/while/loop_counter"}, %arg1: tensor {tf._user_specified_name = "map/while/maximum_iterations"}, %arg2: tensor, %arg3: tensor, %arg4: tensor<*x!tf_type.variant>, %arg5: tensor<*xf32>) -> tensor<*xi1> attributes {tf._construction_context = "kEagerRuntime", tf._original_func_name = "map_while_cond_17"} { + %outputs = "tf.Const"() {device = "", value = dense<16> : tensor} : () -> tensor + %outputs_0 = "tf.Less"(%arg0, %arg1) {device = ""} : (tensor, tensor) -> tensor<*xi1> + %outputs_2 = "tf.Less"(%arg2, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi1> + %outputs_4 = "tf.LogicalAnd"(%outputs_0, %outputs_2) {device = ""} : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + %outputs_6 = "tf.Identity"(%outputs_4) {device = ""} : (tensor<*xi1>) -> tensor<*xi1> + return %outputs_6 : tensor<*xi1> +} + +// Original input argument list (loop_counter, max_iterations, element_index, tensor_list, read_only_tensor_list, scale) +// CHECK-LABEL: @map_while_body_180 +func.func private @map_while_body_180(%arg0: tensor {tf._user_specified_name = "map/while/loop_counter"}, %arg1: tensor {tf._user_specified_name = "map/while/maximum_iterations"}, %arg2: tensor, %arg3: tensor, %arg4: tensor {tf._user_specified_name = "map/TensorArrayUnstack/TensorListFromTensor"}, %arg5: tensor {tf._user_specified_name = "input"}) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.variant>, tensor, tensor) attributes {tf._construction_context = "kEagerRuntime", tf._original_func_name = "map_while_body_18"} { + %outputs = "tf.Const"() {device = "", value = dense<16> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_0 = "tf.Const"() {device = "", value = dense<16> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_2 = "tf.Const"() {device = "", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %outputs_4 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_6 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_8 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_10 = "tf.Const"() {device = "", value = dense<256> : tensor} : () -> tensor + %outputs_12 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_14 = "tf.Range"(%outputs_12, %outputs_10, %outputs_8) {device = ""} : (tensor, tensor, tensor) -> tensor<*xi32> + %outputs_16 = "tf.Cast"(%outputs_14) {Truncate = false, device = ""} : (tensor<*xi32>) -> tensor<*xf32> + %outputs_18 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_20 = "tf.Const"() {device = "", value = dense<257> : tensor} : () -> tensor + %outputs_22 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_24 = "tf.Range"(%outputs_22, %outputs_20, %outputs_18) {device = ""} : (tensor, tensor, tensor) -> tensor<*xi32> + %outputs_26 = "tf.Cast"(%outputs_24) {Truncate = false, device = ""} : (tensor<*xi32>) -> tensor<*xf32> + %outputs_28 = "tf.Const"() {device = "", value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_30 = "tf.Transpose"(%outputs_26, %outputs_28) {device = ""} : (tensor<*xf32>, tensor<1xi32>) -> tensor<*xf32> + %outputs_32 = "tf.AddV2"(%arg0, %outputs_6) {device = ""} : (tensor, tensor) -> tensor<*xi32> + %outputs_34 = "tf.Identity"(%outputs_32) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_36 = "tf.Identity"(%arg1) {device = ""} : (tensor) -> tensor<*xi32> + %outputs_38 = "tf.Mul"(%outputs_16, %arg5) {device = ""} : (tensor<*xf32>, tensor) -> tensor<*xf32> + %outputs_40 = "tf.Reshape"(%outputs_38, %outputs) {device = ""} : (tensor<*xf32>, tensor<2xi32>) -> tensor<*xf32> + %outputs_42 = "tf.AddV2"(%arg2, %outputs_4) {device = ""} : (tensor, tensor) -> tensor<*xi32> + %outputs_44 = "tf.Identity"(%outputs_42) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_46 = "tf.TensorListGetItem"(%arg4, %arg2, %outputs_2) {device = ""} : (tensor, tensor, tensor<0xi32>) -> tensor<*xi32> + %outputs_48 = "tf.Cast"(%outputs_46) {Truncate = false, device = ""} : (tensor<*xi32>) -> tensor<*xf32> + %outputs_50 = "tf.Mul"(%outputs_30, %outputs_48) {device = ""} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %outputs_52 = "tf.Reshape"(%outputs_50, %outputs_0) {device = ""} : (tensor<*xf32>, tensor<2xi32>) -> tensor<*xf32> + %outputs_54 = "tf.MatMul"(%outputs_40, %outputs_52) {device = "", transpose_a = false, transpose_b = false} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + %outputs_56 = "tf.MatrixDeterminant"(%outputs_54) {T = f32, device = ""} : (tensor<*xf32>) -> tensor<*xf32> + %outputs_58 = "tf.TensorListSetItem"(%arg3, %arg2, %outputs_56) {device = "", resize_if_index_out_of_bounds = false} : (tensor, tensor, tensor<*xf32>) -> tensor<*x!tf_type.variant> + %outputs_60 = "tf.Identity"(%outputs_58) {device = ""} : (tensor<*x!tf_type.variant>) -> tensor<*x!tf_type.variant> + return %outputs_34, %outputs_36, %outputs_44, %outputs_60, %arg4, %arg5 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32>, tensor<*x!tf_type.variant>, tensor, tensor +} + +// Converted input argument list (loop_counter, element_index, max_iterations, tensor_list, read_only_tensor_list, scale) +// CHECK-LABEL: @"map_while_body_180/MapFnBody" +// CHECK-SAME: (%arg0: !mlrt.future, %arg1: !mlrt.promise, %arg2: tensor {tf._user_specified_name = "map/while/loop_counter"}, %arg3: tensor, %arg4: tensor {tf._user_specified_name = "map/while/maximum_iterations"}, %arg5: tensor {tf._user_specified_name = "map/TensorArrayUnstack/TensorListFromTensor"}, %arg6: tensor {tf._user_specified_name = "input"}) +// CHECK: [[res:%.*]] = "tf.MatrixDeterminant" +// CHECK-NEXT: [[ta_0:%.*]] = "tf_mlrt.tf_await"(%arg0) +// CHECK-NEXT: [[ta_1:%.*]] = "tf.TensorListSetItem"([[ta_0]], %arg3, [[res]]) +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg1, [[ta_1]]) +// CHECK-NEXT: return + + +// CHECK-LABEL: __inference_while_from_map_fn_810 +// CHECK-SAME: ([[scale:%.*]]: tensor +func.func private @__inference_while_from_map_fn_810(%arg0: tensor {tf._user_specified_name = "input"}) -> tensor<*xf32> attributes {tf._construction_context = "kEagerRuntime", tf._original_func_name = "__inference_while_from_map_fn_81"} { + // CHECK: [[element_index:%.*]] = "tf.Const" + %outputs = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_0 = "tf.Const"() {device = "", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %outputs_2= "tf.Const"() {device = "", value = dense<-1> : tensor} : () -> tensor + %outputs_4 = "tf.Const"() {device = "", value = dense<16> : tensor} : () -> tensor + // CHECK: tf.TensorListReserve + %outputs_6 = "tf.TensorListReserve"(%outputs_2, %outputs_4) {device = ""} : (tensor, tensor) -> tensor>> + %outputs_8 = "tf.Const"() {device = "", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %outputs_10 = "tf.Const"() {device = "", value = dense<-1> : tensor} : () -> tensor + %outputs_12 = "tf.Const"() {device = "", value = dense<16> : tensor} : () -> tensor + // CHECK: [[tensor_list:%.*]] = "tf.TensorListReserve"([[shape:%.*]], [[reserve_size:%.*]]) { + %outputs_14 = "tf.TensorListReserve"(%outputs_10, %outputs_12) {device = ""} : (tensor, tensor) -> tensor>> + // CHECK-NEXT: [[loop_counter:%.*]] = "tf.Const" + %outputs_16 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + // CHECK-NEXT: [[max_iterations:%.*]] = "tf.Const" + %outputs_18 = "tf.Const"() {device = "", value = dense<16> : tensor} : () -> tensor + %outputs_20 = "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_22 = "tf.Const"() {device = "", value = dense<16> : tensor} : () -> tensor + %outputs_24 = "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_26 = "tf.Range"(%outputs_24, %outputs_22, %outputs_20) {device = ""} : (tensor, tensor, tensor) -> tensor<*xi32> + // CHECK: [[read_only_tensor_list:%.*]] = "tf.TensorListFromTensor" + %outputs_28 = "tf.TensorListFromTensor"(%outputs_26, %outputs_0) {device = ""} : (tensor<*xi32>, tensor<0xi32>) -> tensor<*x!tf_type.variant> +// CHECK: [[map_fn_out:%.*]] = tf_mlrt.tf_map_fn + // CHECK-SAME: ([[reserve_size]], [[tensor_list]], [[max_iterations]], [[read_only_tensor_list]], [[scale]]) + // CHECK-SAME: {body_fn = @"map_while_body_180/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} + // CHECK-NOT: tf.While + %outputs_30:6 = "tf.While"(%outputs_16, %outputs_18, %outputs, %outputs_14, %outputs_28, %arg0) {T = [i32, i32, i32, !tf_type.variant, !tf_type.variant, f32], _lower_using_switch_merge = true, _num_original_outputs = 6 : i64, _read_only_resource_inputs = [], body = @map_while_body_180, cond = @map_while_cond_170, device = "", is_stateless = true, output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape], parallel_iterations = 4 : i64, shape_invariant} : (tensor, tensor, tensor, tensor>>, tensor<*x!tf_type.variant>, tensor) -> (tensor, tensor, tensor, tensor, tensor, tensor) + // CHECK-NEXT: "tf.TensorListStack" + // CHECK-SAME: ([[map_fn_out]], + %outputs_32 = "tf.TensorListStack"(%outputs_30#3, %outputs_8) {device = "", num_elements = 16 : i64} : (tensor, tensor<0xi32>) -> tensor<*xf32> + %outputs_34 = "tf.Identity"(%outputs_32) {device = ""} : (tensor<*xf32>) -> tensor<*xf32> + return %outputs_34 : tensor<*xf32> +} + +// ----- + +// Test a while to map_fn conversion in which tensor array is used instead of +// tensor list. + +// CHECK-LABEL: map/while/LoopCond_cond +func.func private @"map/while/LoopCond_cond"(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor, %arg3: tensor, %arg4: tensor<2x!tf_type.resource>>, %arg5: tensor, %arg6: tensor<2x!tf_type.resource>>) -> tensor { + %outputs = "tf.Less"(%arg0, %arg3) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi1> + %outputs_0 = "tf.Less"(%arg1, %arg3) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi1> + %outputs_2 = "tf.LogicalAnd"(%outputs, %outputs_0) {device = ""} : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + %outputs_4 = "tf.ToBool"(%outputs_2) : (tensor<*xi1>) -> tensor + return %outputs_4 : tensor +} + +// CHECK-LABEL: map/while/LoopCond_body +func.func private @"map/while/LoopCond_body"(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor, %arg3: tensor, %arg4: tensor<2x!tf_type.resource>>, %arg5: tensor, %arg6: tensor<2x!tf_type.resource>>) -> (tensor<*xi32>, tensor<*xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) { + %outputs = "tf.Const"() {value = dense<224> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %outputs_2 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %outputs_4 = "tf.Identity"(%arg0) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_6 = "tf.AddV2"(%outputs_4, %outputs_2) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_8 = "tf.Identity"(%arg1) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_10 = "tf.AddV2"(%outputs_8, %outputs_2) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_12 = "tf.Identity"(%arg2) {device = ""} : (tensor) -> tensor + %outputs_14 = "tf.TensorArrayReadV3"(%arg4, %outputs_8, %arg5) {device = ""} : (tensor<2x!tf_type.resource>>, tensor<*xi32>, tensor) -> tensor<*x!tf_type.string> + %outputs_16 = "tf.DecodeJpeg"(%outputs_14) {acceptable_fraction = 1.000000e+00 : f32, channels = 3 : i64, dct_method = "INTEGER_FAST", device = "", fancy_upscaling = true, ratio = 1 : i64, try_recover_truncated = false} : (tensor<*x!tf_type.string>) -> tensor + %outputs_18 = "tf.ExpandDims"(%outputs_16, %outputs_0) {device = ""} : (tensor, tensor) -> tensor<1x?x?x3xui8> + %outputs_20 = "tf.ResizeBilinear"(%outputs_18, %outputs) {align_corners = false, device = "", half_pixel_centers = false} : (tensor<1x?x?x3xui8>, tensor<2xi32>) -> tensor<1x224x224x3xf32> + %outputs_22 = "tf.Squeeze"(%outputs_20) {device = "", squeeze_dims = [0]} : (tensor<1x224x224x3xf32>) -> tensor<224x224x3xf32> + %outputs_24 = "tf.Cast"(%outputs_22) {Truncate = false, device = ""} : (tensor<224x224x3xf32>) -> tensor<224x224x3xui8> + %outputs_26 = "tf.TensorArrayWriteV3"(%arg6, %outputs_8, %outputs_24, %outputs_12) {device = ""} : (tensor<2x!tf_type.resource>>, tensor<*xi32>, tensor<224x224x3xui8>, tensor) -> tensor + return %outputs_6, %outputs_10, %outputs_26, %arg3, %arg4, %arg5, %arg6: tensor<*xi32>, tensor<*xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>> +} + +// CHECK-LABEL: @"map/while/LoopCond_body/MapFnBody" +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.TensorArrayReadV3 +// CHECK-NEXT: tf.DecodeJpeg +// CHECK-NEXT: tf.ExpandDims +// CHECK-NEXT: tf.ResizeBilinear +// CHECK-NEXT: tf.Squeeze +// CHECK-NEXT: tf.Cast +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf.TensorArrayWriteV3 +// CHECK-NEXT: tf_mlrt.tf_promise +// CHECK-NEXT: return + +//CHECK-LABEL: map_while_test +func.func @map_while_test(%arg0: tensor) -> tensor { + %outputs = "tf.Const"() {value = dense<0> : tensor} : () -> tensor<1xi32> + %outputs_0 = "tf.Const"() {value = dense<224> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_2 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %outputs_4 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_6 = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_8 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %outputs_10 = "tf.Shape"(%arg0) {device = ""} : (tensor) -> tensor<1xi32> + // CHECK: [[max_iter:%.*]] = "tf.StridedSlice" + %outputs_12 = "tf.StridedSlice"(%outputs_10, %outputs_6, %outputs_4, %outputs_4) {begin_mask = 0 : i64, device = "", ellipsis_mask = 0 : i64, end_mask = 0 : i64, new_axis_mask = 0 : i64, shrink_axis_mask = 1 : i64} : (tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor + // CHECK-NEXT: tf.Range + %outputs_14 = "tf.Range"(%outputs_2, %outputs_12, %outputs_8) {device = ""} : (tensor, tensor, tensor) -> tensor + // CHECK-NEXT: [[handle_1:%.*]], [[flow_in_1:%.*]] = "tf.TensorArrayV3" + %outputs_16:2 = "tf.TensorArrayV3"(%outputs_12) {clear_after_read = true, device = "", dtype = !tf_type.string, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK-NEXT: [[handle_2:%.*]] = "tf.TensorArrayScatterV3" + %outputs_18 = "tf.TensorArrayScatterV3"(%outputs_16#0, %outputs_14, %arg0, %outputs_16#1) {device = ""} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + // CHECK-NEXT: tf.Range + %outputs_20 = "tf.Range"(%outputs_2, %outputs_12, %outputs_8) {device = ""} : (tensor, tensor, tensor) -> tensor + // CHECK-NEXT: [[tensor_array:%.*]], [[flow_in:%.*]] = "tf.TensorArrayV3" + %outputs_22:2 = "tf.TensorArrayV3"(%outputs_12) {clear_after_read = true, device = "", dtype = ui8, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK-NEXT: tf_mlrt.tf_map_fn + // CHECK-SAME: ([[max_iter]], [[flow_in]], [[max_iter]], [[handle_1]], [[handle_2]], [[tensor_array]]) + // CHECK-SAME: {body_fn = @"map/while/LoopCond_body/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} + // CHECK-NOT: tf.While + %outputs_24:7 = "tf.While"(%outputs, %outputs, %outputs_22#1, %outputs_12, %outputs_16#0, %outputs_18, %outputs_22#0) {_xla_propagate_compile_time_consts = true, body = @"map/while/LoopCond_body", cond = @"map/while/LoopCond_cond", device = "", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor<1xi32>, tensor<1xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) -> (tensor<1xi32>, tensor<1xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) + // CHECK-NEXT: tf.TensorArrayGatherV3 + %outputs_26 = "tf.TensorArrayGatherV3"(%outputs_22#0, %outputs_20, %outputs_24#2) {device = "", element_shape = #tf_type.shape<224x224x3>} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + return %outputs_26 : tensor +} + +// ----- +// Test non-applicable while is NOT converted to map_fn. + +// CHECK-LABEL: func @while_cond_lt9 +func.func @while_cond_lt9(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_body_add2 +func.func @while_body_add2(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_test() +func.func @while_test() -> (tensor) { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: tf.While + %1 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + func.return %1 : tensor +} + +// ----- + +// Test a case that the while body has multiple tensor lists. + +// CHECK-LABEL: tf.MultiListWhileRegion_body +func.func private @tf.MultiListWhileRegion_body(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor>>, %arg4: tensor) -> (tensor, tensor, tensor>>, tensor>>, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[[0.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00, 7.000000e+00], [8.000000e+00, 9.000000e+00, 1.000000e+01, 1.100000e+01, 1.200000e+01, 1.300000e+01, 1.400000e+01, 1.500000e+01]]> : tensor<2x8xf32>} : () -> tensor<2x8xf32> + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[[1.600000e+01, 1.700000e+01, 1.800000e+01, 1.900000e+01, 2.000000e+01, 2.100000e+01, 2.200000e+01, 2.300000e+01], [2.400000e+01, 2.500000e+01, 2.600000e+01, 2.700000e+01, 2.800000e+01, 2.900000e+01, 3.000000e+01, 3.100000e+01]]> : tensor<2x8xf32>} : () -> tensor<2x8xf32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %0 = "tf.GatherV2"(%arg4, %cst_2, %cst_2) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + %1 = "tf.AddV2"(%arg0, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%arg1, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %3 = "tf.GatherV2"(%cst_0, %arg1, %cst_2) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x8xf32>, tensor, tensor) -> tensor<8xf32> + %4 = "tf.Mul"(%0, %3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<8xf32>) -> tensor<8xf32> + %5 = "tf.TensorListSetItem"(%arg2, %arg1, %4) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor<8xf32>) -> tensor>> + %6 = "tf.GatherV2"(%cst, %arg1, %cst_2) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x8xf32>, tensor, tensor) -> tensor<8xf32> + %7 = "tf.Mul"(%0, %6) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<8xf32>) -> tensor<8xf32> + %8 = "tf.TensorListSetItem"(%arg3, %arg1, %7) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor<8xf32>) -> tensor>> + return %1, %2, %5, %8, %arg4 : tensor, tensor, tensor>>, tensor>>, tensor +} + +// CHECK-LABEL: tf.MultiListWhileRegion_body/MapFnBody +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.GatherV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.GatherV2 +// CHECK-NEXT: tf.Mul +// CHECK-NEXT: tf.GatherV2 +// CHECK-NEXT: tf.Mul +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf.TensorListSetItem +// CHECK-NEXT: tf.TensorListSetItem +// CHECK-NEXT: tf_mlrt.tf_promise +// CHECK-NEXT: tf_mlrt.tf_promise +// CHECK-NEXT: return + +// CHECK-LABEL: tf.MultiListWhileRegion_cond +func.func private @tf.MultiListWhileRegion_cond(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor>>, %arg4: tensor) -> tensor { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %0 = "tf.Less"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg1, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + return %2 : tensor +} + +// CHECK-LABEL: multilist_serving +func.func private @multilist_serving(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) -> (tensor<2x8xf32>, tensor<2x8xf32>) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<8> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<2> : tensor} : () -> tensor + // CHECK: TensorListReserve + %0 = "tf.TensorListReserve"(%cst_1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor>> + // CHECK-NEXT: tf_mlrt.tf_map_fn + %1:5 = "tf.While"(%cst, %cst, %0, %0, %arg0) {_lower_using_switch_merge = true, _num_original_outputs = 8 : i64, _read_only_resource_inputs = [], _xla_propagate_compile_time_consts = true, body = @tf.MultiListWhileRegion_body, cond = @tf.MultiListWhileRegion_cond, device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = true, parallel_iterations = 4 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor>>, tensor) + // CHECK-NEXT: TensorListStack + %2 = "tf.TensorListStack"(%1#2, %cst_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 2 : i64} : (tensor>>, tensor<1xi32>) -> tensor<2x8xf32> + %3 = "tf.TensorListStack"(%1#3, %cst_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 2 : i64} : (tensor>>, tensor<1xi32>) -> tensor<2x8xf32> + return %3, %2 : tensor<2x8xf32>, tensor<2x8xf32> +} + + +// ----- + +// Convert a while with multiple tensor array to map_fn + +// CHECK-LABEL: tf.WhileRegion1_body( +func.func private @tf.WhileRegion1_body(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor<2x!tf_type.resource>>, %arg6: tensor<2x!tf_type.resource>>, %arg7: tensor<*xi32>) -> (tensor, tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>, tensor<*xi32>) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%arg0, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.AddV2"(%arg1, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %3 = "tf.RaggedTensorToVariant"(%arg7) {RAGGED_RANK = 0 : i64, Tsplits = i64, Tvalues = i32, batched_input = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*xi32>) -> tensor + %4 = "tf.TensorArrayWriteV3"(%arg5, %arg1, %3, %arg2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + %5 = "tf.RaggedTensorToVariant"(%arg7) {RAGGED_RANK = 0 : i64, Tsplits = i64, Tvalues = f32, batched_input = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*xi32>) -> tensor + %6 = "tf.TensorArrayWriteV3"(%arg6, %arg1, %5, %arg3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + return %0, %1, %4, %6, %arg4, %arg5, %arg6, %arg7 : tensor, tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>, tensor<*xi32> +} + +// CHECK-LABEL: func.func private @"tf.WhileRegion1_body/MapFnBody"(%arg0: !mlrt.future, %arg1: !mlrt.promise, %arg2: !mlrt.future, %arg3: !mlrt.promise, %arg4: tensor, %arg5: tensor, %arg6: tensor, %arg7: tensor<2x!tf_type.resource>>, %arg8: tensor<2x!tf_type.resource>>, %arg9: tensor<*xi32>) attributes {tfrt.cost_threshold = 4294967295 : i64} +// CHECK: [[result_0:%.*]] = "tf.RaggedTensorToVariant" +// CHECK: [[result_1:%.*]] = "tf.RaggedTensorToVariant" +// CHECK-NEXT: [[flow_in_0:%.*]] = "tf_mlrt.tf_await"(%arg0) : (!mlrt.future) -> tensor +// CHECK-NEXT: [[flow_in_1:%.*]] = "tf_mlrt.tf_await"(%arg2) : (!mlrt.future) -> tensor +// CHECK-NEXT: [[flow_out_0:%.*]] = "tf.TensorArrayWriteV3"(%arg7, %arg5, [[result_0]], [[flow_in_0]]) +// CHECK-NEXT: [[flow_out_1:%.*]] = "tf.TensorArrayWriteV3"(%arg8, %arg5, [[result_1]], [[flow_in_1]]) +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg1, [[flow_out_0]]) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: "tf_mlrt.tf_promise"(%arg3, [[flow_out_1]]) : (!mlrt.promise, tensor) -> () +// CHECK-NEXT: return + +// CHECK-LABEL: tf.WhileRegion1_cond +func.func private @tf.WhileRegion1_cond(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor, %arg4: tensor, %arg5: tensor<2x!tf_type.resource>>, %arg6: tensor<2x!tf_type.resource>>, %arg7: tensor<*xi32>) -> (tensor) { + %0 = "tf.Less"(%arg0, %arg4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor<*xi1> + %1 = "tf.Less"(%arg1, %arg4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor<*xi1> + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + %3 = "tf.ToBool"(%2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<*xi1>) -> tensor + return %3 : tensor +} + +// CHECK-LABEL: func.func private @tf.WhileRegion2_body( +func.func private @tf.WhileRegion2_body(%arg0: tensor<*xi32>) -> (tensor, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[-1, 4]> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %max_iter = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<4> : tensor} : () -> tensor + // CHECK: "tf.TensorArrayV3" + %handle_12, %flow_13 = "tf.TensorArrayV3"(%max_iter) {device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = !tf_type.variant, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK: "tf.TensorArrayV3" + %handle_14, %flow_15 = "tf.TensorArrayV3"(%max_iter) {device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = !tf_type.variant, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK: tf_mlrt.tf_map_fn + // CHECK-SAME: {body_fn = @"tf.WhileRegion1_body/MapFnBody", num_tensor_list_or_flow_in = 2 : i32} + %4:8 = "tf.While"(%cst_0, %cst_0, %flow_13, %flow_15, %max_iter, %handle_12, %handle_14, %arg0) {body = @tf.WhileRegion1_body, cond = @tf.WhileRegion1_cond, device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>, tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>, tensor<*xi32>) + // CHECK: TensorArrayGatherV3 + %5 = "tf.TensorArrayGatherV3"(%handle_12, %1, %4#2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + // CHECK: TensorArrayGatherV3 + %6 = "tf.TensorArrayGatherV3"(%handle_14, %2, %4#3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + return %5, %6 : tensor, tensor +} + +// ----- + +// Test a while to map_fn conversion in which tensor array is used instead of +// tensor list and the tensor array size and the number of iterations are bounded +// by separate constants of the same value. + +// CHECK-LABEL: map2/while/LoopCond_body +func.func private @"map2/while/LoopCond_body"(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor, %arg3: tensor, %arg4: tensor<2x!tf_type.resource>>, %arg5: tensor, %arg6: tensor<2x!tf_type.resource>>) -> (tensor<*xi32>, tensor<*xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) { + %outputs = "tf.Const"() {value = dense<224> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %outputs_2 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %outputs_4 = "tf.Identity"(%arg0) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_6 = "tf.AddV2"(%outputs_4, %outputs_2) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_8 = "tf.Identity"(%arg1) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_10 = "tf.AddV2"(%outputs_8, %outputs_2) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi32> + %outputs_12 = "tf.Identity"(%arg2) {device = ""} : (tensor) -> tensor + %outputs_14 = "tf.TensorArrayReadV3"(%arg4, %outputs_8, %arg5) {device = ""} : (tensor<2x!tf_type.resource>>, tensor<*xi32>, tensor) -> tensor<*x!tf_type.string> + %outputs_16 = "tf.DecodeJpeg"(%outputs_14) {acceptable_fraction = 1.000000e+00 : f32, channels = 3 : i64, dct_method = "INTEGER_FAST", device = "", fancy_upscaling = true, ratio = 1 : i64, try_recover_truncated = false} : (tensor<*x!tf_type.string>) -> tensor + %outputs_18 = "tf.ExpandDims"(%outputs_16, %outputs_0) {device = ""} : (tensor, tensor) -> tensor<1x?x?x3xui8> + %outputs_20 = "tf.ResizeBilinear"(%outputs_18, %outputs) {align_corners = false, device = "", half_pixel_centers = false} : (tensor<1x?x?x3xui8>, tensor<2xi32>) -> tensor<1x224x224x3xf32> + %outputs_22 = "tf.Squeeze"(%outputs_20) {device = "", squeeze_dims = [0]} : (tensor<1x224x224x3xf32>) -> tensor<224x224x3xf32> + %outputs_24 = "tf.Cast"(%outputs_22) {Truncate = false, device = ""} : (tensor<224x224x3xf32>) -> tensor<224x224x3xui8> + %outputs_26 = "tf.TensorArrayWriteV3"(%arg6, %outputs_8, %outputs_24, %outputs_12) {device = ""} : (tensor<2x!tf_type.resource>>, tensor<*xi32>, tensor<224x224x3xui8>, tensor) -> tensor + return %outputs_6, %outputs_10, %outputs_26, %arg3, %arg4, %arg5, %arg6: tensor<*xi32>, tensor<*xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>> +} + +// CHECK-LABEL: @"map2/while/LoopCond_body/MapFnBody" +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.TensorArrayReadV3 +// CHECK-NEXT: tf.DecodeJpeg +// CHECK-NEXT: tf.ExpandDims +// CHECK-NEXT: tf.ResizeBilinear +// CHECK-NEXT: tf.Squeeze +// CHECK-NEXT: tf.Cast +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf.TensorArrayWriteV3 +// CHECK-NEXT: tf_mlrt.tf_promise +// CHECK-NEXT: return + +// CHECK-LABEL: map2/while/LoopCond_cond +func.func private @"map2/while/LoopCond_cond"(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor, %arg3: tensor, %arg4: tensor<2x!tf_type.resource>>, %arg5: tensor, %arg6: tensor<2x!tf_type.resource>>) -> tensor { + %cst = "tf.Const"() {value = dense<224> : tensor} : () -> tensor + %outputs = "tf.Less"(%arg0, %cst) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi1> + %outputs_0 = "tf.Less"(%arg1, %cst) {device = ""} : (tensor<*xi32>, tensor) -> tensor<*xi1> + %outputs_2 = "tf.LogicalAnd"(%outputs, %outputs_0) {device = ""} : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + %outputs_4 = "tf.ToBool"(%outputs_2) : (tensor<*xi1>) -> tensor + return %outputs_4 : tensor +} + +//CHECK-LABEL: map2_while_test +func.func private @map2_while_test(%arg0: tensor) -> tensor { + // CHECK-NEXT: tf.Const + %outputs = "tf.Const"() {value = dense<0> : tensor} : () -> tensor<1xi32> + // CHECK-NEXT: [[max_iter:%.*]] = "tf.Const" + %cst_0 = "tf.Const"() {value = dense<224> : tensor} : () -> tensor + %cst_1 = "tf.Const"() {value = dense<256> : tensor} : () -> tensor + %outputs_2 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %outputs_4 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_6 = "tf.Const"() {value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + %outputs_8 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %outputs_10 = "tf.Shape"(%arg0) {device = ""} : (tensor) -> tensor<1xi32> + // CHECK: tf.Range + %outputs_14 = "tf.Range"(%outputs_2, %cst_0, %outputs_8) {device = ""} : (tensor, tensor, tensor) -> tensor + // CHECK-NEXT: tf.TensorArrayV3 + %outputs_16:2 = "tf.TensorArrayV3"(%cst_0) {clear_after_read = true, device = "", dtype = !tf_type.string, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK-NEXT: tf.TensorArrayScatterV3 + %outputs_18 = "tf.TensorArrayScatterV3"(%outputs_16#0, %outputs_14, %arg0, %outputs_16#1) {device = ""} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + // CHECK-NEXT: tf.Range + %outputs_20 = "tf.Range"(%outputs_2, %cst_0, %outputs_8) {device = ""} : (tensor, tensor, tensor) -> tensor + // CHECK-NEXT: [[tensor_array:%.*]], [[flow_in:%.*]] = "tf.TensorArrayV3" + %outputs_22:2 = "tf.TensorArrayV3"(%cst_0) {clear_after_read = true, device = "", dtype = ui8, dynamic_size = false, element_shape = #tf_type.shape<*>, identical_element_shapes = true, tensor_array_name = ""} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + // CHECK-NEXT: tf_mlrt.tf_map_fn + // CHECK-SAME: ([[max_iter]], [[flow_in]], %cst_1 + // CHECK-SAME: {body_fn = @"map2/while/LoopCond_body/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} + // CHECK-NOT: tf.While + %outputs_24:7 = "tf.While"(%outputs, %outputs, %outputs_22#1, %cst_1, %outputs_16#0, %outputs_18, %outputs_22#0) {_xla_propagate_compile_time_consts = true, body = @"map2/while/LoopCond_body", cond = @"map2/while/LoopCond_cond", device = "", is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor<1xi32>, tensor<1xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) -> (tensor<1xi32>, tensor<1xi32>, tensor, tensor, tensor<2x!tf_type.resource>>, tensor, tensor<2x!tf_type.resource>>) + // CHECK-NEXT: tf.TensorArrayGatherV3 + %outputs_26 = "tf.TensorArrayGatherV3"(%outputs_22#0, %outputs_20, %outputs_24#2) {device = "", element_shape = #tf_type.shape<224x224x3>} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + return %outputs_26 : tensor +} + +// ----- +// Test a nest while in which the while body is after the usage. + +// CHECK-LABEL: nested_while +func.func @nested_while(%arg0: tensor {tf.device = "/job:localhost/replica:0/task:0/device:CPU:0"}) -> (tensor<16x16x?xf32>) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[16, -1]> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<16> : tensor} : () -> tensor + // CHECK: tf.TensorListReserve + %0 = "tf.TensorListReserve"(%cst_1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor>> + // CHECK-NEXT: tf_mlrt.tf_map_fn + %1:4 = "tf.While"(%cst, %cst, %0, %arg0) {_lower_using_switch_merge = true, _num_original_outputs = 6 : i64, _read_only_resource_inputs = [], _xla_propagate_compile_time_consts = true, body = @tf.NestedWhileRegion1_body, cond = @tf.NestedWhileRegion1_cond, device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = true, parallel_iterations = 4 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor) + %2 = "tf.TensorListStack"(%1#2, %cst_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 16 : i64} : (tensor>>, tensor<2xi32>) -> tensor<16x16x?xf32> + return %2 : tensor<16x16x?xf32> +} +// CHECK-LABEL: tf.NestedWhileRegion1_body +func.func private @tf.NestedWhileRegion1_body(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> (tensor, tensor, tensor>>, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : tensor<16xi32>} : () -> tensor<16xi32> + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %cst_3 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<16> : tensor} : () -> tensor + %cst_4 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %0 = "tf.TensorListReserve"(%cst_4, %cst_3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor>> + %1 = "tf.AddV2"(%arg0, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%arg1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %3 = "tf.GatherV2"(%cst_1, %arg1, %cst_0) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<16xi32>, tensor, tensor) -> tensor + %4 = "tf.Cast"(%3) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %5 = "tf.Mul"(%arg3, %4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %6:4 = "tf.While"(%cst_0, %cst_0, %0, %5) {_lower_using_switch_merge = true, _num_original_outputs = 6 : i64, _read_only_resource_inputs = [], _xla_propagate_compile_time_consts = true, body = @tf.NestedWhileRegion_body, cond = @tf.NestedWhileRegion_cond, device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = true, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor) + %7 = "tf.TensorListStack"(%6#2, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 16 : i64} : (tensor>>, tensor<1xi32>) -> tensor<16x?xf32> + %8 = "tf.TensorListSetItem"(%arg2, %arg1, %7) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor<16x?xf32>) -> tensor>> + return %1, %2, %8, %arg3 : tensor, tensor, tensor>>, tensor +} + +//CHECK-LABEL: @"tf.NestedWhileRegion1_body/MapFnBody"(%arg0: !mlrt.future, %arg1: !mlrt.promise, %arg2: tensor, %arg3: tensor, %arg4: tensor) +// CHECK: tf.TensorListReserve +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.GatherV2 +// CHECK-NEXT: tf.Cast +// CHECK-NEXT: tf.Mul +// CHECK-NEXT: tf_mlrt.tf_map_fn +// CHECK-NEXT: tf.TensorListStack +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf.TensorListSetItem +// CHECK-NEXT: tf_mlrt.tf_promise +// CHECK-NEXT: return + +// CHECK-LABEL: tf.NestedWhileRegion1_cond +func.func private @tf.NestedWhileRegion1_cond(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> tensor { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<16> : tensor} : () -> tensor + %0 = "tf.Less"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg1, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + return %2 : tensor +} +// CHECK-LABEL: tf.NestedWhileRegion_body +func.func private @tf.NestedWhileRegion_body(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> (tensor, tensor, tensor>>, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]> : tensor<16xi32>} : () -> tensor<16xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%arg0, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.AddV2"(%arg1, %cst_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.GatherV2"(%cst_0, %arg1, %cst) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<16xi32>, tensor, tensor) -> tensor + %3 = "tf.Cast"(%2) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %4 = "tf.Mul"(%arg3, %3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %5 = "tf.TensorListSetItem"(%arg2, %arg1, %4) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor) -> tensor>> + return %0, %1, %5, %arg3 : tensor, tensor, tensor>>, tensor +} + +// CHECK-LABEL: tf.NestedWhileRegion_body/MapFnBody +// CHECK: tf.AddV2 +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf.GatherV2 +// CHECK-NEXT: tf.Cast +// CHECK-NEXT: tf.Mul +// CHECK-NEXT: tf_mlrt.tf_await +// CHECK-NEXT: tf.TensorListSetItem +// CHECK-NEXT: "tf_mlrt.tf_promise +// CHECK-NEXT: return + +// CHECK-LABEL: tf.NestedWhileRegion_cond +func.func private @tf.NestedWhileRegion_cond(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> tensor { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<16> : tensor} : () -> tensor + %0 = "tf.Less"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg1, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + return %2 : tensor +} + +// ----- + +// Test a while to map_fn conversion is skipped if the tensor list cannot be found in the current function body. + +// CHECK-LABEL: map/while_cond +func.func private @"map/while_cond"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> tensor { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor} : () -> tensor + %0 = "tf.Less"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg1, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + return %2 : tensor +} + +// CHECK-LABEL: map/while_body +func.func private @"map/while_body"(%arg0: tensor, %arg1: tensor, %arg2: tensor>>, %arg3: tensor) -> (tensor, tensor, tensor>>, tensor) { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00, 7.000000e+00, 8.000000e+00, 9.000000e+00]> : tensor<9xf32>} : () -> tensor<9xf32> + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0, 1, 2]> : tensor<3xi32>} : () -> tensor<3xi32> + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor<2xi32>} : () -> tensor<2xi32> + %cst_3 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<[0.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00, 7.000000e+00, 8.000000e+00]> : tensor<9xf32>} : () -> tensor<9xf32> + %cst_4 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %0 = "tf.AddV2"(%arg0, %cst_4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Mul"(%arg3, %cst_3) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<9xf32>) -> tensor<9xf32> + %2 = "tf.Reshape"(%1, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<9xf32>, tensor<2xi32>) -> tensor<3x3xf32> + %3 = "tf.AddV2"(%arg1, %cst_4) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + %4 = "tf.GatherV2"(%cst_1, %arg1, %cst_0) {batch_dims = 0 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<3xi32>, tensor, tensor) -> tensor + %5 = "tf.Cast"(%4) {Truncate = false, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor + %6 = "tf.Mul"(%5, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<9xf32>) -> tensor<9xf32> + %7 = "tf.Reshape"(%6, %cst_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<9xf32>, tensor<2xi32>) -> tensor<3x3xf32> + %8 = "tf.MatMul"(%2, %7) {device = "/job:localhost/replica:0/task:0/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> + %9 = "tf.MatrixDeterminant"(%8) {T = f32, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<3x3xf32>) -> tensor + %10 = "tf.TensorListSetItem"(%arg2, %arg1, %9) {device = "/job:localhost/replica:0/task:0/device:CPU:0", resize_if_index_out_of_bounds = false} : (tensor>>, tensor, tensor) -> tensor>> + return %0, %3, %10, %arg3 : tensor, tensor, tensor>>, tensor +} + +//CHECK-LABEL: @func +func.func @func(%arg0: tensor, %arg1: tensor>>) -> tensor<3xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "serving_default_input:0", outputs = "PartitionedCall:0"}} { + %cst = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %cst_0 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<> : tensor<0xi32>} : () -> tensor<0xi32> + %cst_1 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %cst_2 = "tf.Const"() {device = "/job:localhost/replica:0/task:0/device:CPU:0", value = dense<3> : tensor} : () -> tensor + // CHECK-NOT: tf_map_fn + %1:4 = "tf.While"(%cst, %cst, %arg1, %arg0) {_lower_using_switch_merge = true, _num_original_outputs = 6 : i64, _read_only_resource_inputs = [], _xla_propagate_compile_time_consts = true, body = @"map/while_body", cond = @"map/while_cond", device = "/job:localhost/replica:0/task:0/device:CPU:0", is_stateless = true, parallel_iterations = 4 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor) + %2 = "tf.TensorListStack"(%1#2, %cst_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0", num_elements = 3 : i64} : (tensor>>, tensor<0xi32>) -> tensor<3xf32> + return %2 : tensor<3xf32> +} + +// ----- + +// Test a while to map_fn conversion in which a tf.StopGradient is inserted to consume the while result. +// CHECK-LABEL: @while_map_while_body_884030 +func.func private @while_map_while_body_884030(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor>>, %arg4: tensor>> {tf._user_specified_name = "while/map/TensorArrayUnstack/TensorListFromTensor"}) -> (tensor, tensor, tensor, tensor>>, tensor>>) { + %cst = "tf.Const"() <{value = dense<[-1, -1, 1]> : tensor<3xi32>}> : () -> tensor<3xi32> + %cst_0 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %0 = "tf.AddV2"(%arg2, %cst_0) : (tensor, tensor) -> tensor + %1 = "tf.Identity"(%0) : (tensor) -> tensor + %2 = "tf.TensorListGetItem"(%arg4, %arg2, %cst) : (tensor>>, tensor, tensor<3xi32>) -> tensor + %3 = "tf.EncodePng"(%2) <{compression = -1 : i64}> : (tensor) -> tensor + %4 = "tf.TensorListSetItem"(%arg3, %arg2, %3) <{resize_if_index_out_of_bounds = false}> : (tensor>>, tensor, tensor) -> tensor>> + %5 = "tf.Identity"(%4) : (tensor>>) -> tensor>> + %6 = "tf.AddV2"(%arg0, %cst_0) : (tensor, tensor) -> tensor + %7 = "tf.Identity"(%6) : (tensor) -> tensor + %8 = "tf.Identity"(%arg1) : (tensor) -> tensor + return %7, %8, %1, %5, %arg4 : tensor, tensor, tensor, tensor>>, tensor>> +} + +// CHECK-LABEL: while_map_while_body_884030/MapFnBody +// CHECK: tf.AddV2 +// CHECK-NEXT: tf.TensorListGetItem +// CHECK-NEXT: tf.EncodePng +// CHECK-NEXT: tf.AddV2 +// CHECK-NEXT: tf_await +// CHECK-NEXT: tf.TensorListSetItem +// CHECK-NEXT: tf_promise + +// CHECK-LABEL: @while_map_while_cond_884020 +func.func private @while_map_while_cond_884020(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor>>, %arg4: tensor>>) -> tensor { + %cst = "tf.Const"() <{value = dense<11> : tensor}> : () -> tensor + %0 = "tf.Less"(%arg2, %cst) : (tensor, tensor) -> tensor + %1 = "tf.Less"(%arg0, %arg1) : (tensor, tensor) -> tensor + %2 = "tf.LogicalAnd"(%1, %0) : (tensor, tensor) -> tensor + %3 = "tf.Identity"(%2) : (tensor) -> tensor + return %3 : tensor +} + +// CHECK-LABEL: @main +// CHECK: tf.Cast +// CHECK-NEXT: tf.TensorListReserve +// CHECK-NEXT: tf.Transpose +// CHECK-NEXT: tf.TensorListFromTensor +// CHECK-NEXT: tf_mlrt.tf_map_fn +// CHECK-SAME: {body_fn = @"while_map_while_body_884030/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} : (tensor, tensor>>, tensor, tensor>>) -> tensor>> +// CHECK-NEXT: tf.StopGradient +// CHECK-NEXT: tf.TensorListStack +func.func @main(%arg0: tensor<1x?x?x11xf32>) -> tensor<11x!tf_type.string> { + %cst_0 = "tf.Const"() <{value = dense<[3, 1, 2, 0]> : tensor<4xi32>}> : () -> tensor<4xi32> + %cst_10 = "tf.Const"() <{value = dense<0> : tensor}> : () -> tensor + %cst_11 = "tf.Const"() <{value = dense<2> : tensor}> : () -> tensor + %cst_12 = "tf.Const"() <{value = dense<1> : tensor}> : () -> tensor + %cst_13 = "tf.Const"() <{value = dense<[-1, -1, 1]> : tensor<3xi32>}> : () -> tensor<3xi32> + %cst_14 = "tf.Const"() <{value = dense<> : tensor<0xi32>}> : () -> tensor<0xi32> + %cst_15 = "tf.Const"() <{value = dense<-1> : tensor}> : () -> tensor + %cst_16 = "tf.Const"() <{value = dense<11> : tensor}> : () -> tensor + %92 = "tf.Cast"(%arg0) <{Truncate = false}> : (tensor<1x?x?x11xf32>) -> tensor<1x?x?x11xui8> + %0 = "tf.TensorListReserve"(%cst_15, %cst_16) : (tensor, tensor) -> tensor>> + %93 = "tf.Transpose"(%92, %cst_0) : (tensor<1x?x?x11xui8>, tensor<4xi32>) -> tensor<11x?x?x1xui8> + %94 = "tf.TensorListFromTensor"(%93, %cst_13) : (tensor<11x?x?x1xui8>, tensor<3xi32>) -> tensor>> + %95:5 = "tf.While"(%cst_10, %cst_16, %cst_10, %0, %94) <{body = @while_map_while_body_884030, cond = @while_map_while_cond_884020, is_stateless = true, parallel_iterations = 16 : i64, shape_invariant}> {T = [i32, i32, i32, !tf_type.variant, !tf_type.variant], _lower_using_switch_merge = true, _num_original_outputs = 5 : i64, _read_only_resource_inputs = [], device = "", output_shapes = [#tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>, #tf_type.shape<>]} : (tensor, tensor, tensor, tensor>>, tensor>>) -> (tensor, tensor, tensor, tensor>>, tensor>>) + %96 = "tf.StopGradient"(%95#3) : (tensor>>) -> tensor>> + %97 = "tf.TensorListStack"(%96, %cst_14) <{num_elements = 11 : i64}> : (tensor>>, tensor<0xi32>) -> tensor<11x!tf_type.string> + return %97 : tensor<11x!tf_type.string> +} + +// ----- + +// Test a while to map_fn conversion where constants are hoisted to resources. + +// CHECK-LABEL: hoisted_constants_map_while_cond +func.func private @hoisted_constants_map_while_cond(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor>>, %arg3: tensor) -> tensor { + %0 = "tf.Less"(%arg0, %arg3) : (tensor<*xi32>, tensor) -> tensor<*xi1> + %1 = "tf.Less"(%arg1, %arg3) : (tensor<*xi32>, tensor) -> tensor<*xi1> + %2 = "tf.LogicalAnd"(%0, %1) : (tensor<*xi1>, tensor<*xi1>) -> tensor<*xi1> + %3 = "tf.ToBool"(%2) : (tensor<*xi1>) -> tensor + return %3 : tensor +} + +// CHECK-LABEL: hoisted_constants_map_while_body +func.func private @hoisted_constants_map_while_body(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor>>, %arg3: tensor) -> (tensor<*xi32>, tensor<*xi32>, tensor>>, tensor) { + // Resource 1 is constant 1 + %c1 = "tf._TfrtGetResource"() <{container = [""], indices = [1 : i64], shared_name = [""]}> : () -> tensor + %0 = "tf.AddV2"(%arg0, %c1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + %1 = "tf.AddV2"(%arg1, %c1) : (tensor<*xi32>, tensor) -> tensor<*xi32> + + // Dummy work + %dummy = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %2 = "tf.TensorListSetItem"(%arg2, %arg1, %dummy) {resize_if_index_out_of_bounds = false} : (tensor>>, tensor<*xi32>, tensor) -> tensor>> + + return %0, %1, %2, %arg3 : tensor<*xi32>, tensor<*xi32>, tensor>>, tensor +} + +// CHECK-LABEL: @hoisted_constants_main +// CHECK: tf.TensorListReserve +// CHECK-NEXT: tf_mlrt.tf_map_fn +// CHECK-SAME: {body_fn = @"hoisted_constants_map_while_body/MapFnBody", num_tensor_list_or_flow_in = 1 : i32} +// CHECK-NEXT: tf.Const +// CHECK-NEXT: tf.TensorListStack +// CHECK-NOT: tf.While +func.func @hoisted_constants_main() -> tensor<3xf32> { + // Resource 0 is constant 0 + %c0 = "tf._TfrtGetResource"() <{container = [""], indices = [0 : i64], shared_name = [""]}> : () -> tensor + // Resource 2 is constant 3 (max iterations) + %c3 = "tf._TfrtGetResource"() <{container = [""], indices = [2 : i64], shared_name = [""]}> : () -> tensor + + %c_neg1 = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %tensor_list = "tf.TensorListReserve"(%c_neg1, %c3) : (tensor, tensor) -> tensor>> + + %0:4 = "tf.While"(%c0, %c0, %tensor_list, %c3) {body = @hoisted_constants_map_while_body, cond = @hoisted_constants_map_while_cond, is_stateless = true, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor>>, tensor) -> (tensor, tensor, tensor>>, tensor) + %cst_shape = "tf.Const"() {value = dense<[3]> : tensor<1xi32>} : () -> tensor<1xi32> + %stacked = "tf.TensorListStack"(%0#2, %cst_shape) {num_elements = 3 : i64} : (tensor>>, tensor<1xi32>) -> tensor<3xf32> + return %stacked : tensor<3xf32> +} + +// This function defines the resources. +func.func private @_tfrt_resource_init() { + %c0 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + "tf._TfrtSetResource"(%c0) {index = 0 : i64} : (tensor) -> () + + %c1 = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + "tf._TfrtSetResource"(%c1) {index = 1 : i64} : (tensor) -> () + + %c3 = "tf.Const"() {value = dense<3> : tensor} : () -> tensor + "tf._TfrtSetResource"(%c3) {index = 2 : i64} : (tensor) -> () + + return +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/optimize.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/optimize.mlir new file mode 100644 index 0000000000000000000000000000000000000000..7f3a99fce85874ab7ed6b145f52e725cbcbeb619 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/optimize.mlir @@ -0,0 +1,49 @@ +// RUN: tf-tfrt-opt -optimize-tf-for-tfrt -split-input-file -verify-diagnostics %s | FileCheck %s + +// CHECK-LABEL: @fold_device_index +func.func @fold_device_index() -> tensor { + // CHECK-NOT: tf.DeviceIndex + // CHECK: tf.Const + // CHECK-SAME: value = dense<1> : tensor + %0 = "tf.DeviceIndex"() {device = "/device:CPU:0", device_names = ["GPU", "CPU"]} : () -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @not_fold_device_index +func.func @not_fold_device_index() -> tensor { + // CHECK-NOT: tf.Const + // CHECK: tf.DeviceIndex + %0 = "tf.DeviceIndex"() {device = "", device_names = ["CPU", "GPU"]} : () -> tensor + func.return %0 : tensor +} + +// ----- + +// CHECK-LABEL: @eliminate_multinomial +func.func @eliminate_multinomial(%0: tensor<*xf32>, %1: tensor<*xi32>) -> (tensor<*xi64>, tensor<*xi64>) { + // CHECK-NEXT: tf.Multinomial + // CHECK-NEXT: return + %2 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 0 : i64, seed2 = 0 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + %3 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 0 : i64, seed2 = 0 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + func.return %2, %3 : tensor<*xi64>, tensor<*xi64> +} + +// ----- + +// CHECK-LABEL: @not_eliminate_multinomial +func.func @not_eliminate_multinomial(%0: tensor<*xf32>, %1: tensor<*xi32>) -> (tensor<*xi64>, tensor<*xi64>) { + // CHECK-NEXT: tf.Multinomial + // CHECK-SAME: seed = 0 + // CHECK-NEXT: tf.Multinomial + // CHECK-SAME: seed = 1 + // CHECK-NEXT: tf.Multinomial + // CHECK-SAME: seed = 0 + // CHECK-NEXT: return + %2 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 0 : i64, seed2 = 0 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + %3 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 1 : i64, seed2 = 1 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + %4 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 0 : i64, seed2 = 0 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + %5 = "tf.Multinomial"(%0, %1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", seed = 0 : i64, seed2 = 0 : i64} : (tensor<*xf32>, tensor<*xi32>) -> tensor<*xi64> + func.return %2, %3 : tensor<*xi64>, tensor<*xi64> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..da43ae3412be88918e0d0154f123148c7268f9c6 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs.mlir @@ -0,0 +1,20 @@ +// RUN: tf-tfrt-opt %s -pack-inputs="slices=1,0,24,2,24,24" | FileCheck %s + +// Tests basic contiguous slice packing: +// - Input 0 is preserved (not in slice list). +// - Inputs 1 and 2 are packed into a single contiguous i8 buffer with explicit slices. + +module { + // CHECK-LABEL: func.func @main + // CHECK-SAME: (%arg0: tensor<10x10xf32>, %arg1: tensor<48xi8>) -> (tensor<10x10xf32>, tensor<3x2xf32>, tensor<3x2xf32>) + func.func @main(%arg0: tensor<10x10xf32>, %arg1: tensor<3x2xf32>, %arg2: tensor<3x2xf32>) -> (tensor<10x10xf32>, tensor<3x2xf32>, tensor<3x2xf32>) attributes {ifrt.function} { + // CHECK: %[[SLICE0:.*]] = stablehlo.slice %arg1 [0:24] : (tensor<48xi8>) -> tensor<24xi8> + // CHECK: %[[RESHAPE0:.*]] = stablehlo.reshape %[[SLICE0]] : (tensor<24xi8>) -> tensor<3x2x4xi8> + // CHECK: %[[BITCAST0:.*]] = stablehlo.bitcast_convert %[[RESHAPE0]] : (tensor<3x2x4xi8>) -> tensor<3x2xf32> + // CHECK: %[[SLICE1:.*]] = stablehlo.slice %arg1 [24:48] : (tensor<48xi8>) -> tensor<24xi8> + // CHECK: %[[RESHAPE1:.*]] = stablehlo.reshape %[[SLICE1]] : (tensor<24xi8>) -> tensor<3x2x4xi8> + // CHECK: %[[BITCAST1:.*]] = stablehlo.bitcast_convert %[[RESHAPE1]] : (tensor<3x2x4xi8>) -> tensor<3x2xf32> + // CHECK: return %arg0, %[[BITCAST0]], %[[BITCAST1]] : tensor<10x10xf32>, tensor<3x2xf32>, tensor<3x2xf32> + return %arg0, %arg1, %arg2 : tensor<10x10xf32>, tensor<3x2xf32>, tensor<3x2xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs_exclude.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs_exclude.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8ec7daa8ea830728f048d1f3b3dac303e3f270b9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/pack_inputs_exclude.mlir @@ -0,0 +1,21 @@ +// RUN: tf-tfrt-opt %s -pack-inputs="slices=1,0,24,2,28,16" | FileCheck %s + +// Tests advanced slice packing with gaps/padding and unmerged variables: +// - Input 0 is preserved. +// - Inputs 1 and 2 are packed into a single i8 buffer with a 4-byte gap (start 0 and start 28). +// - Inputs 3 and 4 are skipped and remain unchanged. + +module { + // CHECK-LABEL: func.func @main + // CHECK-SAME: (%arg0: tensor<10x10xf32>, %arg1: tensor<5x2xf32>, %arg2: tensor<6x1xf32>, %arg3: tensor<44xi8>) -> (tensor<10x10xf32>, tensor<2x3xf32>, tensor<4x1xf32>, tensor<5x2xf32>, tensor<6x1xf32>) + func.func @main(%arg0: tensor<10x10xf32>, %arg1: tensor<2x3xf32>, %arg2: tensor<4x1xf32>, %arg3: tensor<5x2xf32>, %arg4: tensor<6x1xf32>) -> (tensor<10x10xf32>, tensor<2x3xf32>, tensor<4x1xf32>, tensor<5x2xf32>, tensor<6x1xf32>) attributes {ifrt.function} { + // CHECK: %[[SLICE0:.*]] = stablehlo.slice %arg3 [0:24] : (tensor<44xi8>) -> tensor<24xi8> + // CHECK: %[[RESHAPE0:.*]] = stablehlo.reshape %[[SLICE0]] : (tensor<24xi8>) -> tensor<2x3x4xi8> + // CHECK: %[[BITCAST0:.*]] = stablehlo.bitcast_convert %[[RESHAPE0]] : (tensor<2x3x4xi8>) -> tensor<2x3xf32> + // CHECK: %[[SLICE1:.*]] = stablehlo.slice %arg3 [28:44] : (tensor<44xi8>) -> tensor<16xi8> + // CHECK: %[[RESHAPE1:.*]] = stablehlo.reshape %[[SLICE1]] : (tensor<16xi8>) -> tensor<4x1x4xi8> + // CHECK: %[[BITCAST1:.*]] = stablehlo.bitcast_convert %[[RESHAPE1]] : (tensor<4x1x4xi8>) -> tensor<4x1xf32> + // CHECK: return %arg0, %[[BITCAST0]], %[[BITCAST1]], %arg1, %arg2 : tensor<10x10xf32>, tensor<2x3xf32>, tensor<4x1xf32>, tensor<5x2xf32>, tensor<6x1xf32> + return %arg0, %arg1, %arg2, %arg3, %arg4 : tensor<10x10xf32>, tensor<2x3xf32>, tensor<4x1xf32>, tensor<5x2xf32>, tensor<6x1xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4f147d8e718357e28ab3e9291ee3950fa3c5a054 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op.mlir @@ -0,0 +1,109 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-reconfig-batch-op="tfrt-min-num-batch-threads=2 tfrt-min-max-enqueued-batches=3 tfrt-batch-padding-policy=PAD_UP" %s | FileCheck %s --dump-input=always + +// ----- + +// The num_batch_threads is lowered bound to 2 from the original attribute of 1 + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 10 : i64 + // CHECK-SAME: num_batch_threads = 2 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [6], batch_padding_policy = "PAD_UP", batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +// ----- + +// The num_batch_threads remains 3 (the same as the original attribute) + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 10 : i64 + // CHECK-SAME: num_batch_threads = 3 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [6], batch_padding_policy = "PAD_UP", batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 3 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +// ----- + +// The max_enqueued_batches is changed to 3 from the original attribute of 2 + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 3 : i64 + // CHECK-SAME: num_batch_threads = 2 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [6], batch_padding_policy = "PAD_UP", batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 2 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +// ----- + +// The max_enqueued_batches remains 10 (the same as the original attribute) + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 10 : i64 + // CHECK-SAME: num_batch_threads = 3 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [6], batch_padding_policy = "PAD_UP", batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 3 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_global_prioritization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_global_prioritization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e815a4a979adeca1d9d342b0bd497d01f9cbbf15 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_global_prioritization.mlir @@ -0,0 +1,67 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-reconfig-batch-op="tfrt-batch-queue-global-prioritization-num-threads=4" %s | FileCheck %s --dump-input=always + +// ----- + +// The num_batch_threads is updated to 4 from the original attribute of 2, +// a mixed_priority_policy is set along with all low priority batching params +// being copied from the high priority batching params since no low priority +// settings are provided. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [1, 2] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 2 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: enable_priority_aware_batch_scheduler = true + // CHECK-SAME: max_batch_size = 2 : i64 + // CHECK-SAME: max_enqueued_batches = 2 : i64 + // CHECK-SAME: num_batch_threads = 4 : i64 + // CHECK-SAME: num_warmup_batch_threads = {{[0-9]+}} : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [1, 2], batch_padding_policy = "PAD_UP", batch_timeout_micros = 2 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 2 : i64, max_enqueued_batches = 2 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +// ----- + +// Same as first test, but low_priority_* parameters are already provided +// so not overriden. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [1, 2] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 2 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: enable_priority_aware_batch_scheduler = true + // CHECK-SAME: low_priority_allowed_batch_sizes = [1, 10] + // CHECK-SAME: low_priority_batch_timeout_micros = 7 : i64 + // CHECK-SAME: low_priority_max_batch_size = 8 : i64 + // CHECK-SAME: low_priority_max_enqueued_batches = 9 : i64 + // CHECK-SAME: max_batch_size = 2 : i64 + // CHECK-SAME: max_enqueued_batches = 2 : i64 + // CHECK-SAME: num_batch_threads = 4 : i64 + // CHECK-SAME: num_warmup_batch_threads = {{[0-9]+}} : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [1, 2], batch_padding_policy = "PAD_UP", batch_timeout_micros = 2 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 2 : i64, max_enqueued_batches = 2 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "batch/", low_priority_batch_timeout_micros = 7 : i64, low_priority_max_batch_size = 8 : i64, low_priority_max_enqueued_batches = 9 : i64, low_priority_allowed_batch_sizes = [1, 10]} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_params.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_params.mlir new file mode 100644 index 0000000000000000000000000000000000000000..42ffb3b602e86378ab8eb15b6d960089574e49cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_params.mlir @@ -0,0 +1,50 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-reconfig-batch-op="tfrt-min-num-batch-threads=4 tfrt-min-max-enqueued-batches=4 tfrt-num-batch-threads=3 tfrt-max-batch-size=3 tfrt-batch-timeout-micros=3 tfrt-allowed-batch-sizes=3,4 tfrt-max-enqueued-batches=3 tfrt-enable-large-batch-splitting=true tfrt-mixed-priority-batching-policy=priority_merge tfrt-low-priority-max-batch-size=5 tfrt-low-priority-batch-timeout-micros=5 tfrt-low-priority-allowed-batch-sizes=5,6 tfrt-low-priority-max-enqueued-batches=5 tfrt-num-warmup-batch-threads=2" %s | FileCheck %s --dump-input=always + +// ----- + +// The num_batch_threads is updated to 3 from the original attribute of 2, +// overriding the min_num_batch_threads of 4. +// The max_batch_size is updated to 3 from the original attribute of 2. +// The batch_timeout_micros is updated to 3 from the original attribute of 2. +// The allowed_batch_sizes is updated to [3, 4] from the original attribute of +// [1, 2]. +// The max_enqueued_batches is updated to 3 from the original attribute of 2, +// overriding the min_max_enqueued_batches of 4. +// The enable_large_batch_splitting is updated to true from the original +// attribute of false. +// The mixed_priority_policy is updated to "priority_merge" from the original +// attribute of "". +// The low_priority_max_batch_size is updated to 5 from the original attribute of 2. +// The low_priority_batch_timeout_micros is updated to 5 from the original attribute of 2. +// The low_priority_allowed_batch_sizes is updated to [5, 6] from the original attribute of [1, 2]. +// The low_priority_max_enqueued_batches is updated to 5 from the original attribute of 2. +// The num_warmup_batch_threads is updated to 2 from the original attribute of 0. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [3, 4] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 3 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = true + // CHECK-SAME: low_priority_allowed_batch_sizes = [5, 6] + // CHECK-SAME: low_priority_batch_timeout_micros = 5 : i64 + // CHECK-SAME: low_priority_max_batch_size = 5 : i64 + // CHECK-SAME: low_priority_max_enqueued_batches = 5 : i64 + // CHECK-SAME: max_batch_size = 3 : i64 + // CHECK-SAME: max_enqueued_batches = 3 : i64 + // CHECK-SAME: mixed_priority_policy = "priority_merge" + // CHECK-SAME: num_batch_threads = 3 : i64 + // CHECK-SAME: num_warmup_batch_threads = 2 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [1, 2], batch_padding_policy = "PAD_UP", batch_timeout_micros = 2 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, low_priority_allowed_batch_sizes = [1, 2], low_priority_batch_timeout_micros = 2 : i64, low_priority_max_batch_size = 2 : i64, low_priority_max_enqueued_batches = 2 : i64, max_batch_size = 2 : i64, max_enqueued_batches = 2 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_skip_pass.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_skip_pass.mlir new file mode 100644 index 0000000000000000000000000000000000000000..84c54d5d29a9ce8e5015af8a714394efdbfafd7c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/reconfig_batch_op_skip_pass.mlir @@ -0,0 +1,29 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-reconfig-batch-op="tfrt-min-num-batch-threads=0 tfrt-min-max-enqueued-batches=0" %s | FileCheck %s --dump-input=always + +// ----- + +// Confirm no batch options change when the tfrt-reconfig-batch-op flag is +// empty. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> { + %2 = "tf.Identity"(%arg0) : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> { + // CHECK: "tf.BatchFunction" + // CHECK-SAME: allowed_batch_sizes = [6] + // CHECK-SAME: batch_padding_policy = "PAD_UP" + // CHECK-SAME: batch_timeout_micros = 100000 : i64 + // CHECK-SAME: batching_queue = "" + // CHECK-SAME: container = "" + // CHECK-SAME: enable_large_batch_splitting = false + // CHECK-SAME: max_batch_size = 6 : i64 + // CHECK-SAME: max_enqueued_batches = 10 : i64 + // CHECK-SAME: num_batch_threads = 1 : i64 + // CHECK-SAME: shared_name = "batch/" + %1 = "tf.BatchFunction"(%arg0) {allowed_batch_sizes = [6], batch_padding_policy = "PAD_UP", batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/remove_device_attribute.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/remove_device_attribute.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6e1f648df80adc2cd651999cd433035c1686e1da --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/remove_device_attribute.mlir @@ -0,0 +1,8 @@ +// RUN: tf-tfrt-opt -tfrt-remove-device-attribute %s | FileCheck %s + +func.func @test(%arg0: !tfrt.chain, %arg1: !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) { + %0 = corert.get_op_handler %arg0 "cpu" + // CHECK: %[[RESULT:.*]] = corert.executeop(%[[ARG_0:.*]]) "tf.MatMul"(%[[ARG_1:.*]], %[[ARG_1]]) {T = f32, transpose_a = false, transpose_b = false} : 1 + %1 = corert.executeop(%0) "tf.MatMul"(%arg1, %arg1) {T = f32, device = "cpu", transpose_a = false, transpose_b = false} : 1 + tfrt.return %arg0, %1 : !tfrt.chain, !corert.tensorhandle +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_gpu.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_gpu.mlir new file mode 100644 index 0000000000000000000000000000000000000000..84633e3c7d5d1a577ed2475933298c6e4ef04615 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_gpu.mlir @@ -0,0 +1,15 @@ +// RUN: tf-tfrt-opt -tfrt-lower-cluster-to-runtime-ops-non-tpu -split-input-file -verify-diagnostics %s | FileCheck %s + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:GPU:0"]} { + + // CHECK-LABEL: @converts_cluster + func.func @converts_cluster() { + // CHECK: "tf.XlaLaunch"() + "tf_device.cluster_func"() {_xla_compile_device_type = "GPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } + +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_tpu.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_tpu.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0e90dc442d95910a75c02e12217ef59c8baffd32 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/runtime_lowering_tpu.mlir @@ -0,0 +1,29 @@ +// RUN: tf-tfrt-opt -tfrt-lower-cluster-to-runtime-ops-tpu -split-input-file -verify-diagnostics %s | FileCheck %s + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + + // CHECK-LABEL: @converts_cluster + func.func @converts_cluster() { + // CHECK: %0:2 = "tf_device.launch"() <{{.*}}> ({ + // CHECK: %compilation_status, %program = "tf._TPUCompileMlir"() + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } + +} + +// ----- + +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @missing_num_cores_per_replica() { + // expected-error@+1 {{requires attribute 'num_cores_per_replica'}} + "tf_device.cluster_func"() {_xla_compile_device_type = "TPU", _replication_info = "cluster0", func = @empty_func, step_marker_location = "STEP_MARK_AT_TOP_LEVEL_WHILE_LOOP", topology = "", device_assignment = [], input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..3cf8ef1a5d3d2595fda876e446a885b7ae057f6a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/BUILD @@ -0,0 +1,47 @@ +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +tf_cc_test( + name = "saved_model_test", + srcs = ["saved_model_test.cc"], + data = [ + "testdata/test.mlir", + "testdata/xla_launch.mlir", + "testdata/xla_launch_xla_reduce_window.mlir", + ], + tags = ["no_oss"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tfrt:import_model", + "//tensorflow/compiler/mlir/tfrt:saved_model", + "//tensorflow/compiler/mlir/tfrt:tfrt_compile_options", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/tfrt/fallback:fallback_state", + "//tensorflow/core/tfrt/graph_executor:graph_execution_options", + "//tensorflow/core/tfrt/runtime", + "@com_google_googletest//:gtest_main", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@tf_runtime//:bef", + "@tf_runtime//:hostcontext", + "@xla//xla/tsl/platform:statusor", + ], +) + +filegroup( + name = "testdata", + srcs = glob( + ["testdata/**"], + ), + visibility = ["//visibility:public"], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/saved_model_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/saved_model_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..4f7daf8d3ce5a1c59ee407c8f41c4ad775521630 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/saved_model_test.cc @@ -0,0 +1,248 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/saved_model/saved_model.h" + +#include +#include +#include +#include +#include + +#include +#include +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tfrt/translate/import_model.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/graph_executor/graph_execution_options.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/host_context/resource_context.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +TEST(SavedModelTest, MapSignatures) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/test.mlir"); + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + std::vector inputs; + std::vector> + in_specs; + std::vector outputs; + std::vector> + out_specs; + std::vector bound_inputs; + TF_ASSERT_OK(MapFunctionSignaturesFromTFSavedModelMLIR( + module.get(), [&](const TFRTSavedModelSignatureInfo& sig_info) { + // Only check the signature of "serving_default". + if (sig_info.func_name != "serving_default") return; + + transform(sig_info.input_names, std::back_inserter(inputs), + [](llvm::StringRef x) { return x.str(); }); + in_specs.assign(sig_info.input_specs.begin(), + sig_info.input_specs.end()); + transform(sig_info.output_names, std::back_inserter(outputs), + [](llvm::StringRef x) { return x.str(); }); + out_specs.assign(sig_info.output_specs.begin(), + sig_info.output_specs.end()); + bound_inputs.assign(sig_info.bound_inputs.begin(), + sig_info.bound_inputs.end()); + })); + + ASSERT_EQ(inputs.size(), 1); + EXPECT_EQ(inputs[0], "x"); + ASSERT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs[0], "r"); + + ASSERT_EQ(in_specs.size(), 1); + ASSERT_EQ(in_specs[0].first, tensorflow::DT_INT32); + ASSERT_TRUE(in_specs[0].second.IsIdenticalTo(PartialTensorShape({1, 3}))); + + ASSERT_EQ(out_specs.size(), 1); + ASSERT_EQ(out_specs[0].first, tensorflow::DT_INT32); + ASSERT_TRUE(out_specs[0].second.IsIdenticalTo(PartialTensorShape({1, 1}))); + + ASSERT_EQ(bound_inputs.size(), 2); + + auto global_tensor = + llvm::cast(bound_inputs[0]); + auto asset = llvm::cast(bound_inputs[1]); + + EXPECT_EQ(global_tensor.getSymName(), "y"); + EXPECT_EQ(asset.getSymName(), "z"); +} + +TEST(SavedModelTest, CompileToBEF) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/test.mlir"); + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + tfrt::BefBuffer bef_buffer; + + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/1); + tfrt_stub::GraphExecutionOptions options(runtime.get()); + tfrt::ResourceContext resource_context; + tfrt_stub::ModelRuntimeContext model_context( + &options, options.compile_options.saved_model_dir, &resource_context); + TF_ASSERT_OK(ConvertTfMlirToBef(options.compile_options, module.get(), + &bef_buffer, model_context)); +} + +TEST(SavedModelTest, ConvertTfMlirToBefWithXlaFuncExport) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/" + "xla_launch.mlir"); + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + tfrt::BefBuffer bef_buffer; + + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/1); + tfrt_stub::GraphExecutionOptions options(runtime.get()); + + options.compile_options.device_target = TfrtDeviceInfraTarget::kGpu; + + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_state, + tfrt_stub::FallbackState::Create(SessionOptions(), FunctionDefLibrary())); + + tfrt::ResourceContext resource_context; + tfrt_stub::ModelRuntimeContext model_context( + &options, options.compile_options.saved_model_dir, &resource_context); + + TF_ASSERT_OK(ConvertTfMlirToBef(options.compile_options, module.get(), + &bef_buffer, model_context, + fallback_state.get())); + + // The module contains an XLA function, as well as a while body and a while + // condition within the XLA function. + EXPECT_EQ(fallback_state->process_function_library_runtime() + .GetFunctionLibraryDefinition() + ->num_functions(), + 3); +} + +TEST(SavedModelTest, ConvertTfMlirToBefExportingXlaReduceWindow) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/" + "xla_launch_xla_reduce_window.mlir"); + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + tfrt::BefBuffer bef_buffer; + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/1); + tfrt_stub::GraphExecutionOptions options(runtime.get()); + options.compile_options.device_target = TfrtDeviceInfraTarget::kGpu; + + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_state, + tfrt_stub::FallbackState::Create(SessionOptions(), FunctionDefLibrary())); + + tfrt::ResourceContext resource_context; + tfrt_stub::ModelRuntimeContext model_context( + &options, options.compile_options.saved_model_dir, &resource_context); + + TF_ASSERT_OK(ConvertTfMlirToBef(options.compile_options, module.get(), + &bef_buffer, model_context, + fallback_state.get())); + + // The module contains an XLA function, as well as a sum_reducer function + // referenced by an XlaReduceWindow op. + EXPECT_EQ(fallback_state->process_function_library_runtime() + .GetFunctionLibraryDefinition() + ->num_functions(), + 2); +} + +TEST(SavedModelTest, AddXlaFunctionsOutputFunctionNames) { + std::string saved_model_mlir_path = tensorflow::GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/" + "xla_launch_xla_reduce_window.mlir"); + + mlir::DialectRegistry registry; + mlir::RegisterAllTensorFlowDialects(registry); + mlir::MLIRContext context(registry); + auto module = + mlir::parseSourceFile(saved_model_mlir_path, &context); + ASSERT_TRUE(module); + + tfrt::BefBuffer bef_buffer; + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/1); + tfrt_stub::GraphExecutionOptions options(runtime.get()); + options.compile_options.device_target = TfrtDeviceInfraTarget::kGpu; + + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_state, + tfrt_stub::FallbackState::Create(SessionOptions(), FunctionDefLibrary())); + + tfrt::ResourceContext resource_context; + tfrt_stub::ModelRuntimeContext model_context( + &options, options.compile_options.saved_model_dir, &resource_context); + + std::vector function_names; + TF_ASSERT_OK(ConvertTfMlirToBef(options.compile_options, module.get(), + &bef_buffer, model_context, + fallback_state.get(), &function_names)); + EXPECT_THAT(function_names, ::testing::SizeIs(1)); +} + +// TODO(b/162442824): Add a SavedModel test that covers the error pass. + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/test.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/test.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ceac0e19691ec9c8ad79c2c5f30d3ca75b2dc5b9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/test.mlir @@ -0,0 +1,28 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 398 : i32}, tf_saved_model.semantics} { + "tf_saved_model.global_tensor"() {is_mutable, sym_name = "y", type = tensor<3x1xi32>, value = dense<[[1], [2], [3]]> : tensor<3x1xi32>} : () -> () + "tf_saved_model.asset"() {sym_name = "z", filename = "file"} : () -> () + func.func @serving_default( + %arg0: tensor<1x3xi32> {tf_saved_model.index_path = ["x"]}, + %arg1: tensor>> {tf_saved_model.bound_input = @y}, + %arg2: tensor {tf_saved_model.bound_input = @z} + ) -> (tensor<1x1xi32> {tf_saved_model.index_path = ["r"]}) + attributes { + tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "result:0"}, + tf_saved_model.exported_names = ["serving_default"] + } + { + %0 = "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor>>) -> tensor<3x1xi32> + %1 = "tf.MatMul"(%arg0, %0) {device = "", transpose_a = false, transpose_b = false} : (tensor<1x3xi32>, tensor<3x1xi32>) -> tensor<1x1xi32> + func.return %1 : tensor<1x1xi32> + } + func.func @predict( + ) -> (tensor<0x!tf_type.string> {tf_saved_model.index_path = ["r"]}) + attributes { + tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "result:0"}, + tf_saved_model.exported_names = ["predict"] + } + { + %0 = "tf.Const"() {dtype = !tf_type.string, value = dense<[]> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + func.return %0 : tensor<0x!tf_type.string> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6a2ce6eef50459419be13f0919d84ccaa110c2cd --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch.mlir @@ -0,0 +1,25 @@ + +func.func @while_cond(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func @while_body(%arg0: tensor) -> tensor { + %1 = "tf.AddV2"(%arg0, %arg0) {} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func private @xla_func_0(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %2 = "tf.Const"() {value = dense<0> : tensor} : () -> tensor + %3 = "tf.While"(%2) { cond = @while_cond, body = @while_body, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + func.return %1 : tensor<1x3xf32> +} + +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_0, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch_xla_reduce_window.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch_xla_reduce_window.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a46138bda0a4f80b6e79f64c78c7ad32ad562117 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/saved_model/testdata/xla_launch_xla_reduce_window.mlir @@ -0,0 +1,22 @@ + +func.func private @sum_reducer(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + %0 = "tf.AddV2"(%arg0, %arg1) {device = ""} : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +func.func @xla_func_0(%arg0: tensor<7xf32>, %arg1: tensor) -> tensor<10xf32> { + %cst = "tf.Const"() {value = dense<0> : tensor<1x2xi32>} : () -> tensor<1x2xi32> + %cst_0 = "tf.Const"() {value = dense<1> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_1 = "tf.Const"() {value = dense<2> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_2 = "tf.Const"() {value = dense<3> : tensor<1xi32>} : () -> tensor<1xi32> + %cst_3 = "tf.Const"() {value = dense<4> : tensor<1xi32>} : () -> tensor<1xi32> + %0 = "tf.XlaReduceWindow"(%arg0, %arg1, %cst_0, %cst_1, %cst_2, %cst_3, %cst) {computation = @sum_reducer} : (tensor<7xf32>, tensor, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1xi32>, tensor<1x2xi32>) -> tensor<10xf32> + func.return %0 : tensor<10xf32> +} + +func.func @main(%arg0: tensor<7xf32>) -> tensor<10xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor + %2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_0, operandSegmentSizes = array} : (tensor<7xf32>, tensor) -> tensor<10xf32> + func.return %2 : tensor<10xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/sink_in_invariant_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/sink_in_invariant_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a1ea5b68f35d02a1b8ebf4f193b84c509b8a0ceb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/sink_in_invariant_ops.mlir @@ -0,0 +1,488 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-sink-in-invariant-ops %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +module attributes {tf_saved_model.semantics} { + +// Test sinks in var handle op to batch function. + +// CHECK-LABEL: func private @batched_function +// CHECK: arg1 +func.func private @batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>) -> tensor<1x3xf32> + %1 = "tf.AddV2"(%arg0, %0) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: tf.VarHandleOp + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + + // CHECK: "tf.BatchFunction"(%arg0, %0) + // CHECK: operandSegmentSizes = array + %1 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %1 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in const op to batch function. + +// CHECK-LABEL: func private @batched_function +// CHECK: arg1 +func.func private @batched_function(%arg0: tensor, %arg1: tensor) -> tensor + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK: tf.Const + %1 = "tf.AddV2"(%arg0, %arg1) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: [[handle:%.*]] = "tf.Const"() + %0 = "tf.Const"() {device = "/CPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: "tf.BatchFunction"(%arg0, [[handle]]) + // CHECK-SAME: operandSegmentSizes = array + %1 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in HashTableV2 op to batch function. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1xi32>, %arg1: tensor<*x!tf_type.resource>) -> tensor<1xi32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + %default = "tf.Const"() {device = "/CPU:0", value = dense<0> : tensor<1xi32>} : () -> tensor<1xi32> + // CHECK: tf.HashTableV2 + %0 = "tf.LookupTableFindV2"(%arg1, %arg0, %default) {device = "/device:CPU:0"} : (tensor<*x!tf_type.resource>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32> + func.return %0 : tensor<1xi32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1xi32> {tf_saved_model.index_path = ["input"]}) -> (tensor<1xi32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + %0 = "tf.HashTableV2"() {device = "/device:CPU:0", container = "", shared_name = "variable", key_dtype = i32, value_dtype = i32} : () -> tensor<*x!tf_type.resource> + %1 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1xi32>, tensor<*x!tf_type.resource>) -> tensor<1xi32> + func.return %1 : tensor<1xi32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sink in multiple invariant ops. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor>>, %arg1: tensor>>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK-DAG: [[handle1:%.*]] = "tf.VarHandleOp"() <{{{.*}}, shared_name = "variable1"}> + // CHECK-DAG: [[handle2:%.*]] = "tf.VarHandleOp"() <{{{.*}}, shared_name = "variable2"}> + // CHECK: "tf.ReadVariableOp"([[handle1]]) + // CHECK: "tf.ReadVariableOp"([[handle2]]) + %0 = "tf.ReadVariableOp"(%arg0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %1 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.AddV2"(%0, %1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %3 = "tf.Identity"(%2) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %3 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: tf.VarHandleOp + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable1"} : () -> tensor>> + %1 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable2"} : () -> tensor>> + // CHECK: "tf.BatchFunction"(%0, %1) + %2 = "tf.BatchFunction"(%0, %1) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor>>, tensor>>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in var handle op that used by control flow ops. + +// CHECK-LABEL: func private @some_func +func.func private @some_func( + %arg: tensor>>) -> tensor { + // CHECK: tf.VarHandleOp + // CHECK: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + + func.return %0 : tensor +} + +// CHECK-LABEL: func private @some_other_func +func.func private @some_other_func( + %arg: tensor>>) -> tensor { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + + func.return %0 : tensor +} + +// CHECK-LABEL: func @sink_in_stateful_call +func.func @sink_in_stateful_call(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_stateful_call"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"([[handle]]) + %x = "tf.StatefulPartitionedCall"(%handle) {device = "/CPU:0", config = "", config_proto = "", executor_type = "", f = @some_func} : (tensor>>) -> (tensor) + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +// CHECK-LABEL: func @sink_in_if +func.func @sink_in_if(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_if"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: [[cond:%.*]] = "tf.Const"() + %cond = "tf.Const"() {device = "/CPU:0", value = dense : tensor} : () -> tensor + // CHECK: "tf.If"([[cond]], [[handle]]) + %x = "tf.If"(%cond, %handle) {then_branch = @some_other_func, else_branch = @some_other_func, is_stateless = false} : (tensor, tensor>>) -> tensor + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test doesn't sink in to the callee that invoked by multiple callers. + +// CHECK: func private @some_func([[arg0:.+]]: tensor>>) +func.func private @some_func(%arg0: tensor>>) -> tensor { + // CHECK-NOT: tf.VarHandleOp + // CHECK: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg0) {device = "cpu"} : (tensor>>) -> tensor + + func.return %0 : tensor +} + +// CHECK-LABEL: func @sink_in_stateful_call +func.func @sink_in_stateful_call(%arg0: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_stateful_call"]} { + // CHECK: tf.VarHandleOp + %0 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%0) + %1 = "tf.StatefulPartitionedCall"(%0) {device = "/CPU:0", config = "", config_proto = "", executor_type = "", f = @some_func} : (tensor>>) -> (tensor) + %2 = "tf.AddV2"(%arg0, %1) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// Test VarHandleOp getting sinked when it is used by the called function and returned by the called function. + +// CHECK: func private @func_use_and_return_varhandle([[arg0:.+]]: tensor>>) +func.func private @func_use_and_return_varhandle(%arg0: tensor>>) -> (tensor, tensor>>) { + // CHECK: tf.VarHandleOp + // CHECK-NEXT: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg0) {device = "cpu"} : (tensor>>) -> tensor + + func.return %0, %arg0 : tensor, tensor>> +} + +// CHECK-LABEL: func @sink_in_stateful_call_varhandle_return +func.func @sink_in_stateful_call_varhandle_return(%arg0: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_stateful_call_varhandle_return"]} { + // CHECK: tf.VarHandleOp + %0 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: "tf.StatefulPartitionedCall"(%0) + %1:2 = "tf.StatefulPartitionedCall"(%0) {device = "/CPU:0", config = "", config_proto = "", executor_type = "", f = @func_use_and_return_varhandle} : (tensor>>) -> (tensor, tensor>>) + %2 = "tf.AddV2"(%arg0, %1#0) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func @sink_in_if +func.func @sink_in_if(%arg0: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_if"]} { + // CHECK: tf.VarHandleOp + %0 = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + %cst = "tf.Const"() {device = "/CPU:0", value = dense : tensor} : () -> tensor + // CHECK: "tf.If"(%cst, %0) + %1 = "tf.If"(%cst, %0) {then_branch = @some_func, else_branch = @some_func, is_stateless = false} : (tensor, tensor>>) -> tensor + %2 = "tf.AddV2"(%arg0, %1) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test doesn't sink in var handle op + read variable op. Consider implement when we see it from production. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK-NOT: tf.VarHandleOp + // CHECK-NOT: tf.ReadVariableOp + %1 = "tf.AddV2"(%arg0, %arg1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: "tf.BatchFunction"(%arg0, %1) + %2 = "tf.BatchFunction"(%arg0, %1) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in var handle op if it's used by one callee, and also by read only ops in the current funciton. + +// CHECK-LABEL: func private @batched_function +// CHECK: arg1 +func.func private @batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor>>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK: tf.VarHandleOp + // CHECK: tf.ReadVariableOp + %1 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.AddV2"(%arg0, %1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %3 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + // CHECK: "tf.ReadVariableOp"([[handle]]) + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: "tf.BatchFunction"(%arg0, [[handle]]) + // CHECK-SAME: operandSegmentSizes = array + %2 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in var handle op crossing nested tf.BatchFunction, while the var handle op is only copied at the target. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor>>) -> tensor<1x3xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK: tf.VarHandleOp + // CHECK: tf.ReadVariableOp + %1 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + %2 = "tf.AddV2"(%arg0, %1) {device = "/device:CPU:0"} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %3 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %2 : tensor<1x3xf32> +} + +// CHECK-LABEL: func private @nested_batched_function +func.func private @nested_batched_function(%arg0: tensor<1x3xf32>, %arg1: tensor>>) -> tensor<*xf32> + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK-NEXT: tf.BatchFunction + %0 = "tf.BatchFunction"(%arg0, %arg1) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %0 : tensor<*xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32> {tf_saved_model.index_path = ["input"]}) -> (tensor<*xf32> {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + // CHECK: "tf.ReadVariableOp"([[handle]]) + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: "tf.BatchFunction"(%arg0, [[handle]]) + // CHECK-SAME: operandSegmentSizes = array + %2 = "tf.BatchFunction"(%arg0, %0) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @nested_batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor<1x3xf32>, tensor>>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks crossing nested tf.If, while the sinkable ops are only copied at the target. + +// CHECK-LABEL: func private @then_func +func.func private @then_func( + %cond: tensor, + %arg: tensor>>) -> tensor { + // CHECK: tf.VarHandleOp + // CHECK: tf.ReadVariableOp + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + + func.return %0 : tensor +} + +// CHECK-LABEL: func private @else_func +func.func private @else_func( + %cond: tensor, + %arg: tensor>>) -> tensor { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func private @nested_else_func +func.func private @nested_else_func( + %cond: tensor, + %arg: tensor>>) -> tensor { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func private @nested_then_func( +// CHECK-SAME: [[arg0:%.*]]: tensor, +// CHECK-SAME: [[arg1:%.*]]: tensor>>) +func.func private @nested_then_func( + %cond: tensor, + %arg: tensor>>) -> tensor { + // CHECK-NOT: tf.VarHandleOp + // CHECK: [[const:%.*]] = "tf.Const" + // CHECK: "tf.If"([[const]], [[arg0]], [[arg1]]) + %0 = "tf.If"(%cond, %cond, %arg) {then_branch = @then_func, else_branch = @else_func, is_stateless = false} : (tensor, tensor, tensor>>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func @nested_sink_in_if +func.func @nested_sink_in_if(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_if"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: [[cond:%.*]] = "tf.Const"() + %cond = "tf.Const"() {device = "/CPU:0", value = dense : tensor} : () -> tensor + // CHECK: "tf.If"([[cond]], [[cond]], [[handle]]) + %x = "tf.If"(%cond, %cond, %handle) {then_branch = @nested_then_func, else_branch = @nested_else_func, is_stateless = false} : (tensor, tensor, tensor>>) -> tensor + %r = "tf.AddV2"(%arg, %x) {device = "/CPU:0"} : (tensor, tensor) -> tensor + func.return %r : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks crossing nested tf.While and BatchFunction, while the sinkable ops are only copied at the target. + +// CHECK-LABEL: func private @batched_function +func.func private @batched_function(%arg0: tensor>>) -> tensor + attributes {tf._input_shapes = [#tf_type.shape<1x3>, #tf_type.shape<*>], tf.signature.is_stateful} { + // CHECK: tf.VarHandleOp + // CHECK-NEXT: tf.ReadVariableOp + %1 = "tf.ReadVariableOp"(%arg0) {device = "/device:CPU:0"} : (tensor>>) -> tensor + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func private @while_cond_func +func.func private @while_cond_func( + %arg0: tensor, + %arg1: tensor, + %arg: tensor>>) -> tensor { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[handle]]) + %0 = "tf.ReadVariableOp"(%arg) {device = "cpu"} : (tensor>>) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func private @while_body_func +func.func private @while_body_func( + %arg0: tensor, + %arg1: tensor, + %arg2: tensor>>) -> (tensor, tensor, tensor>>) { + // CHECK: "tf.BatchFunction"(%arg2) + %0 = "tf.BatchFunction"(%arg2) {allowed_batch_sizes = [6], batch_timeout_micros = 100000 : i64, batching_queue = "", container = "", device = "/device:CPU:0", enable_large_batch_splitting = false, f = @batched_function, max_batch_size = 6 : i64, max_enqueued_batches = 10 : i64, num_batch_threads = 1 : i64, operandSegmentSizes = array, shared_name = "batch/"} : (tensor>>) -> tensor + func.return %0, %arg0, %arg2 : tensor, tensor, tensor>> +} + +// CHECK-LABEL: func @nested_sink_in_while_and_batch_functions +func.func @nested_sink_in_while_and_batch_functions(%arg: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["test_sink_in_while_and_batch_functions"]} { + // CHECK: [[handle:%.*]] = "tf.VarHandleOp"() + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: [[cond:%.*]] = "tf.Const"() + %cond = "tf.Const"() {device = "/CPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: "tf.While"([[cond]], [[cond]], [[handle]]) + %x:3 = "tf.While"(%cond, %cond, %handle) {body = @while_body_func, cond = @while_cond_func, is_stateless = false, parallel_iterations = 10 : i64, shape_invariant} : (tensor, tensor, tensor>>) -> (tensor, tensor, tensor>>) + func.return %x#0 : tensor +} + +} + +// ----- + +module attributes {tf_saved_model.semantics} { + +// Test sinks in var handle op to WhileOp with unranked loop signature. +// This verifies that the pass does not corrupt the return type of the loop body +// when the resource is also returned (pass-through). + +// CHECK-LABEL: func private @while_cond +func.func private @while_cond(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>) -> tensor { + %cst = "tf.Const"() {value = dense : tensor} : () -> tensor + func.return %cst : tensor +} + +// CHECK-LABEL: func private @while_body +// CHECK: (%arg0: tensor, %arg1: tensor<*x!tf_type.resource>) -> (tensor, tensor<*x!tf_type.resource>) +func.func private @while_body(%arg0: tensor, %arg1: tensor<*x!tf_type.resource>) -> (tensor, tensor<*x!tf_type.resource>) { + // CHECK: [[sunk_handle:%.*]] = "tf.VarHandleOp"() + // CHECK: "tf.ReadVariableOp"([[sunk_handle]]) + %0 = "tf.ReadVariableOp"(%arg1) {device = "cpu"} : (tensor<*x!tf_type.resource>) -> tensor + %1 = "tf.AddV2"(%arg0, %0) {device = "cpu"} : (tensor, tensor) -> tensor + // CHECK: return {{.*}}, %arg1 + func.return %1, %arg1 : tensor, tensor<*x!tf_type.resource> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor {tf_saved_model.index_path = ["input"]}) -> (tensor {tf_saved_model.index_path = ["r"]}) + attributes {tf_saved_model.exported_names = ["main"]} { + %handle = "tf.VarHandleOp"() {container = "", shared_name = "x"} : () -> tensor>> + // CHECK: "tf.While" + %x:2 = "tf.While"(%arg0, %handle) {body = @while_body, cond = @while_cond, is_stateless = false, parallel_iterations = 10 : i64} : (tensor, tensor>>) -> (tensor, tensor>>) + func.return %x#0 : tensor +} + +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..925e99162ea3ab7d415446697b4242cc978e43ad --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/BUILD @@ -0,0 +1,38 @@ +load("//tensorflow:tensorflow.bzl", "if_oss") +load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + licenses = ["notice"], +) + +glob_lit_tests( + name = "all_tests", + data = [":test_utilities"], + # Custom driver is unsupported in OSS. Fails if one is provided. + # copybara:uncomment driver = "//tensorflow/compiler/mlir:run_lit.sh", + # copybara:comment_begin(JitRt/Auto fusion depreciated) + exclude = [ + "auto-fusion.mlir", + "tf_to_corert_pipeline_cpurt.mlir", + "outline-cpurt-cluster.mlir", + ], + # copybara:comment_end + features = if_oss(["--path=org_tensorflow/tensorflow/compiler/mlir/tfrt"]), + size_override = { + "fallback.mlir": "medium", + }, + test_file_exts = ["mlir"], +) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + data = [ + "//tensorflow/compiler/mlir/tfrt:tf-tfrt-opt", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/attributes.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/attributes.mlir new file mode 100644 index 0000000000000000000000000000000000000000..9dd11dd1948ee6391ed7e0efa5cc4ea955f4e199 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/attributes.mlir @@ -0,0 +1,83 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail + +// _output_shapes and f.* attributes are removed during tf-to-tfrt lowering. +// CHECK-LABEL: func @remove_unused_attr +func.func @remove_unused_attr() { + // CHECK: %out_op_chain = tfrt_fallback_async.executeop.seq(%arg0) key(0) cost({{.*}}) device("/device:CPU:0") "tf.SomeOp2"() + "tf.SomeOp2"() {device = "/device:CPU:0", _output_shapes = ["tfshape$"], f.Tin = [f32], f._read_only_resource_inputs = []} : () -> () + func.return +} + +// CHECK-LABEL: func @basic +func.func @basic( + %arg0: tensor<3x1xf32>, + %arg1: tensor>>) -> (tensor<3x3xf32>) { + %1 = "tf.ReadVariableOp"(%arg1) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + + // CHECK: {{%.*}} = tfrt_fallback_async.executeop {{.*}} device("/device:CPU:0") "tf.MatMul" + // CHECK-SAME: {T = f32, transpose_a = false, transpose_b = false} + %2 = "tf.MatMul"(%arg0, %1) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %2 : tensor<3x3xf32> +} + +// CHECK-LABEL: func @string_type +func.func @string_type(%arg: tensor<1x2x!tf_type.string>) -> tensor { + %multiples = "tf.Const"() { device = "/device:CPU:0", value = dense<[7,3]> : tensor<2xi32> } : () -> tensor<2xi32> + // CHECK: T = !corert.string + %output = "tf.Tile"(%arg, %multiples) { device = "/device:CPU:0" } : (tensor<1x2x!tf_type.string>, tensor<2xi32>) -> tensor + func.return %output : tensor +} + +// CHECK-LABEL: func @shape +func.func @shape() { + %size = "tf.Const"() {value = dense<5> : tensor} : () -> tensor + // CHECK: tf.TensorArrayV3 + // CHECK-SAME: element_shape = #corert.shape<*> + %ta:2 = "tf.TensorArrayV3"(%size) {device = "/device:CPU:0", dtype = f32, element_shape = #tf_type.shape<*>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : (tensor) -> (tensor, tensor) + func.return +} + +// CHECK-LABEL: func @resource +func.func @resource() { + // CHECK: tf.SomeOp + // CHECK-SAME: dtype = !corert.resource + "tf.SomeOp"() {device = "/device:CPU:0", dtype = !tf_type.resource} : () -> () + func.return +} + +// CHECK-LABEL: func @variant +func.func @variant(%arg: tensor) { + // CHECK: tf.ZerosLike + // CHECK-SAME: T = !corert.variant + %0 = "tf.ZerosLike"(%arg) {device = "/device:CPU:0", T = !tf_type.variant} : (tensor) -> tensor + func.return +} + +// Checks that TF quantized attrs are lowered to the corert types +// CHECK-LABEL: func @quantized_types +func.func @quantized_types(%arg0: tensor>>, + %arg1: tensor>>, + %arg2: tensor>>, + %arg3: tensor>>, + %arg4: tensor>>) { + // CHECK: tf.ReadVariableOp + // CHECK-SAME: dtype = !corert.quint8 + %0 = "tf.ReadVariableOp"(%arg0) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = !tf_type.quint8} : (tensor>>) -> tensor<1x3x!tf_type.quint8> + + // CHECK: tf.ReadVariableOp + // CHECK-SAME: dtype = !corert.quint16 + %1 = "tf.ReadVariableOp"(%arg1) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = !tf_type.quint16} : (tensor>>) -> tensor<1x3x!tf_type.quint16> + + // CHECK: tf.ReadVariableOp + // CHECK-SAME: dtype = !corert.qint8 + %2 = "tf.ReadVariableOp"(%arg2) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = !tf_type.qint8} : (tensor>>) -> tensor<1x3x!tf_type.qint8> + + // CHECK: tf.ReadVariableOp + // CHECK-SAME: dtype = !corert.qint16 + %3 = "tf.ReadVariableOp"(%arg3) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = !tf_type.qint16} : (tensor>>) -> tensor<1x3x!tf_type.qint16> + + // CHECK: tf.ReadVariableOp + // CHECK-SAME: dtype = !corert.qint32 + %4 = "tf.ReadVariableOp"(%arg4) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = !tf_type.qint32} : (tensor>>) -> tensor<1x3x!tf_type.qint32> + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/basic.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/basic.mlir new file mode 100644 index 0000000000000000000000000000000000000000..19d4f2a024c5c81ff98dccbeb1aedd74423554cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/basic.mlir @@ -0,0 +1,56 @@ +// RUN: tf-tfrt-opt -pass-pipeline='builtin.module(func.func(tf-tensor-device-copy),tfrt-lower-tf-savedmodel{hoist-invariant-ops=true},tf-to-tfrt{tfrt-cost-threshold=1024 tfrt-merge-inter-dependent-streams=true})' %s | FileCheck %s --dump-input-filter=all + +// CHECK-NOT: tf_saved_model.semantics +// CHECK: tfrt.cost_threshold = 1024 +// CHECK-SAME: tfrt.merge_inter_dependent_streams = true +module attributes {tf_saved_model.semantics} { + +// CHECK-NOT: "tf_saved_model.session_initializer" +"tf_saved_model.session_initializer"() { initializers = [@func_init] } : () -> () + +// CHECK-LABEL: _tfrt_resource_init +// CHECK: tf.VarHandleOp +// CHECK: tf.ReadVariableOp +// CHECK: tfrt_fallback_async.set_resource +// CHECK-SAME: {device = "/device:CPU:0", index = 0 : i64} + + +// CHECK-LABEL: func @init +// CHECK-SAME: {tfrt.cost_threshold = 1 : i64} +func.func @func_init() attributes {tf_saved_model.exported_names = ["init"]} { + func.return +} + +// CHECK-LABEL: func @basic +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain +// CHECK-SAME: [[arg0:%.*]]: !tfrt_fallback.tf_tensor, +// CHECK-SAME: [[arg1:%.*]]: !tfrt_fallback.tf_tensor) +// CHECK-SAME: -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @func_basic( + %arg0: tensor<3x1xf32> {tf_saved_model.index_path = [0]}, + %arg1: tensor>> {tf_saved_model.index_path = [1]}) + -> (tensor<3x3xf32> {tf_saved_model.index_path = []}) + attributes {tf_saved_model.exported_names = ["basic"]} { + %handle = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "x"} : () -> tensor>> + %0 = "tf.ReadVariableOp"(%handle) {_output_shapes = ["tfshape$dim { size: 3 }"], device = "/device:CPU:0", dtype = f32} : (tensor>>) -> tensor<3xf32> + %1 = "tf.ReadVariableOp"(%arg1) {_output_shapes = ["tfshape$dim { size: 1 } dim { size: 3 }"], device = "/device:CPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + + // CHECK-NEXT: [[ready_ch:%.*]] = tfrt.new.chain + // CHECK-NEXT: [[ch:%.*]], [[result:%.*]] = tfrt_fallback_async.get_resource [[ready_ch]] {device = "/device:CPU:0", indices = [0]} : (!tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + // CHECK-NEXT: [[ch1:%.*]], [[var:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) {{.*}} "tf.ReadVariableOp"([[arg1]]) + // CHECK-NEXT: [[r0:%.*]] = tfrt_fallback_async.executeop {{.*}} "tf.MatMul"([[arg0]], [[var]]) + %2 = "tf.MatMul"(%arg0, %1) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + // CHECK-NEXT: [[r1:%.*]] = tfrt_fallback_async.executeop {{.*}} "tf.BiasAdd"([[r0]], [[result]]) + %3 = "tf.BiasAdd"(%2, %0) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], data_format = "NHWC", device = "/device:CPU:0"} : (tensor<3x3xf32>, tensor<3xf32>) -> tensor<3x3xf32> + // CHECK-NEXT: [[r2:%.*]] = tfrt_fallback_async.executeop {{.*}} "tf.Tanh"([[r1]]) {T = f32} + %4 = "tf.Tanh"(%3) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:CPU:0"} : (tensor<3x3xf32>) -> tensor<3x3xf32> + // CHECK-NOT: tf.Identity + %5 = "tf.Identity"(%4) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:CPU:0"} : (tensor<3x3xf32>) -> tensor<3x3xf32> + // CHECK-NOT: tf.IdentityN + %6:2 = "tf.IdentityN"(%5, %4) {T = [f32, f32], _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }", "tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:CPU:0"} : (tensor<3x3xf32>, tensor<3x3xf32>) -> (tensor<3x3xf32>, tensor<3x3xf32>) + // CHECK-NEXT: [[out_ch:%.*]] = tfrt.merge.chains [[ch]], [[ch1]] : !tfrt.chain, !tfrt.chain + // CHECK-NEXT: tfrt.return [[out_ch]], [[r2]] : !tfrt.chain, !tfrt_fallback.tf_tensor + func.return %6#0 : tensor<3x3xf32> +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate.mlir new file mode 100644 index 0000000000000000000000000000000000000000..942e190c1106feec81dc16b5a9b6b0808702d56a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate.mlir @@ -0,0 +1,46 @@ +// RUN: tf-tfrt-opt -tfrt-deduplicate-functions-invoked-by-batch-function %s | FileCheck %s + +// This test verifies the function `compute_1` will be removed to deduplicate +// the functions invoked by BatchFunction with the same shared_name and the +// function `compute_2` will not be removed as the shared_name is different. + +// CHECK-LABEL: func private @batch_0 +// CHECK: f = @compute_0 +func.func private @batch_0(%arg0: tensor) -> tensor<*xi32> { + %0:2= "tf.BatchFunction"(%arg0, %arg0) {_xla_inferred_shapes = [#tf_type.shape<*>, #tf_type.shape<*>], allowed_batch_sizes = [64, 128, 256], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", enable_large_batch_splitting = true, f = @compute_0, max_batch_size = 256 : i64, max_enqueued_batches = 10000 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "computation"} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>) + func.return %0#0 : tensor<*xi32> +} + +// CHECK: func private @compute_0 +func.func private @compute_0(%arg0: tensor {tf._user_specified_name = "0"}, %arg1: tensor) -> (tensor, tensor) { + func.return %arg0, %arg1 : tensor, tensor +} + +// Batch function in batch_1 uses the same shared_name as the one in batch_0, +// so compute_1 is deduped, and compute_0 will be used here. +// CHECK-LABEL: func private @batch_1 +// CHECK: f = @compute_0 +// CHECK-NOT: f = @compute_1 +func.func private @batch_1(%arg0: tensor) -> tensor<*xi32> { + %0:2 = "tf.BatchFunction"(%arg0, %arg0) {_xla_inferred_shapes = [#tf_type.shape<*>, #tf_type.shape<*>], allowed_batch_sizes = [64, 128, 256], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", enable_large_batch_splitting = true, f = @compute_1, max_batch_size = 256 : i64, max_enqueued_batches = 10000 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "computation"} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>) + func.return %0#0 : tensor<*xi32> +} + +// CHECK-NOT: func private @compute_1 +func.func private @compute_1(%arg0: tensor {tf._user_specified_name = "0"}, %arg1: tensor) -> (tensor, tensor) { + func.return %arg0, %arg1 : tensor, tensor +} + +// Batch function in batch_2 uses a different shared_name from the one in +// batch_0, so it should be kept. +// CHECK-LABEL: func private @batch_2 +// CHECK: f = @compute_2 +func.func private @batch_2(%arg0: tensor) -> tensor<*xi32> { + %0:2 = "tf.BatchFunction"(%arg0, %arg0) {_xla_inferred_shapes = [#tf_type.shape<*>, #tf_type.shape<*>], allowed_batch_sizes = [64, 128, 256], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", enable_large_batch_splitting = true, f = @compute_2, max_batch_size = 256 : i64, max_enqueued_batches = 10000 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "computation_unique_name"} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>) + func.return %0#0 : tensor<*xi32> +} + +// CHECK: func private @compute_2 +func.func private @compute_2(%arg0: tensor {tf._user_specified_name = "0"}, %arg1: tensor) -> (tensor, tensor) { + func.return %arg0, %arg1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate_failed.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate_failed.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e16d1cf43bf99e5d862e789bfbc5b49b391b40c1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/batch_function_deduplicate_failed.mlir @@ -0,0 +1,24 @@ +// RUN: not tf-tfrt-opt -tfrt-deduplicate-functions-invoked-by-batch-function %s 2>&1 | FileCheck %s + +// This test verifies the error when two functions are different but invoked by +// the batch functions with same shared_name. + +func.func private @batch_0(%arg0: tensor) -> tensor<*xi32> { + %0:2 = "tf.BatchFunction"(%arg0, %arg0) {_xla_inferred_shapes = [#tf_type.shape<*>, #tf_type.shape<*>], allowed_batch_sizes = [64, 128, 256], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", enable_large_batch_splitting = true, f = @compute_0, max_batch_size = 256 : i64, max_enqueued_batches = 10000 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "computation"} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>) + func.return %0#0 : tensor<*xi32> +} + +func.func private @compute_0(%arg0: tensor {tf._user_specified_name = "0"}, %arg1: tensor) -> (tensor, tensor) { + func.return %arg0, %arg1 : tensor, tensor +} + +func.func private @batch_3(%arg0: tensor) -> tensor<*xi32> { + %0:2 = "tf.BatchFunction"(%arg0, %arg0) {_xla_inferred_shapes = [#tf_type.shape<*>, #tf_type.shape<*>], allowed_batch_sizes = [64, 128, 256], batch_timeout_micros = 5000 : i64, batching_queue = "", container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", enable_large_batch_splitting = true, f = @compute_3, max_batch_size = 256 : i64, max_enqueued_batches = 10000 : i64, num_batch_threads = 2 : i64, operandSegmentSizes = array, shared_name = "computation"} : (tensor, tensor) -> (tensor<*xi32>, tensor<*xi32>) + func.return %0#0 : tensor<*xi32> +} + +// compute_3 has different argument types from compute_1, thus error is reported. +// CHECK: error: func_ops for BatchFunctionOp with the same shared name are different +func.func private @compute_3(%arg0: tensor {tf._user_specified_name = "0"}, %arg1: tensor) -> (tensor, tensor) { + func.return %arg0, %arg1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/const_tensor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/const_tensor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e9e01921aa407894b0ed157aaa18f67a1226da16 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/const_tensor.mlir @@ -0,0 +1,30 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @string_tensor +func.func @string_tensor() -> (tensor<0x!tf_type.string>, tensor<7x!tf_type.string>) { + // CHECK: {shape = [0], value = []} + %0 = "tf.Const"() {value = dense<[]> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + // CHECK: {shape = [7], value = ["has_login_page_feature", "num_terms_inside_postform", "num_terms_outside_postform", "num_terms_outside_postform_without_bp", "query_params_contains_url", "title_with_login_phase", "url_contains_login_terms"]} + %1 = "tf.Const"() {value = dense<["has_login_page_feature", "num_terms_inside_postform", "num_terms_outside_postform", "num_terms_outside_postform_without_bp", "query_params_contains_url", "title_with_login_phase", "url_contains_login_terms"]> : tensor<7x!tf_type.string>} : () -> tensor<7x!tf_type.string> + func.return %0, %1 : tensor<0x!tf_type.string>, tensor<7x!tf_type.string> +} + +// Convert tf.Const to tfrt_fallback_async.const_dense_tensor only on cpu device +// CHECK-LABEL: func @dense_tensor +func.func @dense_tensor() -> tensor<4xui64> { + // CHECK: tfrt_fallback_async.const_dense_tensor dense<[1, 2, 3, 4]> : tensor<4xui64> + %0 = "tf.Const"() {value = dense<[1, 2, 3, 4]> : tensor<4xui64>} : () -> tensor<4xui64> + // CHECK: tfrt_fallback_async.const_dense_tensor dense<1.000000e+00> : tensor<1xbf16> + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<[1.0]> : tensor<1xbf16>} : () -> tensor<4xbf16> + // CHECK: corert.executeop({{.*}}) "tf.Const"() {dtype = ui64, value = dense<[1, 2, 3, 4]> : tensor<4xui64>} : 1 + %2 = "tf.Const"() {device = "/device:GPU:0", value = dense<[1, 2, 3, 4]> : tensor<4xui64>} : () -> tensor<4xui64> + func.return %0 : tensor<4xui64> +} + +// CHECK-LABEL: func @tensor_proto +func.func @tensor_proto() -> tensor { + // tfrt_fallback_async.const_tensor_proto accepts a serialized tensor proto. + // CHECK: tfrt_fallback_async.const_tensor_proto "\08\0C\12\00\22\01@" + %0 = "tf.Const"() {value = #tf_type : tensor} : () -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/control_flow.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/control_flow.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3eafe259cc7dd8807e8bf5d0428243144b5df75e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/control_flow.mlir @@ -0,0 +1,201 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt="enable-while-parallel-iterations=true" %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @cond_false(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @cond_false(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<-1> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @cond_true(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @cond_true(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @cond(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor, %arg2: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @cond(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: [[cond:%.*]] = tfrt_fallback_async.predicate + // CHECK: [[cond_res:%.*]]:2 = tfrt.cond [[cond]] + // CHECK-SAME: @cond_true @cond_false(%arg0, %arg2) : (!tfrt.chain, !tfrt_fallback.tf_tensor) + %2 = "tf.If"(%arg0, %arg1) {else_branch = @cond_false, then_branch = @cond_true, is_stateless = true} : (tensor, tensor) -> tensor + // CHECK: [[out_ch:%.*]] = tfrt.merge.chains [[cond_res]]#0, %arg0 : !tfrt.chain, !tfrt.chain + // CHECK: tfrt.return [[out_ch]], [[cond_res]]#1 : !tfrt.chain, !tfrt_fallback.tf_tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func @cond_stateful(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @cond_stateful(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + // CHECK: [[cond_res:%.*]]:2 = tfrt.cond + // CHECK-SAME: @cond_true @cond_false(%arg0, %arg1) : (!tfrt.chain, !tfrt_fallback.tf_tensor) + %2 = "tf.If"(%1, %arg0) {else_branch = @cond_false, then_branch = @cond_true, is_stateless = false} : (tensor, tensor) -> tensor + // Note: returns %out_op_chain. + // CHECK: tfrt.return [[cond_res]]#0, [[cond_res]]#1 : !tfrt.chain, !tfrt_fallback.tf_tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func @while_cond_lt9 +// CHECK-SAME: ({{%.+}}: !tfrt.chain, {{%.+}}: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @while_cond_lt9(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_body_add2 +// CHECK-SAME: ({{%.+}}: !tfrt.chain, {{%.+}}: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @while_body_add2(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @while_test +// CHECK-SAME: ([[ARG0:%.+]]: !tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +func.func @while_test() -> (tensor) { + // CHECK: [[CONST:%.*]] = tfrt_fallback_async.const_dense_tensor dense<0> : tensor + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + // CHECK: [[pred_res:%.*]]:2 = tfrt.call @"while_cond_lt9/tfrt_predicate"([[ARG0]], [[CONST]]) : (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, i1) + // CHECK: [[while_res:%.]]:2 = tfrt.while [[pred_res]]#1 @"while_body_add2/tfrt_body_1"([[pred_res]]#0, [[CONST]]) + // CHECK-SAME: (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + %1 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + // CHECK: [[out_chain:%.*]] = tfrt.merge.chains [[while_res]]#0, [[ARG0]] + // CHECK: tfrt.return [[out_chain]], [[while_res]]#1 : !tfrt.chain, !tfrt_fallback.tf_tensor + func.return %1 : tensor +} +// CHECK: func @"while_body_add2/tfrt_body_1"([[ch:%.*]]: !tfrt.chain, [[arg:%.*]]: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor, i1) +// CHECK: [[body_res:%.*]]:2 = tfrt.call @while_body_add2([[ch]], [[arg]]) : (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +// CHECK: [[pred_res:%.*]]:2 = tfrt.call @"while_cond_lt9/tfrt_predicate"([[body_res]]#0, [[body_res]]#1) : (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, i1) +// CHECK: tfrt.return [[pred_res]]#0, [[body_res]]#1, [[pred_res]]#1 : !tfrt.chain, !tfrt_fallback.tf_tensor, i1 + +// CHECK: func @"while_cond_lt9/tfrt_predicate"([[ch:%.*]]: !tfrt.chain, [[arg:%.*]]: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, i1) +// CHECK: [[cond_res:%.*]]:2 = tfrt.call @while_cond_lt9([[ch]], [[arg]]) : (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +// CHECK: [[bool_cond:%.*]] = tfrt_fallback_async.predicate [[cond_res]]#1 +// CHECK: tfrt.return [[cond_res]]#0, [[bool_cond]] : !tfrt.chain, i1 + +// CHECK-LABEL: func @multi_while_test +func.func @multi_while_test() -> (tensor, tensor) { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + // CHECK: [[pred_0:%.*]]:2 = tfrt.call @"while_cond_lt9/tfrt_predicate" + // CHECK: tfrt.while [[pred_0]]#1 @"while_body_add2/tfrt_body_10" + // CHECK-SAME: parallel_iterations(10) + // CHECK: [[pred_1:%.*]]:2 = tfrt.call @"while_cond_lt9/tfrt_predicate" + // CHECK: tfrt.while [[pred_1]]#1 @"while_body_add2/tfrt_body_1" + // CHECK-SAME: parallel_iterations(1) + %2 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 10} : (tensor) -> (tensor) + %3 = "tf.While"(%1) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + func.return %2, %3 : tensor, tensor +} + +func.func @side_effect_while_cond_lt9(%arg: tensor>>) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.ReadVariableOp"(%arg) {device = "/device:CPU:0", dtype = i32} : (tensor>>) -> tensor + %2 = "tf.Less"(%1, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +func.func @side_effect_while_body_add2(%arg: tensor>>) -> (tensor>>) { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.ReadVariableOp"(%arg) {device = "/device:CPU:0", dtype = i32} : (tensor>>) -> tensor + %2 = "tf.Add"(%1, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + "tf.AssignVariableOp"(%arg, %2) {device = "/device:CPU:0"} : (tensor>>, tensor) -> () + func.return %arg : tensor>> +} + +// CHECK-LABEL: func @side_effect_while_test +func.func @side_effect_while_test() -> (tensor) { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "c", shared_name = "v"} : () -> tensor>> + // CHECK: [[while_res:%.]]:2 = tfrt.while {{%.*}} @"side_effect_while_body_add2/tfrt_body_1" + // CHECK: [[out_ch:%.*]], [[res:%.*]] = tfrt_fallback_async.executeop.seq([[while_res]]#0) {{.*}} "tf.ReadVariableOp" + %1 = "tf.While"(%0) { cond = @side_effect_while_cond_lt9, body = @side_effect_while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor>>) -> (tensor>>) + %2 = "tf.ReadVariableOp"(%1) {device = "/device:CPU:0", dtype = i32} : (tensor>>) -> tensor + func.return %2 : tensor +} + +func.func @tensor_array_while_cond(%index: tensor, %size: tensor, %flow_0: tensor, %flow_1: tensor, %handle_0: tensor<2x!tf_type.resource>>, %handle_1: tensor<2x!tf_type.resource>>) -> (tensor) { + %0 = "tf.Less"(%index, %size) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @tensor_array_while_body(%index: tensor, %size: tensor, %flow_0: tensor, %flow_1: tensor, %handle_0: tensor<2x!tf_type.resource>>, %handle_1: tensor<2x!tf_type.resource>>) -> (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) { + %cst = "tf.Const"() {value = dense<1.1> : tensor<100x512xf32>} : () -> tensor<100x512xf32> + %one = "tf.Const"() {value = dense<1> : tensor} : () -> tensor + %x = "tf.TensorArrayReadV3"(%handle_0, %index, %flow_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + %y = "tf.MatMul"(%x, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor<100x512xf32>) -> (tensor) + %flow_1_out = "tf.TensorArrayWriteV3"(%handle_1, %index, %y, %flow_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + %next_index = "tf.AddV2"(%index, %one) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %next_index, %size, %flow_0, %flow_1_out, %handle_0, %handle_1 : tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>> +} + +// CHECK-LABEL: func @tensor_array_while_test +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain +func.func @tensor_array_while_test(%indices: tensor, %input_0: tensor, %input_1: tensor) -> (tensor, tensor) { + %index = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> (tensor) + %size = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> (tensor) + %handle_0, %flow_0 = "tf.TensorArrayV3"(%size) {clear_after_read = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = f32, dynamic_size = false, element_shape = #tf_type.shape, identical_element_shapes = true, tensor_array_name = "processed_embeddings/bidirectional_rnn/bw/bw/dynamic_rnn/input_0"} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + %handle_1, %flow_1 = "tf.TensorArrayV3"(%size) {clear_after_read = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = f32, dynamic_size = false, element_shape = #tf_type.shape, identical_element_shapes = true, tensor_array_name = "processed_embeddings/bidirectional_rnn/bw/bw/dynamic_rnn/output_0"} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + %flow_01 = "tf.TensorArrayScatterV3"(%handle_0, %indices, %input_0, %flow_0) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + // CHECK: [[pred_0:%.*]]:2 = tfrt.call @"tensor_array_while_cond/tfrt_predicate"([[in_chain]] + // CHECK: [[while_res_0:%.*]]:7 = tfrt.while {{%.*}} @"tensor_array_while_body/tfrt_body_10"([[pred_0]]#0 + // CHECK-SAME: parallel_iterations(10) + %res_0:6 = "tf.While"(%index, %size, %flow_01, %flow_1, %handle_0, %handle_1) {body = @tensor_array_while_body, cond = @tensor_array_while_cond, device = "", is_stateless = false, parallel_iterations = 10 : i64} : (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) -> (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) + %output_0 = "tf.TensorArrayGatherV3"(%handle_1, %indices, %res_0#3) {device = "/job:localhost/replica:0/task:0/device:CPU:0", element_shape = #tf_type.shape} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + + %handle_2, %flow_2 = "tf.TensorArrayV3"(%size) {clear_after_read = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = f32, dynamic_size = false, element_shape = #tf_type.shape, identical_element_shapes = true, tensor_array_name = "processed_embeddings/bidirectional_rnn/bw/bw/dynamic_rnn/input_0"} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + %handle_3, %flow_3 = "tf.TensorArrayV3"(%size) {clear_after_read = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = f32, dynamic_size = false, element_shape = #tf_type.shape, identical_element_shapes = true, tensor_array_name = "processed_embeddings/bidirectional_rnn/bw/bw/dynamic_rnn/output_0"} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + %flow_21 = "tf.TensorArrayScatterV3"(%handle_2, %indices, %input_1, %flow_2) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor, tensor) -> tensor + // CHECK: [[pred_1:%.*]]:2 = tfrt.call @"tensor_array_while_cond/tfrt_predicate"([[in_chain]] + // CHECK: [[while_res_1:%.*]]:7 = tfrt.while {{%.*}} @"tensor_array_while_body/tfrt_body_10"([[pred_1]]#0 + // CHECK-SAME: parallel_iterations(10) + %res_1:6 = "tf.While"(%index, %size, %flow_21, %flow_3, %handle_2, %handle_3) {body = @tensor_array_while_body, cond = @tensor_array_while_cond, device = "", is_stateless = false, parallel_iterations = 10 : i64} : (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) -> (tensor, tensor, tensor, tensor, tensor<2x!tf_type.resource>>, tensor<2x!tf_type.resource>>) + %output_1 = "tf.TensorArrayGatherV3"(%handle_3, %indices, %res_1#3) {device = "/job:localhost/replica:0/task:0/device:CPU:0", element_shape = #tf_type.shape} : (tensor<2x!tf_type.resource>>, tensor, tensor) -> tensor + func.return %output_0, %output_1 : tensor, tensor +} + +// CHECK: func @"tensor_array_while_body/tfrt_body_10" + +func.func @callee(%arg0: tensor) -> (tensor) { + func.return %arg0: tensor +} + +// CHECK-LABEL: func @call_test +// CHECK-SAME: ([[chain:%.*]]: !tfrt.chain, +func.func @call_test(%arg0: tensor) -> (tensor, tensor, tensor) { + %0 = "tf.Add"(%arg0, %arg0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + // CHECK: [[results_0:%.*]]:2 = tfrt.call @callee([[chain]] + // CHECK-SAME: (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + %1 = "tf.StatefulPartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> (tensor) + // CHECK-NEXT: [[results_1:%.*]]:2 = tfrt.call @callee([[chain]] + // CHECK-SAME: (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + %2 = "tf.PartitionedCall"(%0) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor) -> (tensor) + // CHECK-NEXT: [[results_2:%.*]]:2 = tfrt.call @callee([[chain]] + // CHECK-SAME: (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + %3 = "tf.LegacyCall"(%0) {f = @callee} : (tensor) -> (tensor) + // CHECK: [[results_0]]#1, [[results_1]]#1, [[results_2]]#1 + func.return %1, %2, %3 : tensor, tensor, tensor +} + +func.func @branch0(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.Add" (%arg0, %arg1) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @branch1(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.Add" (%arg0, %arg1) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %1 = "tf.Add" (%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @case_test +// CHECK-SAME: ([[chain:%.*]]: !tfrt.chain, [[tf_idx:%.*]]: !tfrt_fallback.tf_tensor, [[branch_arg0:%.*]]: !tfrt_fallback.tf_tensor, [[branch_arg1:%.*]]: !tfrt_fallback.tf_tensor) +func.func @case_test(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[th_idx:%.*]] = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle [[tf_idx]] + // CHECK-NEXT: [[idx:%.*]] = corert.tensorhandle_to_int32 [[th_idx]] + // CHECK-NEXT: [[out:%.*]] = tfrt.case [[idx]] [@branch0, @branch1]([[chain]], [[branch_arg0]], [[branch_arg1]]) + %0 = "tf.Case"(%arg0, %arg1, %arg2) {_lower_using_switch_merge = true, branches = [@branch0, @branch1], is_stateless = true} : (tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/decompose_resource_op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/decompose_resource_op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..aececb2f5c08d93fdf0885b30bee1ccca90fba89 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/decompose_resource_op.mlir @@ -0,0 +1,18 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline=decompose-resource-ops=true %s | FileCheck %s --dump-input=fail + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 293 : i32}} { + +// CHECK-LABEL: func @gather +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain +// CHECK-SAME: [[arg0:%.*]]: !tfrt_fallback.tf_tensor, [[arg1:%.*]]: !tfrt_fallback.tf_tensor) +// CHECK: [[const:%.*]] = tfrt_fallback_async.const_dense_tensor +// CHECK-NEXT: [[out_chain:%.*]], [[value:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key(0) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"({{.*}}) +// CHECK-NEXT: [[res:%.*]] = tfrt_fallback_async.executeop key(1) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.GatherV2"([[value]], {{.*}}, [[const]]) +// CHECK-NEXT: tfrt.return [[out_chain]], [[res]] : !tfrt.chain, !tfrt_fallback.tf_tensor +func.func @gather(%indices: tensor, + %resource: tensor<*x!tf_type.resource>) -> tensor<*xi32> { + %0 = "tf.ResourceGather"(%resource, %indices) {batch_dims = 0 : i64, device = "/device:CPU:0", validate_indices = true}: (tensor<*x!tf_type.resource>, tensor) -> (tensor<*xi32>) + func.return %0 : tensor<*xi32> +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/derived_attrs.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/derived_attrs.mlir new file mode 100644 index 0000000000000000000000000000000000000000..608a75a2df93a8a260dd28e6d335ab1249e28b55 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/derived_attrs.mlir @@ -0,0 +1,21 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @derived_attrs +func.func @derived_attrs( + %serialized: tensor, + %names: tensor<0x!tf_type.string>, + %sparse_keys: tensor<0x!tf_type.string>, + %dense_keys: tensor<1x!tf_type.string>, + %ragged_keys: tensor<0x!tf_type.string>, + %dense_default: tensor<0xi64>) -> tensor { + + %dense_value = + "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %dense_keys, %ragged_keys, %dense_default) + // CHECK: Tdense = [i64] + // CHECK-SAME: dense_shapes = [#corert.shape<>] + { device = "/device:CPU:0", num_sparse = 0 : i64, dense_shapes = [#tf_type.shape<>], resultSegmentSizes = array} + : (tensor, tensor<0x!tf_type.string>, tensor<0x!tf_type.string>, tensor<1x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0xi64>) + -> tensor + + func.return %dense_value : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/device_conversion.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/device_conversion.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ed523e03a5ba28e4be384f241e57582db457ef55 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/device_conversion.mlir @@ -0,0 +1,11 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @device_test +func.func @device_test( + %arg0: tensor<3x1xf32> {tf_saved_model.index_path = [0]}, + %arg1: tensor<1x3xf32> {tf_saved_model.index_path = [0]}) + -> (tensor<3x3xf32> {tf_saved_model.index_path = []}) { + // CHECK: {{%.*}} = corert.get_op_handler %arg0 "/device:GPU:0" + %2 = "tf.MatMul"(%arg0, %arg1) {T = f32, _output_shapes = ["tfshape$dim { size: 3 } dim { size: 3 }"], device = "/device:GPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %2 : tensor<3x3xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/errors.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/errors.mlir new file mode 100644 index 0000000000000000000000000000000000000000..6698a36c6d88c245b9f779b3dbfd6de36e96a3de --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/errors.mlir @@ -0,0 +1,7 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s -split-input-file -verify-diagnostics + +// expected-error @+1 {{failed to legalize operation 'func.func' that was explicitly marked illegal}} +func.func @test_identity_wrong_type(%arg0: tensor<4x2x!tf_type.string>) -> tensor<4x2x!tf_type.stringref> { + %0 = "tf.SomeOp"(%arg0) : (tensor<4x2x!tf_type.string>) -> tensor<4x2x!tf_type.stringref> + func.return %0 : tensor<4x2x!tf_type.stringref> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f6a1ffff0274518033e0949123b9288516bf6b68 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback.mlir @@ -0,0 +1,115 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail --dump-input-filter=all +// RUN: tf-tfrt-opt -pass-pipeline='builtin.module(tf-to-tfrt{target-tpurt=true tpu-use-core-selector=false})' %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +// CHECK-LABEL: func @_tfrt_fallback_init +// CHECK-SAME: {{.*}} !tfrt.chain +// CHECK: tfrt_fallback_async.createop(%arg0) key(0) device("/device:CPU:0") "tf.ParseExampleV2"() +// CHECK-SAME: Tdense = [f32, f32], dense_shapes = [#corert.shape<>, #corert.shape<>] +// CHECK-SAME: num_sparse = 2 : i64, ragged_split_types = [], ragged_value_types = [] +// CHECK-SAME: sparse_types = [!corert.string, i64]} +// CHECK-SAME: num_args(7) + +// CHECK: tfrt_fallback_async.createop(%0) key(1) device("/device:CPU:0") "tf.ReadVariableOp"() {dtype = f32} num_args(1) + +// CHECK: tfrt_fallback_async.createop(%1) key(2) device("/device:CPU:0") "tf.MatMul"() {T = f32, transpose_a = false, transpose_b = false} num_args(2) + +// CHECK-LABEL: func @main +// CHECK-SAME: {{.*}} !tfrt.chain +// CHECK-SAME: [[serialized:%.*]]: !tfrt_fallback.tf_tensor +func.func @main(%serialized: tensor<32x!tf_type.string>) -> (tensor) attributes {tf.entry_function = {inputs = "input0", outputs = "ParseExample/ParseExampleV2"}} { + %dense_default_0 = "tf.Const"() {device = "/device:CPU:0", dtype = f32, value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + %dense_default_1 = "tf.Const"() {device = "/device:CPU:0", dtype = f32, value = dense<[]> : tensor<0xf32>} : () -> tensor<0xf32> + %dense_keys = "tf.Const"() {device = "/device:CPU:0", dtype = !tf_type.string, value = dense<""> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + %names = "tf.Const"() {device = "/device:CPU:0", dtype = !tf_type.string, value = dense<""> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %ragged_keys = "tf.Const"() {device = "/device:CPU:0", dtype = !tf_type.string, value = dense<""> : tensor<0x!tf_type.string>} : () -> tensor<0x!tf_type.string> + %sparse_keys = "tf.Const"() {device = "/device:CPU:0", dtype = !tf_type.string, value = dense<""> : tensor<2x!tf_type.string>} : () -> tensor<2x!tf_type.string> + + // CHECK: [[outputs:%.*]]:8 = tfrt_fallback_async.executeop key(0) cost({{.*}}) device("/device:CPU:0") "tf.ParseExampleV2" + // CHECK-SAME: ([[serialized]] + // CHECK-NOT: device + // CHECK-SAME: Tdense = [f32, f32] + // CHECK-SAME: dense_shapes = [#corert.shape<>, #corert.shape<>] + // CHECK-SAME: num_sparse = 2 : i64 + // CHECK-SAME: ragged_split_types = [] + // CHECK-SAME: ragged_value_types = [] + // CHECK-SAME: sparse_types = [!corert.string, i64] + %outputs:8 = "tf.ParseExampleV2"(%serialized, %names, %sparse_keys, %dense_keys, %ragged_keys, %dense_default_0, %dense_default_1) + { + Tdense = [f32, f32], dense_shapes = [#tf_type.shape<>, #tf_type.shape<>], + device = "/device:CPU:0", num_sparse = 2 : i64, ragged_split_types = [], ragged_value_types = [], + resultSegmentSizes = array, + sparse_types = [!tf_type.string, !tf_type.string] + } : (tensor<32x!tf_type.string>, tensor<0x!tf_type.string>, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>, tensor<0x!tf_type.string>, tensor<0xf32>, tensor<0xf32>) + -> (tensor, tensor, tensor, tensor, tensor<2xi64>, tensor<2xi64>, tensor<32xf32>, tensor<32xf32>) + + // CHECK: tfrt.return {{.*}}, [[outputs]]#0 + func.return %outputs#0 : tensor +} + +// CHECK-LABEL: func @no_native +func.func @no_native(%arg0: tensor<3x1xf32>, %arg1: tensor>>) -> tensor<3x3xf32> { + // CHECK-NOT: corert.executeop + // CHECK: tfrt_fallback_async.executeop.seq({{.*}}) key(1) cost({{.*}}) device("/device:CPU:0") "tf.ReadVariableOp" + // CHECK: tfrt_fallback_async.executeop key(2) cost({{.*}}) device("/device:CPU:0") "tf.MatMul" + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + %1 = "tf.MatMul"(%arg0, %0) {T = f32, device = "/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %1 : tensor<3x3xf32> +} + +// CHECK-LABEL: func @gpu_device +func.func @gpu_device(%arg0: tensor<3x1xf32>, %arg1: tensor>>) -> tensor<3x3xf32> { + // CHECK: {{%.*}} = corert.get_op_handler %arg0 "/device:GPU:0" + // CHECK: {{.*}} = corert.executeop.seq({{.*}}) "tf.ReadVariableOp"({{.*}}) {dtype = f32} : 1 + // CHECK: {{.*}} = corert.executeop({{.*}}) "tf.MatMul"({{.*}}) {T = f32, transpose_a = false, transpose_b = false} : 1 + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:GPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + %1 = "tf.MatMul"(%arg0, %0) {T = f32, device = "/device:GPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %1 : tensor<3x3xf32> +} + +// CHECK-LABEL: func @tpu_device +func.func @tpu_device(%arg0: tensor<3x1xf32>, %arg1: tensor>>) -> tensor<3x3xf32> { + // CHECK-NOT: corert.executeop + // CHECK: tfrt_fallback_async.executeop.seq({{.*}}) key({{.*}}) cost({{.*}}) device("/device:TPU:0") "tf.ReadVariableOp" + // CHECK: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/device:CPU:0") "tf.MatMul" + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:TPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + %1 = "tf.MatMul"(%arg0, %0) {T = f32, device = "/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %1 : tensor<3x3xf32> +} + +// CHECK-LABEL: func @tfrt_set_resource +// CHECK-SAME: ([[in_ch:%.*]]: !tfrt.chain, +func.func @tfrt_set_resource(%arg0: tensor<3x1xf32>, %arg1: tensor>>) { + // CHECK: [[ch0:%.*]] = tfrt_fallback_async.set_resource [[in_ch]], {{.*}} {device = "/device:CPU:0", index = 0 : i64} + // CHECK: [[ch1:%.*]], [[result:%.*]] = tfrt_fallback_async.executeop.seq([[ch0]]) key({{.*}}) cost({{.*}}) device("/device:CPU:0") "tf.ReadVariableOp" + // CHECK: [[ch2:%.*]] = tfrt_fallback_async.set_resource [[ch1]], [[result]] {device = "/device:CPU:0", index = 1 : i64} + + "tf._TfrtSetResource"(%arg0) {device = "/device:CPU:0", index = 0} : (tensor<3x1xf32>) -> () + %0 = "tf.ReadVariableOp"(%arg1) {device = "/device:CPU:0", dtype = f32} : (tensor>>) -> tensor<1x3xf32> + "tf._TfrtSetResource"(%0) {device = "/device:CPU:0", index = 1} : (tensor<1x3xf32>) -> () + func.return +} + +// CHECK-LABEL: func @tfrt_get_resource +func.func @tfrt_get_resource() -> tensor<3x3xf32> { + // CHECK: [[ready_ch:%.*]] = tfrt.new.chain + // CHECK: [[ch3:%.*]], [[results:%.*]]:2 = tfrt_fallback_async.get_resource [[ready_ch]] {device = "/device:CPU:0", indices = [0, 1]} + // CHECK: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/device:CPU:0") "tf.MatMul"([[results]]#0, [[results]]#1) + %a, %b = "tf._TfrtGetResource"() {device = "/device:CPU:0", indices = [0, 1], shared_name = ["", ""], container = ["", ""]} : () -> (tensor<3x1xf32>, tensor<1x3xf32>) + %1 = "tf.MatMul"(%a, %b) {T = f32, device = "/device:CPU:0", transpose_a = false, transpose_b = false} : (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> + func.return %1 : tensor<3x3xf32> +} + +// CHECK-LABEL: func @tensor_array +func.func @tensor_array() -> (tensor<1x1x512xf32>) { + %value = "tf.Const"() {device = "/device:CPU:0", value = dense<0.1> : tensor<1x512xf32>} : () -> (tensor<1x512xf32>) + %index = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> (tensor) + %size = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> (tensor) + %indices = "tf.Const"() {device = "/device:CPU:0", value = dense<[0]> : tensor<1xi32>} : () -> (tensor<1xi32>) + // CHECK: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.TensorArrayV3" + // CHECK: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.TensorArrayWriteV3" + // CHECK: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.TensorArrayGatherV3" + %handle, %flow = "tf.TensorArrayV3"(%size) {clear_after_read = true, device = "/job:localhost/replica:0/task:0/device:CPU:0", dtype = f32, dynamic_size = false, element_shape = #tf_type.shape, identical_element_shapes = true, tensor_array_name = "output"} : (tensor) -> (tensor<2x!tf_type.resource>>, tensor) + %flow_1 = "tf.TensorArrayWriteV3"(%handle, %index, %value, %flow) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor<2x!tf_type.resource>>, tensor, tensor<1x512xf32>, tensor) -> tensor + %result = "tf.TensorArrayGatherV3"(%handle, %indices, %flow_1) {device = "/job:localhost/replica:0/task:0/device:CPU:0", element_shape = #tf_type.shape<1x512>} : (tensor<2x!tf_type.resource>>, tensor<1xi32>, tensor) -> tensor<1x1x512xf32> + func.return %result : tensor<1x1x512xf32> +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_canonicalization.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_canonicalization.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ab9fc9ff0c5520203aa19257c95b3475a1505c0d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_canonicalization.mlir @@ -0,0 +1,70 @@ +// RUN: tf-tfrt-opt %s -pass-pipeline='builtin.module(func.func(canonicalize))' | FileCheck %s + +// CHECK-LABEL: func @test_const_tensor_canonicalization_single_denst_tensor_operand +func.func @test_const_tensor_canonicalization_single_denst_tensor_operand() -> !tfrt_fallback.tf_tensor { + // CHECK: tfrt_fallback_async.const_dense_tensor + %a = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %ra = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + tfrt.return %ra : !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_const_tensor_canonicalization_single_string_operand +func.func @test_const_tensor_canonicalization_single_string_operand() -> !tfrt_fallback.tf_tensor { + // CHECK: tfrt_fallback_async.const_string_tensor + %a = corert.const_string_tensor {shape = [2], value = ["string", "tensor"]} + %ra = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + tfrt.return %ra : !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_const_tensor_canonicalization_multiple_operands +func.func @test_const_tensor_canonicalization_multiple_operands() -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + // CHECK: tfrt_fallback_async.const_dense_tensor + // CHECK-NEXT: tfrt_fallback_async.const_string_tensor + %a = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %b = corert.const_string_tensor {shape = [2], value = ["string", "tensor"]} + %ra, %rb = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a, %b {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + tfrt.return %ra, %rb : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// Tests the case where the conversion op is partially canonicalizable. +// CHECK-LABEL: func @test_const_tensor_canonicalization_mixed_operands +// CHECK-SAME: ([[arg0:%.*]]: !corert.tensorhandle, [[arg1:%.*]]: !corert.tensorhandle) +func.func @test_const_tensor_canonicalization_mixed_operands(%arg0: !corert.tensorhandle, %arg1: !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + %a = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %b = corert.const_dense_tensor dense<[false, true]> : tensor<2xi1> + // CHECK: [[b:%.*]] = tfrt_fallback_async.const_dense_tensor dense<[false, true]> : tensor<2xi1> + // CHECK-NEXT: [[a:%.*]] = tfrt_fallback_async.const_dense_tensor dense<[true, false]> : tensor<2xi1> + // CHECK-NEXT: tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor [[arg0]] + // CHECK-NEXT: tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor [[arg1]] + %ra, %rarg0, %rb, %rarg1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a, %arg0, %b, %arg1 {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + tfrt.return %ra, %rarg0, %rb, %rarg1 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// Tests that if the conversion op is partially canonicalizable, the non-canonicalizable operands are always separated into individual conversion ops. +// CHECK-LABEL: func @test_const_tensor_canonicalization_mixed_operands_no_consolidation +// CHECK-SAME: ([[arg0:%.*]]: !corert.tensorhandle, [[arg1:%.*]]: !corert.tensorhandle) +func.func @test_const_tensor_canonicalization_mixed_operands_no_consolidation(%arg0: !corert.tensorhandle, %arg1: !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + // CHECK-NEXT: tfrt_fallback_async.const_dense_tensor dense<[true, false]> : tensor<2xi1> + // CHECK-NEXT: tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor [[arg0]] + // CHECK-NEXT: tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor [[arg1]] + + %a = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %rarg0, %rarg1, %ra = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %arg0, %arg1, %a {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + tfrt.return %rarg0, %rarg1, %ra : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_remove_double_conversion +// CHECK-SAME: ([[arg:%.*]]: !tfrt_fallback.tf_tensor +func.func @test_remove_double_conversion(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor) { + // CHECK-NOT: fallback_tensor_to_corert_tensorhandle + // CHECK-NOT: corert_tensorhandle_to_fallback_tensor + + %0 = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %arg {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + %1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %0 {_tfrt_cost = 1 : i64, device = "/CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + + // CHECK: tfrt.return [[arg]] + tfrt.return %1 : !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_inline.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_inline.mlir new file mode 100644 index 0000000000000000000000000000000000000000..318804610396a35bf51e31c4217543f30fec9878 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/fallback_inline.mlir @@ -0,0 +1,23 @@ +// RUN: tf-tfrt-opt %s -inline | FileCheck %s + +func.func @_tfrt_fallback_init(%arg0: !tfrt.chain) -> !tfrt.chain { + %0 = tfrt_fallback_async.createop(%arg0) key(0) device("/device:CPU:0") "tf.Less"() {T = i32} num_args(2) + tfrt.return %0 : !tfrt.chain +} + +func.func @callee(%ch: !tfrt.chain, %arg: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) { + %const = tfrt_fallback_async.const_dense_tensor dense<9> : tensor {_tfrt_cost = 1 : i64} + %result = tfrt_fallback_async.executeop key(0) cost(3) device("/device:CPU:0") "tf.Less"(%arg, %const) {T = i32} : 1 + tfrt.return %ch, %result : !tfrt.chain, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_inline_fallback_ops +// CHECK-SAME: ([[ch:%.*]]: !tfrt.chain, [[arg:%.*]]: !tfrt_fallback.tf_tensor +func.func @test_inline_fallback_ops(%ch: !tfrt.chain, %arg: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) { + // CHECK-NOT: tfrt.call + // CHECK: [[const:%.*]] = tfrt_fallback_async.const_dense_tensor dense<9> : tensor + // CHECK-NEXT: [[result:%.*]] = tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/device:CPU:0") "tf.Less"([[arg]], [[const]]) {T = i32} : 1 + // CHECK-NEXT: tfrt.return [[ch]], [[result]] : !tfrt.chain, !tfrt_fallback.tf_tensor + %out_ch, %result = tfrt.call @callee(%ch, %arg) : (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + tfrt.return %out_ch, %result : !tfrt.chain, !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e8fb56e7cabc553b88e55555c925ec6384c109d2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes.mlir @@ -0,0 +1,32 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline %s | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 567 : i32}} { + // CHECK-LABEL: func @__inference_pruned_35 + func.func @__inference_pruned_35() -> tensor attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "flatmapdataset__4_RetVal"}} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %2 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %1, %2) {device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/device:CPU:0") "tf.FlatMapDataset"({{.*}}) {Targuments = [], metadata = "", output_shapes = [#corert.shape<>], output_types = [i64]} {f = "__inference_Dataset_flat_map_lambda_19"} : 1 + %4 = "tf.FlatMapDataset"(%3) {Targuments = [], device = "/device:CPU:0", f = @__inference_Dataset_flat_map_lambda_190, output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor) -> tensor + func.return %4 : tensor + } + // CHECK-LABEL: __inference_Dataset_flat_map_lambda_190 + func.func private @__inference_Dataset_flat_map_lambda_190(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._original_func_name = "__inference_Dataset_flat_map_lambda_19", tf._tf_data_function = true, tf.signature.is_stateful} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %2 = "tf.Const"() {device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %2, %1) {device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/device:CPU:0") "tf.MapDataset"({{.*}}) {Targuments = [], metadata = "", output_shapes = [#corert.shape<>], output_types = [i64], preserve_cardinality = true, use_inter_op_parallelism = true} {f = "__inference_Dataset_map_lambda_16"} : 1 + %4 = "tf.MapDataset"(%3) {device = "/device:CPU:0", f = @__inference_Dataset_map_lambda_160, f._tf_data_function = true, output_shapes = [#tf_type.shape<>], output_types = [i64], preserve_cardinality = true, use_inter_op_parallelism = true, metadata = ""} : (tensor) -> tensor + %5 = "tf.Identity"(%4) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %5 : tensor + } + // CHECK-LABEL: __inference_Dataset_map_lambda_160 + func.func private @__inference_Dataset_map_lambda_160(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._tf_data_function = true} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Mul"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %2 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes_multiple_callers.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes_multiple_callers.mlir new file mode 100644 index 0000000000000000000000000000000000000000..fc5a340d350eb22d21ee550bf317689a97807bc8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/func_attributes_multiple_callers.mlir @@ -0,0 +1,40 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline %s | FileCheck %s + +// Checks that the ops' function attribute references the original function name +// `funcB` for `funcB_renamed` after the module is lowered to TFRT. Note that, +// `funcB_renamed` are called twice, so `CreateGuaranteeAllFuncsOneUsePass` will +// make a replicaion of `funcB_renamed` with a different name. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 567 : i32}} { + // CHECK-LABEL: @funcA + func.func @funcA() -> (tensor, tensor) attributes {tf.entry_function = {control_outputs = "", inputs = "", outputs = "flatmapdataset__4_RetVal"}} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %2 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %1, %2) {device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/device:CPU:0") "tf.FlatMapDataset"({{.*}}) {Targuments = [], metadata = "", output_shapes = [#corert.shape<>], output_types = [i64]} {f = "funcB"} : 1 + %4 = "tf.FlatMapDataset"(%3) {Targuments = [], device = "/device:CPU:0", f = @funcB_renamed, output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor) -> tensor + %5 = "tf.RangeDataset"(%1, %2, %0) {device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/device:CPU:0") "tf.FlatMapDataset"({{.*}}) {Targuments = [], metadata = "", output_shapes = [#corert.shape<>], output_types = [i64]} {f = "funcB"} : 1 + %6 = "tf.FlatMapDataset"(%5) {Targuments = [], device = "/device:CPU:0", f = @funcB_renamed, output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor) -> tensor + func.return %4, %6 : tensor, tensor + } + // CHECK-LABEL: @funcB_renamed + func.func private @funcB_renamed(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._original_func_name = "funcB", tf._tf_data_function = true, tf.signature.is_stateful} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %2 = "tf.Const"() {device = "/device:CPU:0", value = dense<5> : tensor} : () -> tensor + %3 = "tf.RangeDataset"(%0, %2, %1) {device = "/device:CPU:0", output_shapes = [#tf_type.shape<>], output_types = [i64], metadata = ""} : (tensor, tensor, tensor) -> tensor + // CHECK: tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/device:CPU:0") "tf.MapDataset"({{.*}}) {Targuments = [], metadata = "", output_shapes = [#corert.shape<>], output_types = [i64], preserve_cardinality = true, use_inter_op_parallelism = true} {f = "funcC"} : 1 + %4 = "tf.MapDataset"(%3) {device = "/device:CPU:0", f = @funcC_renamed, f._tf_data_function = true, output_shapes = [#tf_type.shape<>], output_types = [i64], preserve_cardinality = true, use_inter_op_parallelism = true, metadata = ""} : (tensor) -> tensor + %5 = "tf.Identity"(%4) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %5 : tensor + } + // CHECK-LABEL: @funcC_renamed + func.func private @funcC_renamed(%arg0: tensor {tf._user_specified_name = "args_0"}) -> tensor attributes {tf._tf_data_function = true, tf._original_func_name = "funcC"} { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Mul"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.Identity"(%1) {device = "/device:CPU:0"} : (tensor) -> tensor + func.return %2 : tensor + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/insert_fallback_tensor_copy.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/insert_fallback_tensor_copy.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a91619a212249367f98856c0ca5b50d9d7d1b570 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/insert_fallback_tensor_copy.mlir @@ -0,0 +1,55 @@ +// RUN: tf-tfrt-opt %s -tfrt-insert-fallback-tensor-copy | FileCheck %s + +// CHECK-LABEL: func @test_insert_copy +// CHECK-SAME: ([[arg:%.*]]: !tfrt_fallback.tf_tensor +func.func @test_insert_copy(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) attributes {tfrt.cost_threshold = 1024} { + // CHECK: [[value:%.*]] = tfrt_fallback_async.executeop key({{.*}}) {{.*}} "tf.AddV2"([[arg]], [[arg]]) + %0 = tfrt_fallback_async.executeop key(0) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + // CHECK: [[copy:%.*]] = tfrt_fallback_async.copy_if_small [[value]] + // CHECK: tfrt_fallback_async.executeop key({{.*}}) {{.*}} "tf.AddV2"([[copy]], [[copy]]) + %1 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %2 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %3 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %4 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %5 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %6 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %7 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %8 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %9 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + %10 = tfrt_fallback_async.executeop key(1) cost(512) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%0, %0) {T = f32} : 1 + tfrt.return %1, %2, %3 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_insert_copy_for_arg +// CHECK-SAME: ([[arg:%.*]]: !tfrt_fallback.tf_tensor +func.func @test_insert_copy_for_arg(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) attributes {tfrt.cost_threshold = 1024} { + // CHECK: [[copy:%.*]] = tfrt_fallback_async.copy_if_small [[arg]] + // CHECK: tfrt_fallback_async.executeop key({{.*}}) {{.*}} "tf.AddV2"([[copy]], [[copy]]) + %0 = tfrt_fallback_async.executeop key(0) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %1 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %2 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %3 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %4 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %5 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %6 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %7 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %8 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %9 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + %10 = tfrt_fallback_async.executeop key(1) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + tfrt.return %0, %1, %2 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_no_copy_for_return +func.func @test_no_copy_for_return(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) attributes {tfrt.cost_threshold = 1024} { + // CHECK-NOT: tfrt_fallback_async.copy_if_small + %0 = tfrt_fallback_async.executeop key(0) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AddV2"(%arg, %arg) {T = f32} : 1 + tfrt.return %arg, %0 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: func @test_no_copy_for_few_uses +func.func @test_no_copy_for_few_uses(%arg: !tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) attributes {tfrt.cost_threshold = 1024} { + // CHECK-NOT: tfrt_fallback_async.copy_if_small + %0 = tfrt_fallback_async.executeop key(0) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.Relu"(%arg) {T = f32} : 1 + %1 = tfrt_fallback_async.executeop key(0) cost(1024) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.Relu"(%arg) {T = f32} : 1 + tfrt.return %0, %1 : !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/merge_tf_if_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/merge_tf_if_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..9af7714a2cf76d363d7eb614713474918e7315a2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/merge_tf_if_ops.mlir @@ -0,0 +1,129 @@ +// RUN: tf-tfrt-opt -tfrt-merge-tf-if-ops %s | FileCheck %s -dump-input=fail + +func.func @no_side_effect_then_0(%x: tensor, %y: tensor) -> (tensor, tensor) { + %0 = "tf.AddV2"(%x, %y) : (tensor, tensor) -> tensor + func.return %0, %0 : tensor, tensor +} + +func.func @no_side_effect_else_0(%x: tensor, %y: tensor) -> (tensor, tensor) { + %0 = "tf.Const"() {value = dense<1> : tensor } : () -> tensor + %1 = "tf.AddV2"(%x, %0) : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%y, %1) : (tensor, tensor) -> tensor + func.return %1, %2 : tensor, tensor +} + +func.func @no_side_effect_then_1(%x: tensor, %y: tensor) -> tensor { + %0 = "tf.AddV2"(%x, %y) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @no_side_effect_else_1(%x: tensor, %y: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<2> : tensor } : () -> tensor + %1 = "tf.AddV2"(%x, %0) : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%y, %1) : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +func.func @nested_if_op_then_0(%cond: tensor, %x: tensor, %y: tensor) -> (tensor, tensor) { + %0 = "tf.AddV2"(%x, %y) : (tensor, tensor) -> tensor + func.return %0, %0 : tensor, tensor +} + +func.func @nested_if_op_else_0(%cond: tensor, %x: tensor, %y: tensor) -> (tensor, tensor) { + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_0, then_branch = @no_side_effect_then_0, is_stateless = true} : (tensor, tensor, tensor) -> (tensor, tensor) + func.return %0, %1 : tensor, tensor +} + +func.func @nested_if_op_then_1(%cond: tensor, %x: tensor, %y: tensor) -> tensor { + %0 = "tf.AddV2"(%x, %y) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @nested_if_op_else_1(%cond: tensor, %x: tensor, %y: tensor) -> tensor { + %0 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_1, then_branch = @no_side_effect_then_1, is_stateless = true} : (tensor, tensor, tensor) -> tensor + func.return %0 : tensor +} + +// CHECK-LABEL: func private @merge_stateless_merged_if_0_0_then +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK: [[r0:%.*]] = "tf.AddV2"([[x]], [[y]]) +// CHECK: return [[r0]], [[r0]], [[r0]] + +// CHECK-LABEL: func private @merge_stateless_merged_if_0_0_else +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK-DAG: [[cst:%.*]] = "tf.Const"() <{value = dense<1> : tensor}> +// CHECK-DAG: [[cst_0:%.*]] = "tf.Const"() <{value = dense<2> : tensor}> +// CHECK: [[r0:%.*]] = "tf.AddV2"([[x]], [[cst]]) +// CHECK: [[r1:%.*]] = "tf.AddV2"([[y]], [[r0]]) +// CHECK: [[r2:%.*]] = "tf.AddV2"([[x]], [[cst_0]]) +// CHECK: [[r3:%.*]] = "tf.AddV2"([[y]], [[r2]]) +// CHECK: return [[r0]], [[r1]], [[r3]] + +// CHECK-LABEL: func @merge_stateless +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor, [[cond:%.*]]: tensor) +func.func @merge_stateless(%x: tensor, %y: tensor, %cond: tensor) -> (tensor, tensor, tensor) { + // CHECK-NEXT: [[res:%.*]]:3 = "tf.If"([[cond]], [[x]], [[y]]) + // CHECK-SAME: <{else_branch = @merge_stateless_merged_if_0_0_else, is_stateless = true, then_branch = @merge_stateless_merged_if_0_0_then}> + // CHECK-SAME: (tensor, tensor, tensor) -> (tensor, tensor, tensor) + // CHECK-NEXT: return [[res]]#0, [[res]]#1, [[res]]#2 + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_0, then_branch = @no_side_effect_then_0, is_stateless = true} : (tensor, tensor, tensor) -> (tensor, tensor) + %2 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_1, then_branch = @no_side_effect_then_1, is_stateless = true} : (tensor, tensor, tensor) -> tensor + func.return %0, %1, %2 : tensor, tensor, tensor +} + +// CHECK-LABEL: func private @merge_nested_if_op_merged_if_0_0_then +// CHECK-SAME: ([[cond:%.*]]: tensor, [[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK-NEXT: [[r0:%.*]] = "tf.AddV2"([[x]], [[y]]) : (tensor, tensor) -> tensor +// CHECK-NEXT: return [[r0]], [[r0]], [[r0]] + +// CHECK-LABEL: func private @merge_nested_if_op_merged_if_0_0_else_merged_if_1_0_then +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK-NEXT: [[r0:%.*]] = "tf.AddV2"([[x]], [[y]]) : (tensor, tensor) -> tensor +// CHECK-NEXT: return [[r0]], [[r0]], [[r0]] + +// CHECK-LABEL: func private @merge_nested_if_op_merged_if_0_0_else_merged_if_1_0_else +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK-NEXT: [[cst:%.*]] = "tf.Const"() <{value = dense<2> : tensor}> +// CHECK-NEXT: [[cst_0:%.*]] = "tf.Const"() <{value = dense<1> : tensor}> +// CHECK-NEXT: [[r0:%.*]] = "tf.AddV2"([[x]], [[cst_0]]) +// CHECK-NEXT: [[r1:%.*]] = "tf.AddV2"([[y]], [[r0]]) +// CHECK-NEXT: [[r2:%.*]] = "tf.AddV2"([[x]], [[cst]]) +// CHECK-NEXT: [[r3:%.*]] = "tf.AddV2"([[y]], [[r2]]) +// CHECK-NEXT: return [[r0]], [[r1]], [[r3]] + +// CHECK-LABEL: func private @merge_nested_if_op_merged_if_0_0_else +// CHECK-SAME: ([[cond:%.*]]: tensor, [[x:%.*]]: tensor, [[y:%.*]]: tensor) +// CHECK-NEXT: [[r0:%.*]]:3 = "tf.If"(%arg0, %arg1, %arg2) <{else_branch = @merge_nested_if_op_merged_if_0_0_else_merged_if_1_0_else, is_stateless = true, then_branch = @merge_nested_if_op_merged_if_0_0_else_merged_if_1_0_then}> : (tensor, tensor, tensor) -> (tensor, tensor, tensor) +// CHECK-NEXT: return [[r0]]#0, [[r0]]#1, [[r0]]#2 + +// CHECK-LABEL: func @merge_nested_if_op +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor, [[cond:%.*]]: tensor, [[nested_cond:%.*]]: tensor) +func.func @merge_nested_if_op(%x: tensor, %y: tensor, %cond: tensor, %nested_cond: tensor) -> (tensor, tensor, tensor) { + // CHECK-NEXT: [[res:%.*]]:3 = "tf.If"([[cond]], [[nested_cond]], [[x]], [[y]]) + // CHECK-SAME: <{else_branch = @merge_nested_if_op_merged_if_0_0_else, is_stateless = true, then_branch = @merge_nested_if_op_merged_if_0_0_then}> + // CHECK-SAME: (tensor, tensor, tensor, tensor) -> (tensor, tensor, tensor) + // CHECK-NEXT: return [[res]]#0, [[res]]#1, [[res]]#2 + %0, %1 = "tf.If"(%cond, %nested_cond, %x, %y) {else_branch = @nested_if_op_else_0, then_branch = @nested_if_op_then_0, is_stateless = true} : (tensor, tensor, tensor, tensor) -> (tensor, tensor) + %2 = "tf.If"(%cond, %nested_cond, %x, %y) {else_branch = @nested_if_op_else_1, then_branch = @nested_if_op_then_1, is_stateless = true} : (tensor, tensor, tensor, tensor) -> tensor + func.return %0, %1, %2 : tensor, tensor, tensor +} + +// CHECK-LABEL: func @merge_side_effect +// CHECK-SAME: ([[x:%.*]]: tensor, [[y:%.*]]: tensor, [[cond:%.*]]: tensor) +func.func @merge_side_effect(%x: tensor, %y: tensor, %cond: tensor) -> (tensor, tensor, tensor) { + // CHECK-NEXT: tf.If + // CHECK-SAME: is_stateless = false + // CHECK-NEXT: return + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_0, then_branch = @no_side_effect_then_0, is_stateless = true} : (tensor, tensor, tensor) -> (tensor, tensor) + %2 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_1, then_branch = @no_side_effect_then_1, is_stateless = false} : (tensor, tensor, tensor) -> tensor + func.return %0, %1, %2 : tensor, tensor, tensor +} + +// CHECK-LABEL: func @multiple_uses +func.func @multiple_uses(%x: tensor, %y: tensor, %cond: tensor) -> (tensor, tensor, tensor) { + // CHECK-NEXT: tf.If + // CHECK-SAME: <{else_branch = @multiple_uses_merged_if_0_0_else, is_stateless = true, then_branch = @multiple_uses_merged_if_0_0_then}> + %0, %1 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_0, then_branch = @no_side_effect_then_0, is_stateless = true} : (tensor, tensor, tensor) -> (tensor, tensor) + %2 = "tf.If"(%cond, %x, %y) {else_branch = @no_side_effect_else_1, then_branch = @no_side_effect_then_1, is_stateless = true} : (tensor, tensor, tensor) -> tensor + func.return %0, %1, %2 : tensor, tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/optimize_tf_control_flow_side_effect.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/optimize_tf_control_flow_side_effect.mlir new file mode 100644 index 0000000000000000000000000000000000000000..8c8f5ff70550884f40d1c4955c3f09fb83508056 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/optimize_tf_control_flow_side_effect.mlir @@ -0,0 +1,176 @@ +// RUN: tf-tfrt-opt -split-input-file -tfrt-optimize-tf-control-flow-side-effect %s | FileCheck %s + +func.func @no_side_effect_cond(%arg: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %1 = "tf.Less"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func @no_side_effect_body(%arg: tensor) -> tensor { + %0 = "tf.AddV2"(%arg, %arg) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @no_side_effect_body2(%arg: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<1> : tensor } : () -> tensor + %1 = "tf.AddV2"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @set_stateless +func.func @set_stateless(%arg: tensor, %cond: tensor) -> (tensor, tensor) { + // CHECK: tf.While + // CHECK-SAME: is_stateless = true + %0 = "tf.While"(%arg) { cond = @no_side_effect_cond, body = @no_side_effect_body, is_stateless = false} : (tensor) -> (tensor) + // CHECK: tf.If + // CHECK-SAME: is_stateless = true + %1 = "tf.If"(%cond, %arg) {else_branch = @no_side_effect_body, then_branch = @no_side_effect_body2, is_stateless = false} : (tensor, tensor) -> tensor + func.return %0, %1 : tensor, tensor +} + +// CHECK-LABEL: func @nested_set_stateless +func.func @nested_set_stateless(%arg: tensor, %cond: tensor) -> (tensor, tensor) { + // CHECK: tf.If + // CHECK-SAME: is_stateless = true + %1 = "tf.If"(%cond, %arg) {else_branch = @no_side_effect_body, then_branch = @no_side_effect_nested_body, is_stateless = false} : (tensor, tensor) -> tensor + // CHECK: tf.While + // CHECK-SAME: is_stateless = true + %0 = "tf.While"(%arg) { cond = @no_side_effect_cond, body = @no_side_effect_nested_body, is_stateless = false} : (tensor) -> (tensor) + func.return %0, %1 : tensor, tensor +} + +// CHECK-LABEL: func @no_side_effect_nested_body +func.func @no_side_effect_nested_body(%arg: tensor) -> tensor { + %const = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %cond = "tf.Less"(%arg, %const) : (tensor, tensor) -> tensor + // CHECK: tf.If + // CHECK-SAME: is_stateless = true + %0 = "tf.If"(%cond, %arg) {else_branch = @no_side_effect_body, then_branch = @no_side_effect_body2, is_stateless = false} : (tensor, tensor) -> tensor + // CHECK: tf.While + // CHECK-SAME: is_stateless = true + %1 = "tf.While"(%0) { cond = @no_side_effect_cond, body = @no_side_effect_body, is_stateless = false} : (tensor) -> (tensor) + func.return %1 : tensor +} + +// ----- + +func.func @no_side_effect_cond(%arg: tensor, %handle: tensor>>) -> tensor { + %0 = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %1 = "tf.Less"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func @side_effect_body(%arg: tensor, %handle: tensor>>) -> (tensor, tensor>>) { + %0 = "tf.ReadVariableOp"(%handle) : (tensor>>) -> tensor + %1 = "tf.AddV2"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1, %handle : tensor, tensor>> +} + +func.func @no_side_effect_body(%arg: tensor, %handle: tensor>>) -> (tensor, tensor>>) { + %0 = "tf.AddV2"(%arg, %arg) : (tensor, tensor) -> tensor + func.return %0, %handle : tensor, tensor>> +} + +// CHECK-LABEL: func @no_set_stateless +func.func @no_set_stateless(%arg: tensor, %cond: tensor) -> (tensor, tensor) { + %handle = "tf.VarHandleOp"() { container = "", shared_name = "var" } : () -> tensor>> + "tf.AssignVariableOp"(%handle, %arg) : (tensor>>, tensor) -> () + // CHECK: tf.While + // CHECK-SAME: is_stateless = false + %0, %h1 = "tf.While"(%arg, %handle) { cond = @no_side_effect_cond, body = @side_effect_body, is_stateless = false} : (tensor, tensor>>) -> (tensor, tensor>>) + // CHECK: tf.If + // CHECK-SAME: is_stateless = false + %1, %h2 = "tf.If"(%cond, %arg, %handle) {else_branch = @no_side_effect_body, then_branch = @side_effect_body, is_stateless = false} : (tensor, tensor, tensor>>) -> (tensor, tensor>>) + func.return %0, %1 : tensor, tensor +} + + +// CHECK-LABEL: func @side_effect_nested_body +func.func @side_effect_nested_body(%arg: tensor, %handle: tensor>>) -> (tensor, tensor>>) { + %const = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + // CHECK: tf.While + // CHECK-SAME: is_stateless = false + %0, %h1 = "tf.While"(%arg, %handle) { cond = @no_side_effect_cond, body = @side_effect_body, is_stateless = false} : (tensor, tensor>>) -> (tensor, tensor>>) + %cond = "tf.Less"(%0, %const) : (tensor, tensor) -> tensor + // CHECK: tf.If + // CHECK-SAME: is_stateless = false + %1, %h2 = "tf.If"(%cond, %0, %handle) {else_branch = @no_side_effect_body, then_branch = @side_effect_body, is_stateless = false} : (tensor, tensor, tensor>>) -> (tensor, tensor>>) + func.return %1, %handle : tensor, tensor>> +} + +// CHECK-LABEL: func @nested_no_set_stateless +func.func @nested_no_set_stateless(%arg: tensor, %cond: tensor) -> (tensor, tensor) { + %handle = "tf.VarHandleOp"() { container = "", shared_name = "var" } : () -> tensor>> + "tf.AssignVariableOp"(%handle, %arg) : (tensor>>, tensor) -> () + // CHECK: tf.While + // CHECK-SAME: is_stateless = false + %0, %h1 = "tf.While"(%arg, %handle) { cond = @no_side_effect_cond, body = @side_effect_nested_body, is_stateless = false} : (tensor, tensor>>) -> (tensor, tensor>>) + // CHECK: tf.If + // CHECK-SAME: is_stateless = false + %1, %h2 = "tf.If"(%cond, %arg, %handle) {else_branch = @no_side_effect_body, then_branch = @side_effect_nested_body, is_stateless = false} : (tensor, tensor, tensor>>) -> (tensor, tensor>>) + func.return %0, %1 : tensor, tensor +} + +// ----- + +// Set stateless if the body contains only read-only side-effecting ops. + +func.func private @cond(%arg: tensor, %handle: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %1 = "tf.Less"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func private @body(%arg: tensor, %handle: tensor) -> (tensor, tensor) { + %default = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %0 = "tf.LookupTableFindV2"(%handle, %arg, %default) : (tensor, tensor, tensor) -> tensor + func.return %0, %handle : tensor, tensor +} + +// CHECK-LABEL: func @set_readonly_stateless +func.func @set_readonly_stateless(%arg: tensor, %cond: tensor) -> tensor { + %handle = "tf.HashTableV2"() {container = "", key_dtype = i32, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i32} : () -> tensor + // CHECK: tf.While + // CHECK-SAME: is_stateless = true + %0, %handle_2 = "tf.While"(%arg, %handle) { cond = @cond, body = @body, is_stateless = false} : (tensor, tensor) -> (tensor, tensor) + func.return %0: tensor +} + +// ----- + +// Set stateless if write side-effecting ops are inside the initializer. + +module attributes {tf_saved_model.semantics} { + +"tf_saved_model.session_initializer"() {initializers = [@init]} : () -> () + +func.func @init() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_init"]} { + %keys = "tf.Const"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32> } : () -> tensor<4xi32> + %values = "tf.Const"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32> } : () -> tensor<4xi32> + %handle = "tf.HashTableV2"() {container = "", key_dtype = i32, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i32} : () -> tensor + "tf.LookupTableImportV2"(%handle, %keys, %values) : (tensor, tensor<4xi32>, tensor<4xi32>) -> () + func.return +} + +func.func private @cond(%arg: tensor, %handle: tensor) -> tensor { + %0 = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %1 = "tf.Less"(%arg, %0) : (tensor, tensor) -> tensor + func.return %1 : tensor +} + +func.func private @body(%arg: tensor, %handle: tensor) -> (tensor, tensor) { + %default = "tf.Const"() {value = dense<16> : tensor } : () -> tensor + %0 = "tf.LookupTableFindV2"(%handle, %arg, %default) : (tensor, tensor, tensor) -> tensor + func.return %0, %handle : tensor, tensor +} + +// CHECK-LABEL: func @set_readonly_stateless +func.func @set_readonly_stateless(%arg: tensor {tf_saved_model.index_path = ["arg"]}, %cond: tensor {tf_saved_model.index_path = ["cond"]}) -> (tensor {tf_saved_model.index_path = ["output"]}) attributes {tf_saved_model.exported_names = ["main"]} { + %handle = "tf.HashTableV2"() {container = "", key_dtype = i32, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i32} : () -> tensor + // CHECK: tf.While + // CHECK-SAME: is_stateless = true + %0, %handle_2 = "tf.While"(%arg, %handle) { cond = @cond, body = @body, is_stateless = false} : (tensor, tensor) -> (tensor, tensor) + func.return %0: tensor +} + +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/remove_tf_if_const_args.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/remove_tf_if_const_args.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d5a1335887acad0e035d0e60b90c2172e6154e22 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/remove_tf_if_const_args.mlir @@ -0,0 +1,49 @@ +// RUN: tf-tfrt-opt -tfrt-remove-tf-if-const-args %s | FileCheck %s -dump-input-filter=all + +func.func @then(%x: tensor, %y: tensor) -> (tensor) { + %0 = "tf.AddV2"(%x, %y) : (tensor, tensor) -> tensor + func.return %0 : tensor +} + +func.func @else(%x: tensor, %y: tensor) -> (tensor) { + %0 = "tf.Const"() {value = dense<1> : tensor } : () -> tensor + %1 = "tf.AddV2"(%x, %0) : (tensor, tensor) -> tensor + %2 = "tf.AddV2"(%y, %1) : (tensor, tensor) -> tensor + func.return %2 : tensor +} + +// CHECK-LABEL: func private @then_removed_const_args_0 +// CHECK-SAME: ([[x:%.*]]: tensor) +// CHECK: [[const:%.*]] = "tf.Const" +// CHECK-SAME: value = dense<10> : tensor +// CHECK: [[r:%.*]] = "tf.StatefulPartitionedCall"([[x]], [[const]]) +// CHECK-SAME: f = @then} +// CHECK: return [[r]] + +// CHECK-LABEL: func private @else_removed_const_args_0 +// CHECK-SAME: ([[x:%.*]]: tensor) +// CHECK: [[const:%.*]] = "tf.Const" +// CHECK-SAME: value = dense<10> : tensor +// CHECK: [[r:%.*]] = "tf.StatefulPartitionedCall"([[x]], [[const]]) +// CHECK-SAME: f = @else} +// CHECK: return [[r]] + +// CHECK-LABEL: func @remove_const_args +// CHECK-SAME: ([[x:%.*]]: tensor, [[cond:%.*]]: tensor) +func.func @remove_const_args(%x: tensor, %cond: tensor) -> (tensor) { + %0 = "tf.Const"() {value = dense<10> : tensor } : () -> tensor + // CHECK: [[res:%.*]] = "tf.If"([[cond]], [[x]]) + // CHECK-SAME: {else_branch = @else_removed_const_args_0, is_stateless = false, then_branch = @then_removed_const_args_0} + // CHECK-NEXT: return [[res]] + %1 = "tf.If"(%cond, %x, %0) {else_branch = @else, then_branch = @then, is_stateless = false} : (tensor, tensor, tensor) -> tensor + func.return %1 : tensor +} + +// CHECK-LABEL: func @multiple_uses +func.func @multiple_uses(%x: tensor, %cond: tensor) -> (tensor) { + %0 = "tf.Const"() {value = dense<10> : tensor } : () -> tensor + // CHECK: [[res:%.*]] = "tf.If" + // CHECK-SAME: {else_branch = @else_removed_const_args_1, is_stateless = false, then_branch = @then_removed_const_args_1} + %1 = "tf.If"(%cond, %0, %x) {else_branch = @else, then_branch = @then, is_stateless = false} : (tensor, tensor, tensor) -> tensor + func.return %1 : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/reorder_assert.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/reorder_assert.mlir new file mode 100644 index 0000000000000000000000000000000000000000..85e4b099bad1a183b26395f41014e3bf39ba1370 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/reorder_assert.mlir @@ -0,0 +1,47 @@ +// RUN: tf-tfrt-opt -tfrt-reorder-tf-assert %s | FileCheck %s + +// CHECK-LABEL: @reorder_assert +func.func @reorder_assert(%key0: tensor, %key1: tensor) -> (tensor, tensor) { + %error_message = "tf.Const"() {value = dense<"error"> : tensor} : () -> tensor + %default = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %handle = "tf.HashTableV2"() {container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", key_dtype = !tf_type.string, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + + // CHECK: tf.LookupTableFindV2 + // CHECK-NOT: tf.Assert + // CHECK: tf.LookupTableFindV2 + // CHECK: tf.Assert + + %value0 = "tf.LookupTableFindV2"(%handle, %key0, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + %cond = "tf.Equal"(%value0, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0", incompatible_shape_error = true} : (tensor, tensor) -> tensor + "tf.Assert"(%cond, %error_message) {device = "/job:localhost/replica:0/task:0/device:CPU:0", summarize = 3 : i64} : (tensor, tensor) -> () + %value1 = "tf.LookupTableFindV2"(%handle, %key1, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + func.return %value0, %value1 : tensor, tensor +} + +func.func private @else_branch(%arg0: tensor) -> tensor { + %cst = "tf.Const"() {value = dense<"Empty SparseTensor with shape"> : tensor} : () -> tensor + "tf.Assert"(%arg0, %cst) {device = "/job:localhost/replica:0/task:0/device:CPU:0", summarize = 3 : i64} : (tensor, tensor) -> () + func.return %arg0 : tensor +} + +func.func private @then_branch(%arg0: tensor) -> tensor { + func.return %arg0 : tensor +} + +// CHECK-LABEL: @reorder_assert_only_if +func.func @reorder_assert_only_if(%key0: tensor, %key1: tensor) -> (tensor, tensor) { + %error_message = "tf.Const"() {value = dense<"error"> : tensor} : () -> tensor + %default = "tf.Const"() {value = dense<-1> : tensor} : () -> tensor + %handle = "tf.HashTableV2"() {container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", key_dtype = !tf_type.string, shared_name = "hash_table", use_node_name_sharing = false, value_dtype = i64} : () -> tensor + + // CHECK: tf.LookupTableFindV2 + // CHECK-NOT: tf.If + // CHECK: tf.LookupTableFindV2 + // CHECK: tf.If + + %value0 = "tf.LookupTableFindV2"(%handle, %key0, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + %cond = "tf.Equal"(%value0, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0", incompatible_shape_error = true} : (tensor, tensor) -> tensor + %unused = "tf.If"(%cond, %cond) {device = "/job:localhost/replica:0/task:0/device:CPU:0", else_branch = @else_branch, is_stateless = false, then_branch = @then_branch} : (tensor, tensor) -> tensor + %value1 = "tf.LookupTableFindV2"(%handle, %key1, %default) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor, tensor, tensor) -> tensor + func.return %value0, %value1 : tensor, tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/side_effects.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/side_effects.mlir new file mode 100644 index 0000000000000000000000000000000000000000..32ebed3b85704a8f6486505b46415c6c29a7f086 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/side_effects.mlir @@ -0,0 +1,15 @@ +// RUN: tf-tfrt-opt -tf-to-tfrt %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @assign_variable +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain) -> !tfrt.chain +func.func @assign_variable() { + // CHECK: [[ch1:%.*]], %results = tfrt_fallback_async.executeop.seq([[in_chain]]) key(0) cost({{.*}}) device("/device:CPU:0") "tf.VarHandleOp" + // CHECK-NEXT: [[ch2:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key(1) cost({{.*}}) device("/device:CPU:0") "tf.AssignVariableOp" + // CHECK-NEXT: [[out_ch:%.*]] = tfrt.merge.chains [[ch1]], [[ch2]] + // CHECK-NEXT: tfrt.return [[out_ch]] + + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<4.200000e+01> : tensor} : () -> tensor + %1 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shape = #tf_type.shape<>, shared_name = "x"} : () -> tensor>> + "tf.AssignVariableOp"(%1, %0) {device = "/device:CPU:0"} : (tensor>>, tensor) -> () + func.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3f1628f8543601e365a42fc4b727aa71dfd12613 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline.mlir @@ -0,0 +1,73 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline="enable-optimizer=true tfrt-cost-threshold=1024" %s | FileCheck %s --dump-input=fail + +// CHECK: tfrt.cost_threshold = 1024 : i64 +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 462 : i32}} { + +// CHECK-LABEL: func @__forward_call_369 +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain +// CHECK-SAME: [[arg1:%.*]]: !tfrt_fallback.tf_tensor {tf._user_specified_name = "inputs"}, +// CHECK-SAME: [[arg2:%.*]]: !tfrt_fallback.tf_tensor, [[arg3:%.*]]: !tfrt_fallback.tf_tensor, [[arg4:%.*]]: !tfrt_fallback.tf_tensor, [[arg5:%.*]]: !tfrt_fallback.tf_tensor) +// CHECK-SAME: -> (!tfrt.chain +// CHECK: [[o1:%.*]] = tfrt_fallback_async.const_dense_tensor +// CHECK: [[o2_chain:%.*]], [[o2:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"([[arg3]]) +// CHECK-NEXT: [[o3_chain:%.*]], [[o3:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"([[arg2]]) +// CHECK-NEXT: [[o4_chain:%.*]], [[o4:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"([[arg5]]) +// CHECK-NEXT: [[o5_chain:%.*]], [[o5:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"([[arg4]]) +// CHECK-NEXT: [[o6:%.*]] = tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf._FusedConv2D"([[arg1]], [[o3]], [[o2]]) +// CHECK-NEXT: [[o7:%.*]] = tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.AvgPool"([[o6]]) +// CHECK-NEXT: [[o8:%.*]] = tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.Reshape"([[o7]], [[o1]]) +// CHECK-NEXT: [[o9:%.*]] = tfrt_fallback_async.executeop key({{[0-9]+}}) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf._FusedMatMul"([[o8]], [[o5]], [[o4]]) +// CHECK-NEXT: [[out_chain:%.*]] = tfrt.merge.chains [[o2_chain]], [[o3_chain]], [[o4_chain]], [[o5_chain]] +// CHECK-NEXT: tfrt.return [[out_chain]], [[o9]], [[o5]], [[o8]], [[o6]], [[arg1]], [[o3]] : !tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor + func.func @__forward_call_369(%arg0: tensor<16x224x224x3xf32> {tf._user_specified_name = "inputs"}, %arg1: tensor<*x!tf_type.resource>, %arg2: tensor<*x!tf_type.resource>, %arg3: tensor<*x!tf_type.resource>, %arg4: tensor<*x!tf_type.resource>) -> (tensor, tensor<*xf32>, tensor, tensor<16x112x112x?xf32>, tensor<16x224x224x3xf32>, tensor<*xf32>) attributes {tf.entry_function = {control_outputs = "", inputs = "inputs_0,conv1_conv2d_readvariableop_resource,conv1_biasadd_readvariableop_resource,fc1000_matmul_readvariableop_resource,fc1000_biasadd_readvariableop_resource", outputs = "identity_RetVal,fc1000_matmul_readvariableop_RetVal,flatten_reshape_RetVal,relu_RetVal,inputs_RetVal,conv1_conv2d_readvariableop_RetVal"}} { + %0:6 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.ReadVariableOp"(%arg2) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %outputs_0, %control_1 = tf_executor.island wraps "tf.ReadVariableOp"(%arg1) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %outputs_2, %control_3 = tf_executor.island wraps "tf.ReadVariableOp"(%arg4) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %outputs_4, %control_5 = tf_executor.island wraps "tf.ReadVariableOp"(%arg3) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor<*xf32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<[-1, 16384]> : tensor<2xi32>} : () -> tensor<2xi32> + %outputs_8, %control_9 = tf_executor.island wraps "tf.Conv2D"(%arg0, %outputs_0) {data_format = "NHWC", device = "", dilations = [1, 1, 1, 1], explicit_paddings = [], padding = "SAME", strides = [1, 2, 2, 1], use_cudnn_on_gpu = true} : (tensor<16x224x224x3xf32>, tensor<*xf32>) -> tensor<16x112x112x?xf32> + %outputs_10, %control_11 = tf_executor.island wraps "tf.BiasAdd"(%outputs_8, %outputs) {data_format = "NHWC", device = ""} : (tensor<16x112x112x?xf32>, tensor<*xf32>) -> tensor<16x112x112x?xf32> + %outputs_12, %control_13 = tf_executor.island wraps "tf.Relu"(%outputs_10) {device = ""} : (tensor<16x112x112x?xf32>) -> tensor<16x112x112x?xf32> + %outputs_14, %control_15 = tf_executor.island wraps "tf.AvgPool"(%outputs_12) {data_format = "NHWC", device = "", ksize = [1, 7, 7, 1], padding = "VALID", strides = [1, 7, 7, 1]} : (tensor<16x112x112x?xf32>) -> tensor<16x16x16x?xf32> + %outputs_16, %control_17 = tf_executor.island wraps "tf.Reshape"(%outputs_14, %outputs_6) {device = ""} : (tensor<16x16x16x?xf32>, tensor<2xi32>) -> tensor + %outputs_18, %control_19 = tf_executor.island wraps "tf.MatMul"(%outputs_16, %outputs_4) {device = "", transpose_a = false, transpose_b = false} : (tensor, tensor<*xf32>) -> tensor + %outputs_20, %control_21 = tf_executor.island wraps "tf.BiasAdd"(%outputs_18, %outputs_2) {data_format = "NHWC", device = ""} : (tensor, tensor<*xf32>) -> tensor + %outputs_22, %control_23 = tf_executor.island wraps "tf.Identity"(%outputs_20) {device = ""} : (tensor) -> tensor + tf_executor.fetch %outputs_22, %outputs_4, %outputs_16, %outputs_12, %arg0, %outputs_0 : tensor, tensor<*xf32>, tensor, tensor<16x112x112x?xf32>, tensor<16x224x224x3xf32>, tensor<*xf32> + } + func.return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5 : tensor, tensor<*xf32>, tensor, tensor<16x112x112x?xf32>, tensor<16x224x224x3xf32>, tensor<*xf32> + } + + func.func @while_cond_lt9(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor + } + + func.func @while_body_add2(%arg0: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<2> : tensor} : () -> tensor + %1 = "tf.Add"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor + } + + // CHECK-LABEL: func @while_test + // CHECK-SAME: ([[ARG0:%.+]]: !tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + func.func @while_test() -> (tensor) { + // The predicate function should be inlined. + // CHECK-DAG: tfrt_fallback_async.const_dense_tensor dense<9> : tensor + // CHECK-DAG: tfrt_fallback_async.const_dense_tensor dense<0> : tensor + // CHECK-NEXT: tfrt_fallback_async.executeop key({{.*}}) cost({{.*}}) device("/device:CPU:0") "tf.Less" + // CHECK-NEXT: [[pred:%.*]] = tfrt_fallback_async.predicate + // CHECK-NEXT: tfrt.while [[pred]] @"[[while_func_prefix:.*]]/tfrt_body_1" + // CHECK-NEXT: tfrt.merge.chains + // CHECK-NEXT: tfrt.return + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %1 = "tf.While"(%0) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor) -> (tensor) + func.return %1 : tensor + } + // CHECK: func @"[[while_func_prefix]]/tfrt_body_1" + // CHECK-NOT: tfrt.call + + // CHECK: func @"[[while_cond_prefix:.*]]/tfrt_predicate" +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline_refvar.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline_refvar.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e5d55dbb88d2982ad4d1b2d937d1ceea297b0796 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/tf_to_corert_pipeline_refvar.mlir @@ -0,0 +1,18 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @__inference_pruned_131 +// CHECK-SAME: ([[in_chain:%.*]]: !tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) +// CHECK-NEXT: [[o_chain:%.*]], [[o:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key(0) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.VarHandleOp"() +// CHECK-NEXT: [[o_chain_0:%.*]], [[o1:%.*]] = tfrt_fallback_async.executeop.seq([[in_chain]]) key(1) cost({{.*}}) device("/job:localhost/replica:0/task:0/device:CPU:0") "tf.ReadVariableOp"([[o]]) {dtype = f32} : 1 +// CHECK-NEXT: [[out_ch:%.*]] = tfrt.merge.chains [[o_chain]], [[o_chain_0]] +// CHECK-NEXT: tfrt.return [[out_ch]], [[o1]] : !tfrt.chain, !tfrt_fallback.tf_tensor +module attributes {tf.devices = ["/job:localhost/replica:0/task:0/device:CPU:0"], tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 679 : i32}} { + func.func @__inference_pruned_131() -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "variable", outputs = "identity_retval_RetVal"}} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.VariableV2"() {container = "", device = "/job:localhost/replica:0/task:0/device:CPU:0", shape = #tf_type.shape<>, shared_name = "v_load_44"} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Identity"(%outputs) {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (tensor) -> tensor<*xf32> + tf_executor.fetch %outputs_0 : tensor<*xf32> + } + func.return %0 : tensor<*xf32> + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/whileop.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/whileop.mlir new file mode 100644 index 0000000000000000000000000000000000000000..321303d2225fb6fc3b380bd8ff51f3cf27c9d3e4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tf_to_corert/whileop.mlir @@ -0,0 +1,33 @@ +// RUN: tf-tfrt-opt -tf-executor-to-tfrt-pipeline="enable-optimizer=true tfrt-cost-threshold=1024" %s | FileCheck %s --dump-input=fail + +// Check that unused While op results and the associated ops are removed. + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 462 : i32}} { + func.func @while_cond_lt9(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<9> : tensor} : () -> tensor + %1 = "tf.Less"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1 : tensor + } + + func.func @while_body_add2(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %0 = "tf.Const"() {device = "/device:CPU:0", value = dense<1> : tensor} : () -> tensor + %1 = "tf.AddV2"(%arg0, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + %2 = "tf.Div"(%arg1, %0) {device = "/device:CPU:0"} : (tensor, tensor) -> tensor + func.return %1, %2 : tensor, tensor + } + + // CHECK-LABEL: func @while_test_remove_unused_results + // CHECK: [[pred:%.*]] = tfrt_fallback_async.predicate + // CHECK-NEXT: tfrt.while [[pred]] @"[[while_func_prefix:.*]]/tfrt_body_1" + // CHECK-SAME: (!tfrt.chain, !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) + // CHECK-NOT: func.call + func.func @while_test_remove_unused_results(%arg0: tensor, %arg1: tensor) -> (tensor, tensor) { + %0:2 = "tf.While"(%arg0, %arg1) { cond = @while_cond_lt9, body = @while_body_add2, is_stateless = false, parallel_iterations = 1} : (tensor, tensor) -> (tensor, tensor) + %1:2 = func.call @while_body_add2(%arg0, %arg1) : (tensor, tensor) -> (tensor, tensor) + func.return %0#0, %1#0 : tensor, tensor + } + + // CHECK: func @"[[while_func_prefix]]/tfrt_body_1" + // CHECK: "tf.AddV2" + // CHECK-NOT: "tf.Div" +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..d1301b69c24d7b62a0d42fd7d81b6f5a37203927 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/BUILD @@ -0,0 +1,142 @@ +load("@tf_runtime//tools:mlir_to_bef.bzl", "glob_tfrt_lit_tests", "mlir_to_bef") +load("//tensorflow:tensorflow.bzl", "tf_cc_shared_test") +# copybara:uncomment load("//third_party/tf_runtime_google/cpp_tests:gen_tests.bzl", "tfrt_cc_test_and_strict_benchmark") + +# copybara:uncomment package(default_applicable_licenses = ["//tensorflow:license"]) + +# Bundle together all of the test utilities that are used by tests. +filegroup( + name = "test_utilities", + testonly = True, + srcs = [ + "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_translate", + "//tensorflow/core/runtime_fallback:tf_bef_executor", + "//tensorflow/core/runtime_fallback/util:fallback_test_util", + "@llvm-project//llvm:FileCheck", + "@llvm-project//llvm:not", + "@llvm-project//mlir:run_lit.sh", + "@tf_runtime//tools:tfrt_translate", + ], +) + +# copybara:uncomment_begin(TFRT lit issue b/290857552) +# glob_tfrt_lit_tests( +# data = [":test_utilities"], +# # Custom driver is unsupported in OSS. Fails if one is provided. +# # copybara:uncomment driver = "//tensorflow/compiler/mlir:run_lit.sh", +# exclude = [ +# "compile.benchmark.large.mlir", +# "batch_function_fallback.mlir", +# "create_op.mlir", +# "custom_thread_pool.mlir", +# ], +# # copybara:uncomment flaky = ["compile.error.mlir"], +# size_override = { +# "compile.benchmark.small.mlir": "medium", +# "batching_fallback.mlir": "medium", +# }, +# tags_override = { +# "async_op_thread.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "batching_fallback.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "compile.benchmark.small.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "convert_tensorhandle_to_fallback_tensor.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "fallback.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "fallback_tensor_conversion_host.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "kernel_fallback_op_handler.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "mnist.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "runtime_fallback_op_handler.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "tf_delegate.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "tf_ops.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "tf_ops_error.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# "tfrt_forwarding.mlir": ["nomsan"], # Can't instrument code in precompiled lib (cuDNN) +# }, +# tfrt_translate = "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_translate", +# ) +# copybara:uncomment_end + +mlir_to_bef( + name = "batch_function_fallback.mlir", + tfrt_translate = "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_translate", +) + +mlir_to_bef( + name = "create_op.mlir", + tfrt_translate = "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_translate", +) + +mlir_to_bef( + name = "custom_thread_pool.mlir", + tfrt_translate = "//tensorflow/compiler/mlir/tfrt:tfrt_fallback_translate", +) + +# copybara:uncomment_begin(internal benchmarking) +# # C++ benchmarks for batch function runtime fallback. +# tfrt_cc_test_and_strict_benchmark( +# name = "batch_function_fallback_benchmark_test", +# srcs = ["batch_function_fallback_benchmark_test.cc"], +# data = ["batch_function_fallback.mlir.bef"], +# enable_xprof = True, +# includes = ["third_party/tf_runtime/include"], +# owners = ["tf-runtime-testing"], +# tags = [ +# "need_main", +# "no_gpu", +# ], +# deps = [ +# "//devtools/build/runtime:get_runfiles_dir", +# "@com_google_absl//absl/log:check", +# "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", +# "//tensorflow/core/platform:env", +# "//tensorflow/core/platform:resource_loader", +# "//tensorflow/core/platform:status", +# "//tensorflow/core/runtime_fallback/kernel:kernel_fallback_op_handler", +# "//tensorflow/core/runtime_fallback/kernel:kernel_fallback_tensor", +# "//tensorflow/core/runtime_fallback/runtime:runtime_fallback_alwayslink", +# "//tensorflow/core/runtime_fallback/util:fallback_test_util", +# "//tensorflow/core/runtime_fallback/util:tensor_util", +# "//tensorflow/core/tfrt/utils:fallback_tensor", +# "@eigen_archive//:eigen3", +# "@tf_runtime//:bef", +# "@tf_runtime//:befexecutor", +# "@tf_runtime//:core_runtime_alwayslink", +# "@tf_runtime//:hostcontext_alwayslink", +# "@tf_runtime//:mlirtobef", +# "@tf_runtime//:support", +# "@tf_runtime//:tensor", +# "@tf_runtime//backends/cpu:core_runtime_alwayslink", +# "@tf_runtime//backends/cpu:test_ops_alwayslink", +# ], +# ) +# copybara:uncomment_end + +tf_cc_shared_test( + name = "kernel_fallback_compat_test", + srcs = ["kernel_fallback_compat_test.cc"], + data = [ + "create_op.mlir.bef", + "custom_thread_pool.mlir.bef", + ], + tags = ["no_oss"], + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir:tfrt_fallback_async_opdefs", + "//tensorflow/core:all_kernels", + "//tensorflow/core:lib", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/runtime_fallback/kernel:kernel_fallback_compat_request_state", + "//tensorflow/core/runtime_fallback/runtime:runtime_fallback_alwayslink", + "//tensorflow/core/runtime_fallback/util:fallback_test_util", + "//tensorflow/core/tfrt/fallback:op_kernel_runner", + "//tensorflow/core/tfrt/runtime", + "//tensorflow/core/tfrt/utils:thread_pool", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + "@tf_runtime//:bef", + "@tf_runtime//:befexecutor", + "@tf_runtime//:core_runtime", + "@tf_runtime//:hostcontext", + "@tf_runtime//:init_tfrt_dialects", + "@tf_runtime//:support", + "@tf_runtime//:tracing", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/async_op_thread.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/async_op_thread.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f63c65f6209596d25a9ccef4261d4859beac8fac --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/async_op_thread.mlir @@ -0,0 +1,30 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd 2>&1 | FileCheck %s + +func.func @test_async_op_kernel_thread() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("/CPU:0") "tf.Const"() + { dtype = i32, value = dense<[2]> : tensor<1xi32> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("/CPU:0") + "tf.TestAsyncTfrtAsyncThread"() {T = i32} num_args(1) + %ch3 = tfrt_fallback_async.createop(%ch2) key(2) device("/CPU:0") + "tf.TestPrintThreadName"() num_args(0) + + %ch4, %0 = tfrt_fallback_async.executeop.seq(%ch3) key(0) cost(100) + device("/CPU:0") "tf.Const"() + { dtype = i32, value = dense<[2]> : tensor<1xi32> } : 1 + // Given TestAsyncTfrtAsyncThread will invoke done callback in thread with + // name "test_thread_in_compute_async", + // CHECK: TestAsyncTfrtAsyncThread thread name: test_thread_in_compute_async + %ch5, %1 = tfrt_fallback_async.executeop.seq(%ch4) key(1) cost(100) + device("/CPU:0") "tf.TestAsyncTfrtAsyncThread"(%0) {T = i32} : 1 + // ... when TestPrintThreadName is part of the next op in sequence of + // TestAsyncTfrtAsyncThread op, + // ... then TestPrintThreadName should not run in the thread in + // TestAsyncTfrtAsyncThread. + // CHECK-NOT: TestPrintThreadName thread name: test_thread_in_compute_async + %ch6 = tfrt_fallback_async.executeop.seq(%ch5) key(2) cost(100) + device("/CPU:0") "tf.TestPrintThreadName"() : 0 + + tfrt.return %ch6: !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e06e3da0950008296ba57e055a4cf91c9b0eda4e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback.mlir @@ -0,0 +1,160 @@ +func.func @f(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor, %arg2: !tfrt_fallback.tf_tensor, %arg3: !tfrt_fallback.tf_tensor, %arg4: !tfrt_fallback.tf_tensor, + %arg5: !tfrt_fallback.tf_tensor, %arg6: !tfrt_fallback.tf_tensor, %arg7: !tfrt_fallback.tf_tensor, %arg8: !tfrt_fallback.tf_tensor, %arg9: !tfrt_fallback.tf_tensor, + %arg10: !tfrt_fallback.tf_tensor, %arg11: !tfrt_fallback.tf_tensor, %arg12: !tfrt_fallback.tf_tensor, %arg13: !tfrt_fallback.tf_tensor, %arg14: !tfrt_fallback.tf_tensor, + %arg15: !tfrt_fallback.tf_tensor, %arg16: !tfrt_fallback.tf_tensor, %arg17: !tfrt_fallback.tf_tensor, %arg18: !tfrt_fallback.tf_tensor, %arg19: !tfrt_fallback.tf_tensor, + %arg20: !tfrt_fallback.tf_tensor, %arg21: !tfrt_fallback.tf_tensor, %arg22: !tfrt_fallback.tf_tensor, %arg23: !tfrt_fallback.tf_tensor, %arg24: !tfrt_fallback.tf_tensor, + %arg25: !tfrt_fallback.tf_tensor, %arg26: !tfrt_fallback.tf_tensor, %arg27: !tfrt_fallback.tf_tensor, %arg28: !tfrt_fallback.tf_tensor, %arg29: !tfrt_fallback.tf_tensor, + %arg30: !tfrt_fallback.tf_tensor, %arg31: !tfrt_fallback.tf_tensor, %arg32: !tfrt_fallback.tf_tensor, %arg33: !tfrt_fallback.tf_tensor, %arg34: !tfrt_fallback.tf_tensor, + %arg35: !tfrt_fallback.tf_tensor, %arg36: !tfrt_fallback.tf_tensor, %arg37: !tfrt_fallback.tf_tensor, %arg38: !tfrt_fallback.tf_tensor, %arg39: !tfrt_fallback.tf_tensor, + %arg40: !tfrt_fallback.tf_tensor, %arg41: !tfrt_fallback.tf_tensor, %arg42: !tfrt_fallback.tf_tensor, %arg43: !tfrt_fallback.tf_tensor, %arg44: !tfrt_fallback.tf_tensor, + %arg45: !tfrt_fallback.tf_tensor, %arg46: !tfrt_fallback.tf_tensor, %arg47: !tfrt_fallback.tf_tensor, %arg48: !tfrt_fallback.tf_tensor, %arg49: !tfrt_fallback.tf_tensor, + %arg50: !tfrt_fallback.tf_tensor, %arg51: !tfrt_fallback.tf_tensor, %arg52: !tfrt_fallback.tf_tensor, %arg53: !tfrt_fallback.tf_tensor, %arg54: !tfrt_fallback.tf_tensor, + %arg55: !tfrt_fallback.tf_tensor, %arg56: !tfrt_fallback.tf_tensor, %arg57: !tfrt_fallback.tf_tensor, %arg58: !tfrt_fallback.tf_tensor, %arg59: !tfrt_fallback.tf_tensor, + %arg60: !tfrt_fallback.tf_tensor, %arg61: !tfrt_fallback.tf_tensor, %arg62: !tfrt_fallback.tf_tensor, %arg63: !tfrt_fallback.tf_tensor, %arg64: !tfrt_fallback.tf_tensor, + %arg65: !tfrt_fallback.tf_tensor, %arg66: !tfrt_fallback.tf_tensor, %arg67: !tfrt_fallback.tf_tensor, %arg68: !tfrt_fallback.tf_tensor, %arg69: !tfrt_fallback.tf_tensor, + %arg70: !tfrt_fallback.tf_tensor, %arg71: !tfrt_fallback.tf_tensor, %arg72: !tfrt_fallback.tf_tensor, %arg73: !tfrt_fallback.tf_tensor, %arg74: !tfrt_fallback.tf_tensor, + %arg75: !tfrt_fallback.tf_tensor, %arg76: !tfrt_fallback.tf_tensor, %arg77: !tfrt_fallback.tf_tensor, %arg78: !tfrt_fallback.tf_tensor, %arg79: !tfrt_fallback.tf_tensor, + %arg80: !tfrt_fallback.tf_tensor, %arg81: !tfrt_fallback.tf_tensor, %arg82: !tfrt_fallback.tf_tensor, %arg83: !tfrt_fallback.tf_tensor, %arg84: !tfrt_fallback.tf_tensor, + %arg85: !tfrt_fallback.tf_tensor, %arg86: !tfrt_fallback.tf_tensor, %arg87: !tfrt_fallback.tf_tensor, %arg88: !tfrt_fallback.tf_tensor, %arg89: !tfrt_fallback.tf_tensor, + %arg90: !tfrt_fallback.tf_tensor, %arg91: !tfrt_fallback.tf_tensor, %arg92: !tfrt_fallback.tf_tensor, %arg93: !tfrt_fallback.tf_tensor, %arg94: !tfrt_fallback.tf_tensor, + %arg95: !tfrt_fallback.tf_tensor, %arg96: !tfrt_fallback.tf_tensor, %arg97: !tfrt_fallback.tf_tensor, %arg98: !tfrt_fallback.tf_tensor, %arg99: !tfrt_fallback.tf_tensor, + %arg100: !tfrt_fallback.tf_tensor, %arg101: !tfrt_fallback.tf_tensor, %arg102: !tfrt_fallback.tf_tensor, %arg103: !tfrt_fallback.tf_tensor, %arg104: !tfrt_fallback.tf_tensor, + %arg105: !tfrt_fallback.tf_tensor, %arg106: !tfrt_fallback.tf_tensor, %arg107: !tfrt_fallback.tf_tensor, %arg108: !tfrt_fallback.tf_tensor, %arg109: !tfrt_fallback.tf_tensor, + %arg110: !tfrt_fallback.tf_tensor, %arg111: !tfrt_fallback.tf_tensor, %arg112: !tfrt_fallback.tf_tensor) + -> (!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + tfrt.return %arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7, %arg8, %arg9, %arg10,%arg11, %arg12, %arg13, %arg14, + %arg15, %arg16, %arg17, %arg18, %arg19, %arg20, %arg21, %arg22, %arg23, %arg24, %arg25, %arg26, %arg27, %arg28, %arg29, + %arg30, %arg31, %arg32, %arg33, %arg34, %arg35, %arg36, %arg37, %arg38, %arg39, %arg40, %arg41, %arg42, %arg43, %arg44, + %arg45, %arg46, %arg47, %arg48, %arg49, %arg50, %arg51, %arg52, %arg53, %arg54, %arg55, %arg56, %arg57, %arg58, %arg59, + %arg60, %arg61, %arg62, %arg63, %arg64, %arg65, %arg66, %arg67, %arg68, %arg69, %arg70, %arg71, %arg72, %arg73, %arg74, + %arg75, %arg76, %arg77, %arg78, %arg79, %arg80, %arg81, %arg82, %arg83, %arg84, %arg85, %arg86, %arg87, %arg88, %arg89, + %arg90, %arg91, %arg92, %arg93, %arg94, %arg95, %arg96, %arg97, %arg98, %arg99, %arg100, %arg101, %arg102, %arg103, %arg104, + %arg105, %arg106, %arg107, %arg108, %arg109, %arg110, %arg111, %arg112 + :!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +func.func @main(%arg0: !tfrt.chain, %arg1: !tfrt_fallback.tf_tensor, %arg2: !tfrt_fallback.tf_tensor, %arg3: !tfrt_fallback.tf_tensor, %arg4: !tfrt_fallback.tf_tensor, + %arg5: !tfrt_fallback.tf_tensor, %arg6: !tfrt_fallback.tf_tensor, %arg7: !tfrt_fallback.tf_tensor, %arg8: !tfrt_fallback.tf_tensor, %arg9: !tfrt_fallback.tf_tensor, + %arg10: !tfrt_fallback.tf_tensor, %arg11: !tfrt_fallback.tf_tensor, %arg12: !tfrt_fallback.tf_tensor, %arg13: !tfrt_fallback.tf_tensor, %arg14: !tfrt_fallback.tf_tensor, + %arg15: !tfrt_fallback.tf_tensor, %arg16: !tfrt_fallback.tf_tensor, %arg17: !tfrt_fallback.tf_tensor, %arg18: !tfrt_fallback.tf_tensor, %arg19: !tfrt_fallback.tf_tensor, + %arg20: !tfrt_fallback.tf_tensor, %arg21: !tfrt_fallback.tf_tensor, %arg22: !tfrt_fallback.tf_tensor, %arg23: !tfrt_fallback.tf_tensor, %arg24: !tfrt_fallback.tf_tensor, + %arg25: !tfrt_fallback.tf_tensor, %arg26: !tfrt_fallback.tf_tensor, %arg27: !tfrt_fallback.tf_tensor, %arg28: !tfrt_fallback.tf_tensor, %arg29: !tfrt_fallback.tf_tensor, + %arg30: !tfrt_fallback.tf_tensor, %arg31: !tfrt_fallback.tf_tensor, %arg32: !tfrt_fallback.tf_tensor, %arg33: !tfrt_fallback.tf_tensor, %arg34: !tfrt_fallback.tf_tensor, + %arg35: !tfrt_fallback.tf_tensor, %arg36: !tfrt_fallback.tf_tensor, %arg37: !tfrt_fallback.tf_tensor, %arg38: !tfrt_fallback.tf_tensor, %arg39: !tfrt_fallback.tf_tensor, + %arg40: !tfrt_fallback.tf_tensor, %arg41: !tfrt_fallback.tf_tensor, %arg42: !tfrt_fallback.tf_tensor, %arg43: !tfrt_fallback.tf_tensor, %arg44: !tfrt_fallback.tf_tensor, + %arg45: !tfrt_fallback.tf_tensor, %arg46: !tfrt_fallback.tf_tensor, %arg47: !tfrt_fallback.tf_tensor, %arg48: !tfrt_fallback.tf_tensor, %arg49: !tfrt_fallback.tf_tensor, + %arg50: !tfrt_fallback.tf_tensor, %arg51: !tfrt_fallback.tf_tensor, %arg52: !tfrt_fallback.tf_tensor, %arg53: !tfrt_fallback.tf_tensor, %arg54: !tfrt_fallback.tf_tensor, + %arg55: !tfrt_fallback.tf_tensor, %arg56: !tfrt_fallback.tf_tensor, %arg57: !tfrt_fallback.tf_tensor, %arg58: !tfrt_fallback.tf_tensor, %arg59: !tfrt_fallback.tf_tensor, + %arg60: !tfrt_fallback.tf_tensor, %arg61: !tfrt_fallback.tf_tensor, %arg62: !tfrt_fallback.tf_tensor, %arg63: !tfrt_fallback.tf_tensor, %arg64: !tfrt_fallback.tf_tensor, + %arg65: !tfrt_fallback.tf_tensor, %arg66: !tfrt_fallback.tf_tensor, %arg67: !tfrt_fallback.tf_tensor, %arg68: !tfrt_fallback.tf_tensor, %arg69: !tfrt_fallback.tf_tensor, + %arg70: !tfrt_fallback.tf_tensor, %arg71: !tfrt_fallback.tf_tensor, %arg72: !tfrt_fallback.tf_tensor, %arg73: !tfrt_fallback.tf_tensor, %arg74: !tfrt_fallback.tf_tensor, + %arg75: !tfrt_fallback.tf_tensor, %arg76: !tfrt_fallback.tf_tensor, %arg77: !tfrt_fallback.tf_tensor, %arg78: !tfrt_fallback.tf_tensor, %arg79: !tfrt_fallback.tf_tensor, + %arg80: !tfrt_fallback.tf_tensor, %arg81: !tfrt_fallback.tf_tensor, %arg82: !tfrt_fallback.tf_tensor, %arg83: !tfrt_fallback.tf_tensor, %arg84: !tfrt_fallback.tf_tensor, + %arg85: !tfrt_fallback.tf_tensor, %arg86: !tfrt_fallback.tf_tensor, %arg87: !tfrt_fallback.tf_tensor, %arg88: !tfrt_fallback.tf_tensor, %arg89: !tfrt_fallback.tf_tensor, + %arg90: !tfrt_fallback.tf_tensor, %arg91: !tfrt_fallback.tf_tensor, %arg92: !tfrt_fallback.tf_tensor, %arg93: !tfrt_fallback.tf_tensor, %arg94: !tfrt_fallback.tf_tensor, + %arg95: !tfrt_fallback.tf_tensor, %arg96: !tfrt_fallback.tf_tensor, %arg97: !tfrt_fallback.tf_tensor, %arg98: !tfrt_fallback.tf_tensor, %arg99: !tfrt_fallback.tf_tensor, + %arg100: !tfrt_fallback.tf_tensor, %arg101: !tfrt_fallback.tf_tensor, %arg102: !tfrt_fallback.tf_tensor, %arg103: !tfrt_fallback.tf_tensor, %arg104: !tfrt_fallback.tf_tensor, + %arg105: !tfrt_fallback.tf_tensor, %arg106: !tfrt_fallback.tf_tensor, %arg107: !tfrt_fallback.tf_tensor, %arg108: !tfrt_fallback.tf_tensor, %arg109: !tfrt_fallback.tf_tensor, + %arg110: !tfrt_fallback.tf_tensor, %arg111: !tfrt_fallback.tf_tensor, %arg112: !tfrt_fallback.tf_tensor) + -> (!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + + %res:112 = tfrt_fallback_async.batch_function device("/device:CPU:0") @f (%arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7, %arg8, %arg9, %arg10,%arg11, %arg12, %arg13, %arg14, + %arg15, %arg16, %arg17, %arg18, %arg19, %arg20, %arg21, %arg22, %arg23, %arg24, %arg25, %arg26, %arg27, %arg28, %arg29, + %arg30, %arg31, %arg32, %arg33, %arg34, %arg35, %arg36, %arg37, %arg38, %arg39, %arg40, %arg41, %arg42, %arg43, %arg44, + %arg45, %arg46, %arg47, %arg48, %arg49, %arg50, %arg51, %arg52, %arg53, %arg54, %arg55, %arg56, %arg57, %arg58, %arg59, + %arg60, %arg61, %arg62, %arg63, %arg64, %arg65, %arg66, %arg67, %arg68, %arg69, %arg70, %arg71, %arg72, %arg73, %arg74, + %arg75, %arg76, %arg77, %arg78, %arg79, %arg80, %arg81, %arg82, %arg83, %arg84, %arg85, %arg86, %arg87, %arg88, %arg89, + %arg90, %arg91, %arg92, %arg93, %arg94, %arg95, %arg96, %arg97, %arg98, %arg99, %arg100, %arg101, %arg102, %arg103, %arg104, + %arg105, %arg106, %arg107, %arg108, %arg109, %arg110, %arg111, %arg112) { + num_batch_threads = 16, + max_batch_size = 1, + allowed_batch_sizes = [1], + batch_timeout_micros = 0, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32, i32, i32, i32, i32], + Tcaptured = [i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32], + Tout = [i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, + i32, i32, i32, i32, i32, i32, i32]} : 112 + + tfrt.return %arg0, %res#0, %res#1, %res#2, %res#3, %res#4, %res#5, %res#6, %res#7, %res#8, %res#9, %res#10, %res#11, %res#12, %res#13, %res#14, + %res#15, %res#16, %res#17, %res#18, %res#19, %res#20, %res#21, %res#22, %res#23, %res#24, %res#25, %res#26, %res#27, %res#28, %res#29, + %res#30, %res#31, %res#32, %res#33, %res#34, %res#35, %res#36, %res#37, %res#38, %res#39, %res#40, %res#41, %res#42, %res#43, %res#44, + %res#45, %res#46, %res#47, %res#48, %res#49, %res#50, %res#51, %res#52, %res#53, %res#54, %res#55, %res#56, %res#57, %res#58, %res#59, + %res#60, %res#61, %res#62, %res#63, %res#64, %res#65, %res#66, %res#67, %res#68, %res#69, %res#70, %res#71, %res#72, %res#73, %res#74, + %res#75, %res#76, %res#77, %res#78, %res#79, %res#80, %res#81, %res#82, %res#83, %res#84, %res#85, %res#86, %res#87, %res#88, %res#89, + %res#90, %res#91, %res#92, %res#93, %res#94, %res#95, %res#96, %res#97, %res#98, %res#99, %res#100, %res#101, %res#102, %res#103, %res#104, + %res#105, %res#106, %res#107, %res#108, %res#109, %res#110, %res#111 + :!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, + !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback_benchmark_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback_benchmark_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..e68fd67937f3f1bb9d9aa5c7dbf4d34773eebc70 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batch_function_fallback_benchmark_test.cc @@ -0,0 +1,157 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "base/logging.h" +#include "testing/base/public/benchmark.h" +#include +#include "absl/log/check.h" +#include "unsupported/Eigen/CXX11/Tensor" // from @eigen_archive +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/runtime_fallback/kernel/kernel_fallback_op_handler.h" +#include "tensorflow/core/runtime_fallback/util/fallback_test_util.h" +#include "tensorflow/core/tfrt/utils/fallback_tensor.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_executor/bef_file.h" // from @tf_runtime +#include "tfrt/core_runtime/core_runtime.h" // from @tf_runtime +#include "tfrt/host_context/async_value.h" // from @tf_runtime +#include "tfrt/host_context/async_value_ref.h" // from @tf_runtime +#include "tfrt/host_context/chain.h" // from @tf_runtime +#include "tfrt/host_context/concurrent_work_queue.h" // from @tf_runtime +#include "tfrt/host_context/execution_context.h" // from @tf_runtime +#include "tfrt/host_context/function.h" // from @tf_runtime +#include "tfrt/host_context/host_allocator.h" // from @tf_runtime +#include "tfrt/host_context/host_context.h" // from @tf_runtime +#include "tfrt/host_context/resource_context.h" // from @tf_runtime +#include "tfrt/support/forward_decls.h" // from @tf_runtime +#include "tfrt/support/rc_array.h" // from @tf_runtime +#include "tfrt/tensor/dense_host_tensor.h" // from @tf_runtime +#include "tfrt/tensor/tensor.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +// Creates a BEF file with a program that runs +// tfrt_fallback_async.batch_function with a empty function forwarding inputs or +// outputs. +std::pair> CreateBefFile( + tfrt::HostContext* host) { + std::string file_path = GetDataDependencyFilepath( + "tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/" + "batch_function_fallback.mlir.bef"); + + std::string data; + CHECK_OK(ReadFileToString(Env::Default(), file_path, &data)); + + tfrt::BefBuffer bef_buffer(data.begin(), data.end()); + + auto bef_file = tfrt::BEFFile::Open(bef_buffer, host->GetKernelRegistry(), + host->diag_handler(), host->allocator()); + CHECK(bef_file); + return std::make_pair(std::move(bef_buffer), std::move(bef_file)); +} + +std::unique_ptr CreateTestCoreRuntime() { + auto corert = tfrt::CoreRuntime::Create( + /*diag_handler=*/[](const tfrt::DecodedDiagnostic& + diag) { LOG(ERROR) << diag.message(); }, + tfrt::CreateMallocAllocator(), + tfrt::CreateMultiThreadedWorkQueue(16, 16)); + CHECK(corert); + auto fallback_op_handler = tensorflow::tfd::CreateKernelFallbackOpHandler( + corert->get(), corert->get()->GetHostContext()->GetHostDeviceRef()); + CHECK(fallback_op_handler); + corert.get()->RegisterOpHandler("tfkernel", fallback_op_handler.get()); + return std::move(corert.get()); +} + +tfrt::RCArray CreateTestArguments(const tfrt::Function* func, + tfrt::HostContext* host) { + Tensor tensor(DataType::DT_INT32, TensorShape({1})); + std::vector> arguments; + arguments.reserve(func->argument_types().size()); + arguments.push_back(tfrt::GetReadyChain()); + for (int i = 1, e = func->argument_types().size(); i < e; ++i) { + arguments.push_back( + tfrt::MakeAvailableAsyncValueRef(tensor)); + } + + return tfrt::RCArray(arguments); +} + +TEST(BatchFunctionTest, Basic) { + auto corert = CreateTestCoreRuntime(); + auto* host = corert->GetHostContext(); + auto [bef_buffer, bef_file] = CreateBefFile(host); + auto* func = bef_file->GetFunction("main"); + CHECK(func); + CHECK_EQ(func->result_types().size(), 113); + CHECK_EQ(func->argument_types().size(), 113); + + auto arguments = CreateTestArguments(func, host); + + tfrt::ResourceContext resource_ctx; + auto exec_ctx = tfd::CreateFallbackTestExecutionContext(host, &resource_ctx); + + std::vector> results; + results.resize(func->result_types().size()); + std::vector> result_tensors; + result_tensors.resize(func->result_types().size() - 1); + + func->Execute(exec_ctx, arguments.values(), results); + host->Await(results); + + for (auto& result : results) { + EXPECT_FALSE(result->IsError()); + } +} + +// Runs a BEF function that batches a function that does nothing just to measure +// the runtime overhead. The BEF function signature is adapted from a real model +// and is useful for benchmarking ops with large attributes and many +// input/output. +void BM_BatchFunctionFallbackWithLargeAttributesAndManyInputsOutputs( + benchmark::State& state) { + auto corert = CreateTestCoreRuntime(); + auto* host = corert->GetHostContext(); + auto [bef_buffer, bef_file] = CreateBefFile(host); + auto* func = bef_file->GetFunction("main"); + CHECK(func); + CHECK_EQ(func->result_types().size(), 113); + CHECK_EQ(func->argument_types().size(), 113); + + auto arguments = CreateTestArguments(func, host); + + tfrt::ResourceContext resource_ctx; + auto exec_ctx = tfd::CreateFallbackTestExecutionContext(host, &resource_ctx); + + std::vector> results; + results.resize(func->result_types().size()); + + for (auto _ : state) { + func->Execute(exec_ctx, arguments.values(), results); + host->Await(results); + results.clear(); + results.resize(func->result_types().size()); + } +} + +BENCHMARK(BM_BatchFunctionFallbackWithLargeAttributesAndManyInputsOutputs); + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batching_fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batching_fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..457966ff75eb072ac678258beac1cf9d789b6ed4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/batching_fallback.mlir @@ -0,0 +1,210 @@ +// RUN: tfrt_fallback_translate --mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd:1,1 | FileCheck %s +// RUN: tfrt_fallback_translate --mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd:8 | FileCheck %s + +func.func @matmul_cpu(%ch: !tfrt.chain, %a: !tfrt_fallback.tf_tensor, %b: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor) { + // Enqueue a sleep onto blocking work queue, %ch0 is fulfilled when sleeping is done. + %us = tfrt.constant.i32 1000 + %ch0 = "tfrt_test.blocking.usleep"(%us) : (i32) -> (!tfrt.chain) + + %ch1 = tfrt.merge.chains %ch, %ch0 : !tfrt.chain, !tfrt.chain + + %ch2 = tfrt_fallback_async.createop(%ch1) key(0) device("/CPU:0") "tf.MatMul"() {T = i32} num_args(2) + + %ch3, %result = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("/CPU:0") "tf.MatMul"(%a, %b) {T = i32} : 1 + + %s = "tfrt_test.get_string"() { value = "Running @matmul_cpu" } : () -> !tfrt.string + %ch4 = "tfrt_test.print_string"(%s, %ch3) : (!tfrt.string, !tfrt.chain) -> (!tfrt.chain) + + %ch5 = "tfrt_fallback_async.print_tensor"(%result, %ch4) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch5, %result : !tfrt.chain, !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: --- Running 'batch_function_fallback_concat_test' +func.func @batch_function_fallback_concat_test() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a1 = tfrt_fallback_async.const_dense_tensor dense<[[2, 2], [2, 2]]> : tensor<2x2xi32> + %a2 = tfrt_fallback_async.const_dense_tensor dense<[[3, 3], [3, 3]]> : tensor<2x2xi32> + %b = tfrt_fallback_async.const_dense_tensor dense<[[1, 1], [1, 1]]> : tensor<2x2xi32> + + // Two batch_size=2 batches get concatenated. + %result_1 = tfrt_fallback_async.batch_function device("/device:CPU:0") @matmul_cpu (%a1, %b) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 1000000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [i32], + Tout = [i32]} : 1 + + %result_2 = tfrt_fallback_async.batch_function device("/device:CPU:0") @matmul_cpu (%a2, %b) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 1000000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [i32], + Tout = [i32]} : 1 + + // Since batch function kernel scheduling is async, the above 2 batches can arrive in any order. + // CHECK: Running @matmul_cpu + // CHECK-NEXT: Tensor + + // CHECK: Tensor !tfrt.chain + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch2 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'batch_function_fallback_timeout_test' +func.func @batch_function_fallback_timeout_test() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_dense_tensor dense<[[4, 4], [4, 4]]> : tensor<2x2xi32> + %b = tfrt_fallback_async.const_dense_tensor dense<[[1, 1], [1, 1]]> : tensor<2x2xi32> + + // One batch_size=2 batches get padded and processed after timeout. + %result = tfrt_fallback_async.batch_function device("/device:CPU:0") @matmul_cpu (%a, %b) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 1000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [i32], + Tout = [i32]} : 1 + + // CHECK: Running @matmul_cpu + // CHECK-NEXT: Tensor + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'batch_function_fallback_no_padding_test' +func.func @batch_function_fallback_no_padding_test() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_dense_tensor dense<[[4, 4], [4, 4]]> : tensor<2x2xi32> + %b = tfrt_fallback_async.const_dense_tensor dense<[[1, 1], [1, 1]]> : tensor<2x2xi32> + + // One batch_size=2 batches get processed after timeout. + %result = tfrt_fallback_async.batch_function device("/device:CPU:0") @matmul_cpu (%a, %b) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 1000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + disable_padding = true, + Tin = [i32], + Tcaptured = [i32], + Tout = [i32]} : 1 + + // CHECK: Running @matmul_cpu + // As no padding is appended, the tensor shape printed inside the batch function + // is [2,2] instead of [4,2] + // CHECK-NEXT: Tensor + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// This test is for testing support for Adaptive Batch Scheduler, and it is +// triggered by num_batch_threads<=0 +// CHECK-LABEL: --- Running 'batch_function_fallback_zero_batch_thread_test' +func.func @batch_function_fallback_zero_batch_thread_test() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_dense_tensor dense<[[2, 2], [2, 2]]> : tensor<2x2xi32> + %b = tfrt_fallback_async.const_dense_tensor dense<[[2, 2], [2, 2]]> : tensor<2x2xi32> + + // One batch_size=2 batches get padded and processed after timeout. + %result = tfrt_fallback_async.batch_function device("/device:CPU:0") @matmul_cpu (%a, %b) { + num_batch_threads = 0, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 1000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [i32], + Tout = [i32]} : 1 + + // CHECK: Running @matmul_cpu + // CHECK-NEXT: Tensor + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// Test to verify the fix for b/190394141: +// Given a function that returns multiple values referenced to the same value, +func.func @returns_multiple_refs(%ch: !tfrt.chain, %a: !tfrt_fallback.tf_tensor) -> (!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) { + tfrt.return %ch, %a, %a : !tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor +} + +// ... when the function returns_multiple_refs is invoked by the batch_function, +// ... then the code should not crash. +// CHECK-LABEL: Running 'test_batch_returns_multiple_refs' +func.func @test_batch_returns_multiple_refs() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %0 = tfrt_fallback_async.const_dense_tensor dense<[[1, 1], [1, 1]]> : tensor<2x2xi32> + + %1, %2 = tfrt_fallback_async.batch_function device("/device:CPU:0") @returns_multiple_refs (%0) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 10000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [], + Tout = [i32, i32]} : 2 + + %3, %4 = tfrt_fallback_async.batch_function device("/device:CPU:0") @returns_multiple_refs (%0) { + num_batch_threads = 1, + max_batch_size = 4, + allowed_batch_sizes = [4], + batch_timeout_micros = 10000, + container = "container", + shared_name = "shared_name", + batching_queue = "batching_queue", + enable_large_batch_splitting = false, + Tin = [i32], + Tcaptured = [], + Tout = [i32, i32]} : 2 + + + %ch1 = "tfrt_fallback_async.print_tensor"(%1, %ch0) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch2 = "tfrt_fallback_async.print_tensor"(%3, %ch1) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch2 : !tfrt.chain +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.large.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.large.mlir new file mode 100644 index 0000000000000000000000000000000000000000..db7d1ae926de1f28cc20ccde8994252801c29ca5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.large.mlir @@ -0,0 +1,48 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s \ +// RUN: | tf_bef_executor --test_init_function=register_op_handlers_kernel \ +// RUN: --host_allocator_type=malloc \ +// RUN: --work_queue_type=mstd:72 \ +// RUN: | FileCheck %s --dump-input=always + +// A set of benchmarks for large tensor inputs. These benchmarks measure +// codegen/runtime efficiency for executing parallel/concurrent code. Fallback +// kernels rely on Eigen for parallelizing compute operation. + +module @kernels attributes { tfrt.compiled } { + func.func @log1p(%arg0: tensor) -> tensor { + %0 = "tf.Log1p"(%arg0): (tensor) -> tensor + func.return %0 : tensor + } +} + +// CHECK: --- Running 'BM_fallback_log1p_f32' +func.func @BM_fallback_log1p_f32() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(1) device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<1.0> : tensor<1024x1024xf32> + } num_args(0) + + %ch2 = tfrt_fallback_async.createop(%ch1) key(2) device("/CPU:0") + "tf.Log1p"() { T = f32 } num_args(1) + + %ch3, %const = tfrt_fallback_async.executeop.seq(%ch2) key(1) cost(100) + device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<1.0> : tensor<1024x1024xf32> + } : 1 + + %ch4 = tfrt_test.benchmark "BM_fallback_log1p_f32"( + %const : !tfrt_fallback.tf_tensor, + %ch3 : !tfrt.chain + ) + duration_secs = 1, max_count = 10000, num_warmup_runs = 10 + { + %result = tfrt_fallback_async.executeop key(2) cost(100) device("/CPU:0") + "tf.Log1p"(%const) { T = f32 } : 1 + tfrt.return %result : !tfrt_fallback.tf_tensor + } + + tfrt.return %ch4 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.small.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.small.mlir new file mode 100644 index 0000000000000000000000000000000000000000..34931052a38c05946acd7f923038b4afce18addb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/compile.benchmark.small.mlir @@ -0,0 +1,91 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s \ +// RUN: | tf_bef_executor --test_init_function=register_op_handlers_kernel \ +// RUN: | FileCheck %s --dump-input=always + +// A set of benchmarks measuring runtime/compiler overheads for running +// kernels with small inputs. These benchmarks do not launch any concurrent +// tasks and do not use concurrent work queue or intra-op thread pool. + +module @kernels attributes { tfrt.compiled } { + func.func @rsqrt(%arg0: tensor) -> tensor { + %0 = "tf.Rsqrt"(%arg0): (tensor) -> tensor + func.return %0 : tensor + } + + func.func @rsqrt_tanh(%arg0: tensor) -> tensor { + %0 = "tf.Rsqrt"(%arg0): (tensor) -> tensor + %1 = "tf.Tanh"(%0): (tensor) -> tensor + func.return %1 : tensor + } +} + +// CHECK: --- Running 'BM_fallback_rsqrt_f32' +func.func @BM_fallback_rsqrt_f32() { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(1) device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + } num_args(0) + + %ch2 = tfrt_fallback_async.createop(%ch1) key(2) device("/CPU:0") + "tf.Rsqrt"() { T = f32 } num_args(1) + + %ch3, %const = tfrt_fallback_async.executeop.seq(%ch1) key(1) cost(100) + device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + } : 1 + + tfrt_test.benchmark "BM_fallback_rsqrt_f32"( + %const : !tfrt_fallback.tf_tensor, + %ch2 : !tfrt.chain, + %ch3 : !tfrt.chain + ) + duration_secs = 3, max_count = 100000, num_warmup_runs = 10 + { + %result = tfrt_fallback_async.executeop key(2) cost(100) device("/CPU:0") + "tf.Rsqrt"(%const) { T = f32 } : 1 + tfrt.return %result : !tfrt_fallback.tf_tensor + } + + tfrt.return +} + +// CHECK: --- Running 'BM_fallback_rsqrt_tanh_f32' +func.func @BM_fallback_rsqrt_tanh_f32() { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(3) device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + } num_args(0) + + %ch2 = tfrt_fallback_async.createop(%ch1) key(4) device("/CPU:0") + "tf.Rsqrt"() { T = f32 } num_args(1) + + %ch3 = tfrt_fallback_async.createop(%ch2) key(5) device("/CPU:0") + "tf.Tanh"() { T = f32 } num_args(1) + + %ch4, %const = tfrt_fallback_async.executeop.seq(%ch3) key(3) cost(100) + device("/CPU:0") + "tf.Const"() { + dtype = f32, value = dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + } : 1 + + tfrt_test.benchmark "BM_fallback_rsqrt_tanh_f32"( + %const : !tfrt_fallback.tf_tensor, + %ch4 : !tfrt.chain + ) + duration_secs = 3, max_count = 100000, num_warmup_runs = 10 + { + %result0 = tfrt_fallback_async.executeop key(4) cost(100) device("/CPU:0") + "tf.Rsqrt"(%const) { T = f32 } : 1 + %result1 = tfrt_fallback_async.executeop key(5) cost(100) device("/CPU:0") + "tf.Tanh"(%result0) { T = f32 } : 1 + tfrt.return %result1 : !tfrt_fallback.tf_tensor + } + + tfrt.return +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/convert_tensorhandle_to_fallback_tensor.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/convert_tensorhandle_to_fallback_tensor.mlir new file mode 100644 index 0000000000000000000000000000000000000000..129af017997a2f98bee5b52dd0d315fcc5b7e4ef --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/convert_tensorhandle_to_fallback_tensor.mlir @@ -0,0 +1,27 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd 2>&1 | FileCheck %s + +// This test is to verify the fix for the bug in converting tensorhandle to +// fallback tensor. The TensorHandle should not be moved to the output, +// otherwise, the input will be set to null and cannot be referenced again. +// This bug happens if the AsyncTensor is unavailable during the conversion. + +func.func @test_convert_tensorhandle_to_fallback_tensor() { + %us = tfrt.constant.i32 1000 + + // Given an available TensorHandle with an unavailable AsyncTensor, which will + // become available after 1000 ms. + // CHECK: Created TensorHandle (test string) + %1 = "tfrt_fallback_test.create_tensorhandle_with_delayed_async_tensor"(%us) : (i32) -> !corert.tensorhandle + + %2 = "tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor"(%1) {_tfrt_cost = 1 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:0"} : (!corert.tensorhandle) -> !tfrt_fallback.tf_tensor + + // ... when `tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor` is + // invoked twice with the same input, + // ... then the test should not crash. + %3 = "tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor"(%1) {_tfrt_cost = 1 : i64, device = "/job:localhost/replica:0/task:0/device:CPU:1"} : (!corert.tensorhandle) -> !tfrt_fallback.tf_tensor + + // CHECK: Slept for 1000 microseconds + // CHECK: Marked AsyncTensor available. + + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/create_op.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/create_op.mlir new file mode 100644 index 0000000000000000000000000000000000000000..86f0827c44f087596615b11cb8157f6e1ff3a0a9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/create_op.mlir @@ -0,0 +1,23 @@ +func.func @init(%in_ch: !tfrt.chain) -> !tfrt.chain { + %ch0 = "tfrt_fallback_async.createop"(%in_ch) + { + device = "cpu", + num_args = 2 : i64, + op_attrs = [["T", i32]], + op_func_attrs = [], + op_key = 0 : i64, + op_name = "tf.AddV2" + } : (!tfrt.chain) -> (!tfrt.chain) + %ch1 = "tfrt_fallback_async.createop"(%ch0) + { + device = "cpu", + num_args = 1 : i64, + op_attrs = [["Targuments", []], + ["output_shapes", [#corert.shape<>]], + ["output_types", [i64]]], + op_func_attrs = [["f", "dummy_fn"]], + op_key = 1 : i64, + op_name = "tf.FlatMapDataset" + } : (!tfrt.chain) -> (!tfrt.chain) + tfrt.return %ch1 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/custom_thread_pool.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/custom_thread_pool.mlir new file mode 100644 index 0000000000000000000000000000000000000000..ccdd4e0e9126c2c8adc1cd8a8b8da4f10b7a9e1d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/custom_thread_pool.mlir @@ -0,0 +1,19 @@ +func.func @init(%in_ch: !tfrt.chain) -> !tfrt.chain { + %ch0 = "tfrt_fallback_async.createop"(%in_ch) + { + device = "cpu", + num_args = 2 : i64, + op_attrs = [["T", i32]], + op_func_attrs = [], + op_key = 0 : i64, + op_name = "tf.AddV2" + } : (!tfrt.chain) -> (!tfrt.chain) + tfrt.return %ch0 : !tfrt.chain +} + +func.func @run(%in_ch: !tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor){ + %x_th= corert.const_dense_tensor dense<1> : tensor + %x = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %x_th {_tfrt_cost = 1 : i64, device = "cpu"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + %res = tfrt_fallback_async.executeop key(0) cost(100) device("cpu") "tf.AddV2"(%x, %x) { T = i32 } : 1 + tfrt.return %in_ch, %res : !tfrt.chain, !tfrt_fallback.tf_tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2a5354b8c37f2b7a7411a2fb2a5bbec94a73014d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback.mlir @@ -0,0 +1,454 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor --test_init_function=register_op_handlers_kernel | FileCheck %s + +func.func @register_op_handlers_kernel() { + %fallback = "corert.create_kernel_fallback_op_handler"() : () -> !corert.ophandler + %cpu = "corert.create_cpu_op_handler"(%fallback) : (!corert.ophandler) -> !corert.ophandler + %cpu_ordinal = tfrt.constant.i32 1 + %cpu1 = "corert.create_cpu_op_handler_with_ordinal"(%fallback, %cpu_ordinal) : (!corert.ophandler, i32) -> !corert.ophandler + corert.register_op_handler %cpu "cpu" + corert.register_op_handler %cpu1 "cpu" + corert.register_op_handler %fallback "tfkernel" + tfrt.return +} + +// CHECK-LABEL: --- Running 'test_tf_random_uniform' +func.func @test_tf_random_uniform() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("CPU:0") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("CPU:0") "tf.RandomUniform"() {dtype = f32, T = i64} num_args(1) + + // Test tf.RandomUniform. + %ch, %shape_th = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("CPU:0") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } : 1 + %random_uniform = tfrt_fallback_async.executeop key(1) cost(100) device("CPU:0") "tf.RandomUniform"(%shape_th) {dtype = f32, T = i64} : 1 + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch3 : !tfrt.chain +} + +func.func @test_tf_random_uniform_on_cpu1() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("CPU:1") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("CPU:1") "tf.RandomUniform"() {dtype = f32, T = i64} num_args(1) + + // Test tf.RandomUniform. + %ch, %shape_th = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("CPU:1") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } : 1 + %random_uniform = tfrt_fallback_async.executeop key(1) cost(100) device("CPU:1") "tf.RandomUniform"(%shape_th) {dtype = f32, T = i64} : 1 + + %res_th = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %random_uniform {_tfrt_cost = 1 : i64, device = "CPU:1"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + + // CHECK: float shape: [2,2] values: + %ch3 = "corert.print_tensorhandle"(%res_th, %ch) : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch3 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_parse_single_example' +func.func @test_parse_single_example() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + // %serialized is a tensorflow::Example proto whose content is + // features { + // feature { + // key: "key_0" + // value { + // int64_list { + // value: 100 + // } + // } + // } + // feature { + // key: "key_1" + // value { + // int64_list { + // value: 200 + // } + // } + // } + // } + // + // Note that in MLIR, non-printable bytes are printed as escaped hex numbers. + %serialized_th = corert.const_string_tensor {shape = [], value = ["\0A!\0A\0E\0A\05key_0\12\05\1A\03\0A\01d\0A\0F\0A\05key_1\12\06\1A\04\0A\02\C8\01"]} + %serialized = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %serialized_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle) -> (!tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(2) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor<1xi64>} num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(3) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor<1xi64>} num_args(0) + %ch3 = tfrt_fallback_async.createop(%ch2) key(4) device("CPU:0") "tf.ParseSingleExample"() + {Tdense = [i64, i64], dense_keys = ["key_0", "key_1"], num_sparse = 0 : i64, sparse_types = [], sparse_keys = [], dense_shapes = [#corert.shape, #corert.shape]} num_args(3) + + %ch4, %zero = tfrt_fallback_async.executeop.seq(%ch3) key(2) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor<1xi64>} : 1 + %ch5, %one = tfrt_fallback_async.executeop.seq(%ch3) key(3) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor<1xi64>} : 1 + %dense_values:2 = tfrt_fallback_async.executeop key(4) cost(100) device("CPU:0") "tf.ParseSingleExample"(%serialized, %zero, %one) + {Tdense = [i64, i64], dense_keys = ["key_0", "key_1"], num_sparse = 0 : i64, sparse_types = [], sparse_keys = [], dense_shapes = [#corert.shape, #corert.shape]} : 2 + + // CHECK: Tensor + %ch6 = "tfrt_fallback_async.print_tensor"(%dense_values#0, %ch5) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + // CHECK: Tensor + %ch7 = "tfrt_fallback_async.print_tensor"(%dense_values#1, %ch6) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch7 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_const_tensor_proto' +func.func @test_const_tensor_proto() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_tensor_proto "\08\0C\12\00\22\01@" + + // CHECK: Tensor + %ch1 = "tfrt_fallback_async.print_tensor"(%a, %ch0) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_const_dense_tensor' +func.func @test_const_dense_tensor() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_dense_tensor dense<[true, false]> : tensor<2xi1> {_tfrt_cost = 1 : i64} + + // CHECK: Tensor + %ch1 = "tfrt_fallback_async.print_tensor"(%a, %ch0) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_const_string_tensor' +func.func @test_const_string_tensor() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_string_tensor {shape = [2], value = ["const", "string"], _tfrt_cost = 1 : i64} + + // CHECK: Tensor + %ch1 = "tfrt_fallback_async.print_tensor"(%a, %ch0) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_const_string_tensor_same_value' +func.func @test_const_string_tensor_same_value() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a = tfrt_fallback_async.const_string_tensor {shape = [2], value = ["string"], _tfrt_cost = 1 : i64} + + // CHECK: Tensor + %ch1 = "tfrt_fallback_async.print_tensor"(%a, %ch0) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_predicate' +func.func @test_predicate() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %cpu = corert.get_op_handler %ch0 "cpu" + + %a = tfrt_fallback_async.const_dense_tensor dense<0.0> : tensor {_tfrt_cost = 1 : i64} + %ra = tfrt_fallback_async.predicate %a {_tfrt_cost = 1 : i64} + // CHECK: false + %ch1 = "tfrt_test.print_bool"(%ch0, %ra) : (!tfrt.chain, i1) -> !tfrt.chain + + %b = tfrt_fallback_async.const_dense_tensor dense<1> : tensor {_tfrt_cost = 1 : i64} + %rb = tfrt_fallback_async.predicate %b {_tfrt_cost = 1 : i64} + // CHECK: true + %ch2 = "tfrt_test.print_bool"(%ch1, %rb) : (!tfrt.chain, i1) -> !tfrt.chain + + %c = tfrt_fallback_async.const_string_tensor {shape = [], value = [""], _tfrt_cost = 1 : i64} + %rc = tfrt_fallback_async.predicate %c {_tfrt_cost = 1 : i64} + // CHECK: false + %ch3 = "tfrt_test.print_bool"(%ch2, %rc) : (!tfrt.chain, i1) -> !tfrt.chain + + %d = tfrt_fallback_async.const_string_tensor {shape = [], value = ["string"], _tfrt_cost = 1 : i64} + %rd = tfrt_fallback_async.predicate %d {_tfrt_cost = 1 : i64} + // CHECK: true + %ch4 = "tfrt_test.print_bool"(%ch3, %rd) : (!tfrt.chain, i1) -> !tfrt.chain + + %e = tfrt_fallback_async.const_dense_tensor dense<[]> : tensor<0xi32> {_tfrt_cost = 1 : i64} + %re = tfrt_fallback_async.predicate %e {_tfrt_cost = 1 : i64} + // CHECK: false + %ch5 = "tfrt_test.print_bool"(%ch4, %re) : (!tfrt.chain, i1) -> !tfrt.chain + + %f = tfrt_fallback_async.const_dense_tensor dense<[0]> : tensor<1xi32> {_tfrt_cost = 1 : i64} + %rf = tfrt_fallback_async.predicate %f {_tfrt_cost = 1 : i64} + // CHECK: true + %ch6 = "tfrt_test.print_bool"(%ch5, %rf) : (!tfrt.chain, i1) -> !tfrt.chain + + tfrt.return %ch6 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_fallback_resource' +func.func @test_fallback_resource() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a_th = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %b_th = corert.const_dense_tensor dense<[false, true]> : tensor<2xi1> + %ra, %rb = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a_th, %b_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.set_resource %ch0, %ra {device = "cpu", index = 0 : i64} + %ch2 = tfrt_fallback_async.set_resource %ch1, %rb {device = "cpu", index = 1 : i64} + %ch3, %b, %a = tfrt_fallback_async.get_resource %ch2 {_tfrt_cost = 1 : i64, device = "cpu", indices = [1 : i64, 0 : i64]} : (!tfrt.chain) -> (!tfrt.chain, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + // CHECK: Tensor + %ch4 = "tfrt_fallback_async.print_tensor"(%a, %ch3) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + // CHECK: Tensor + %ch5 = "tfrt_fallback_async.print_tensor"(%b, %ch4) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch5 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_i1_dtype' +func.func @test_i1_dtype() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %a_th = corert.const_dense_tensor dense<[true, false]> : tensor<2xi1> + %b_th = corert.const_dense_tensor dense<[false, true]> : tensor<2xi1> + %a, %b = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %a_th, %b_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(5) device("CPU:0") "tf.LogicalAnd"() num_args(2) + %ch2, %c = tfrt_fallback_async.executeop.seq(%ch1) key(5) cost(100) device("CPU:0") "tf.LogicalAnd"(%a, %b) : 1 + + // CHECK: Tensor + %ch3 = "tfrt_fallback_async.print_tensor"(%c, %ch2) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch3 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_step_container' +func.func @test_step_container() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %size_th = corert.const_dense_tensor dense<5> : tensor + %index_th = corert.const_dense_tensor dense<1> : tensor + %value_th = corert.const_dense_tensor dense<[1.0, 2.0, 3.0]> : tensor<3xf32> + + %size, %index, %value = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %size_th, %index_th, %value_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(6) device("CPU:0") "tf.TensorArrayV3"() {dtype = f32, element_shape = #corert.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} num_args(1) + %ch2 = tfrt_fallback_async.createop(%ch1) key(7) device("CPU:0") "tf.TensorArrayWriteV3"() {T = f32} num_args(4) + %ch3 = tfrt_fallback_async.createop(%ch2) key(8) device("CPU:0") "tf.TensorArrayReadV3"() {dtype = f32} num_args(3) + %ch4 = tfrt_fallback_async.createop(%ch3) key(9) device("CPU:0") "tf.TensorArrayCloseV3"() num_args(1) + + %ch5, %ta:2 = tfrt_fallback_async.executeop.seq(%ch4) key(6) cost(100) device("CPU:0") "tf.TensorArrayV3"(%size) {dtype = f32, element_shape = #corert.shape<3>, dynamic_size = false, clear_after_read = true, identical_element_shapes = true, tensor_array_name = "ta"} : 2 + %ch6, %write = tfrt_fallback_async.executeop.seq(%ch5) key(7) cost(100) device("CPU:0") "tf.TensorArrayWriteV3"(%ta#0, %index, %value, %ta#1) {T = f32} : 1 + %ch7, %read = tfrt_fallback_async.executeop.seq(%ch6) key(8) cost(100) device("CPU:0") "tf.TensorArrayReadV3"(%ta#0, %index, %write) {dtype = f32} : 1 + %ch8 = tfrt_fallback_async.executeop.seq(%ch7) key(9) cost(100) device("CPU:0") "tf.TensorArrayCloseV3"(%ta#0) + + // CHECK: Tensor + %ch9 = "tfrt_fallback_async.print_tensor"(%read, %ch8) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch9 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_select' +func.func @test_select() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %arg_th = corert.const_dense_tensor dense<[1, 2, 3, 4]> : tensor<4xi64> + %cond_th = corert.const_dense_tensor dense<[true, false, true, false]> : tensor<4xi1> + %arg, %cond = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %arg_th, %cond_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(10) device("CPU:0") "tf.ZerosLike"() {T = i64} num_args(1) + %ch2 = tfrt_fallback_async.createop(%ch1) key(11) device("CPU:0") "tf.Select"() {T = i64} num_args(3) + + %ch3, %zeros = tfrt_fallback_async.executeop.seq(%ch2) key(10) cost(100) device("CPU:0") "tf.ZerosLike"(%arg) {T = i64} : 1 + %0 = tfrt_fallback_async.executeop key(11) cost(100) device("CPU:0") "tf.Select"(%cond, %arg, %zeros) {T = i64} : 1 + + // CHECK: Tensor + %ch4 = "tfrt_fallback_async.print_tensor"(%0, %ch3) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +func.func @while_cond_false(%in: !tfrt.chain, %x: !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) { + %ch0 = tfrt_fallback_async.createop(%in) key(12) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} num_args(0) + %ch1, %zero = tfrt_fallback_async.executeop.seq(%ch0) key(12) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} : 1 + %zero_th = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %zero {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + + tfrt.return %ch1, %zero_th : !tfrt.chain, !corert.tensorhandle +} + +func.func @while_body_print(%in: !tfrt.chain, %x: !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) { + %ch0 = tfrt.new.chain + + %ch1 = "corert.print_tensorhandle"(%x, %ch0) : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %in, %x : !tfrt.chain, !corert.tensorhandle +} + +// While loop test +// CHECK-LABEL: --- Running 'while_fallback_condition' +func.func @while_fallback_condition() { + %ch0 = tfrt.new.chain + + %cpu = corert.get_op_handler %ch0 "cpu" + + %one = corert.executeop(%cpu) + "tfrt_test.create_dense_tensor"() { shape = [1, 1], values = [1 : i32] } : 1 + + // CHECK-NOT: DenseHostTensor dtype = i32, shape = [1, 1], values = [1] + %ch1, %result = corert.while @while_cond_false @while_body_print (%ch0, %one) : (!corert.tensorhandle) -> (!corert.tensorhandle) + + tfrt.return +} + +// CHECK-LABEL: --- Running 'test_copy_if_small' +func.func @test_copy_if_small() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %small = tfrt_fallback_async.const_dense_tensor dense<[1, 2, 3, 4]> : tensor<4xi64> {_tfrt_cost = 1 : i64} + %small_copies:2 = tfrt_fallback_async.copy_if_small %small {_tfrt_cost = 1 : i64} : (!tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + // CHECK: Tensor + // CHECK: Tensor + %ch1 = "tfrt_fallback_async.print_tensor"(%small_copies#0, %ch0) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch2 = "tfrt_fallback_async.print_tensor"(%small_copies#1, %ch1) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + %large = tfrt_fallback_async.const_dense_tensor dense<100> : tensor<128xi64> {_tfrt_cost = 1 : i64} + %large_copies:2 = tfrt_fallback_async.copy_if_small %large {_tfrt_cost = 1 : i64} : (!tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + // CHECK: Tensor + // CHECK: Tensor + %ch3 = "tfrt_fallback_async.print_tensor"(%large_copies#0, %ch2) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch4 = "tfrt_fallback_async.print_tensor"(%large_copies#1, %ch3) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + %string = tfrt_fallback_async.const_string_tensor {shape = [1], value = ["string"], _tfrt_cost = 1 : i64} + %string_copies:2 = tfrt_fallback_async.copy_if_small %string {_tfrt_cost = 1 : i64} : (!tfrt_fallback.tf_tensor) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + // CHECK: Tensor + // CHECK: Tensor + %ch5 = "tfrt_fallback_async.print_tensor"(%string_copies#0, %ch4) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch6 = "tfrt_fallback_async.print_tensor"(%string_copies#1, %ch5) : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch4 : !tfrt.chain +} + +func.func @branch0(%ch0: !tfrt.chain, %arg0: !corert.tensorhandle, %arg1: !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) { + %cpu = corert.get_op_handler %ch0 "cpu" + %res = corert.executeop(%cpu) "tfrt_test.add"(%arg0, %arg1) : 1 + tfrt.return %ch0, %res : !tfrt.chain, !corert.tensorhandle +} + +func.func @branch1(%ch0: !tfrt.chain, %arg0: !corert.tensorhandle, %arg1: !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) { + %cpu = corert.get_op_handler %ch0 "cpu" + %th = corert.executeop(%cpu) + "tfrt_test.create_dense_tensor"() { shape = [1], values = [4 : i32] } : 1 + %add0 = corert.executeop(%cpu) "tfrt_test.add"(%arg0, %arg1) : 1 + %res = corert.executeop(%cpu) "tfrt_test.add"(%add0, %th) : 1 + tfrt.return %ch0, %res : !tfrt.chain, !corert.tensorhandle +} + +// CHECK-LABEL: --- Running 'test_case_fallback_index' +func.func @test_case_fallback_index() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %cpu = corert.get_op_handler %ch0 "cpu" + + %ch1 = tfrt_fallback_async.createop(%ch0) key(13) device("CPU:0") "tf.Const"() {dtype = i32, value = dense<0> : tensor} num_args(0) + %ch2, %fallback_idx0 = tfrt_fallback_async.executeop.seq(%ch1) key(13) cost(100) device("CPU:0") "tf.Const"() {dtype = i32, value = dense<0> : tensor} : 1 + %fallback_idx_th0 = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %fallback_idx0 {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + %idx0 = corert.tensorhandle_to_int32 %fallback_idx_th0 + + %ch5 = tfrt_fallback_async.createop(%ch0) key(14) device("CPU:0") "tf.Const"() {dtype = i32, value = dense<1> : tensor} num_args(0) + %ch6, %fallback_idx1 = tfrt_fallback_async.executeop.seq(%ch5) key(14) cost(100) device("CPU:0") "tf.Const"() {dtype = i32, value = dense<1> : tensor} : 1 + %fallback_idx_th1 = tfrt_fallback_async.fallback_tensor_to_corert_tensorhandle %fallback_idx1 {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!tfrt_fallback.tf_tensor) -> (!corert.tensorhandle) + %idx1 = corert.tensorhandle_to_int32 %fallback_idx_th1 + + %arg0 = corert.executeop(%cpu) + "tfrt_test.create_dense_tensor"() { shape = [1], values = [2 : i32] } : 1 + %arg1 = corert.executeop(%cpu) + "tfrt_test.create_dense_tensor"() { shape = [1], values = [4 : i32] } : 1 + + %ch3, %res0 = tfrt.case %idx0 [@branch0, @branch1] (%ch0, %arg0, %arg1) : (!tfrt.chain, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) + + // CHECK: DenseHostTensor dtype = i32, shape = [1], values = [6] + %ch4 = "corert.print_tensorhandle"(%res0, %ch3) : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + %ch7, %res1 = tfrt.case %idx1 [@branch0, @branch1] (%ch4, %arg0, %arg1) : (!tfrt.chain, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt.chain, !corert.tensorhandle) + + // CHECK: DenseHostTensor dtype = i32, shape = [1], values = [10] + %ch8 = "corert.print_tensorhandle"(%res1, %ch7) : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch8 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_fallback_to_host' +func.func @test_fallback_to_host() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(2) device("/CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<2x2xf32> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(3) device("/CPU:0") "tf.VarHandleOp" () { dtype = f32, shape = #corert.shape<2x2>, container = "c", shared_name = "v0"} num_args(0) + %ch3 = tfrt_fallback_async.createop(%ch2) key(4) device("/CPU:0") "tf.AssignVariableOp" () { dtype = f32 } num_args(2) + %ch4 = tfrt_fallback_async.createop(%ch3) key(5) device("/TPU:0") "tf.ReadVariableOp" () { dtype = f32 } num_args(1) + + %ch5, %val = tfrt_fallback_async.executeop.seq(%ch4) key(2) cost(100) device("/CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<2x2xf32> } : 1 + %ch6, %varh = tfrt_fallback_async.executeop.seq(%ch4) key(3) cost(100) device("/CPU:0") "tf.VarHandleOp" () { dtype = f32, shape = #corert.shape<2x2>, container = "c", shared_name = "v0"} : 1 + %ch7 = tfrt_fallback_async.executeop.seq(%ch6) key(4) cost(100) device("/CPU:0") "tf.AssignVariableOp" (%varh, %val) { dtype = f32 } : 0 + %ch8, %val2 = tfrt_fallback_async.executeop.seq(%ch7) key(5) cost(100) device("/TPU:0") "tf.ReadVariableOp" (%varh) { dtype = f32 } : 1 + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch9 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_custom_allocator' +func.func @test_custom_allocator() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %allocator = "tfrt_fallback_async.get_test_allocator"() : () -> (!tfrt_fallback.tf_allocator) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("/CPU:0") "tf.Cast"() { SrcT = i32, DstT = f32 } num_args(1) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("/CPU:0") "tf.Cast"() { SrcT = f32, DstT = i32 } num_args(1) + %ch3 = tfrt_fallback_async.createop(%ch2) key(2) device("/CPU:0") "tf.Const"() { dtype = i32, value = dense<123> : tensor } num_args(0) + + %ch4, %const = tfrt_fallback_async.executeop.seq(%ch3) key(2) cost(100) device("/CPU:0") "tf.Const"() { dtype = i32, value = dense<123> : tensor } : 1 + // CHECK: Using TestAllocator + %val0 = tfrt_fallback_async.executeop.allocator(%allocator) key(0) cost(100) device("/CPU:0") "tf.Cast"(%const) { SrcT = i32, DstT = f32 } : 1 + // CHECK: Using TestAllocator + %ch5, %val1 = tfrt_fallback_async.executeop.seq.allocator(%ch4, %allocator) key(1) cost(100) device("/CPU:0") "tf.Cast"(%val0) { SrcT = f32, DstT = i32 } : 1 + + // CHECK: Tensor + %ch6 = "tfrt_fallback_async.print_tensor"(%val0, %ch5) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + // CHECK: Tensor + %ch7 = "tfrt_fallback_async.print_tensor"(%val1, %ch6) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch7 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_custom_allocator_async_opkernel' +func.func @test_custom_allocator_async_opkernel() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %allocator = "tfrt_fallback_async.get_test_allocator"() : () -> (!tfrt_fallback.tf_allocator) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("/CPU:0") "tf.Const"() { dtype = i32, value = dense<123> : tensor } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("/CPU:0") "tf.TestAsyncIdentity"() { T = i32 } num_args(1) + + %ch3, %const = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("/CPU:0") "tf.Const"() { dtype = i32, value = dense<123> : tensor } : 1 + %val = tfrt_fallback_async.executeop.allocator(%allocator) key(1) cost(100) device("/CPU:0") "tf.TestAsyncIdentity"(%const) { SrcT = i32, DstT = f32 } : 1 + + // CHECK: Tensor + %ch4 = "tfrt_fallback_async.print_tensor"(%val, %ch3) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch4 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback_tensor_conversion_host.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback_tensor_conversion_host.mlir new file mode 100644 index 0000000000000000000000000000000000000000..1f35124a5e0ef9505e00f115817e1918eee5f33a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/fallback_tensor_conversion_host.mlir @@ -0,0 +1,154 @@ +// RUN: tfrt_translate -mlir-to-bef %s | tf_bef_executor --test_init_function=register_op_handlers | FileCheck %s --dump-input=fail + +func.func @register_op_handlers() { + %fallback = "corert.create_runtime_fallback_op_handler"() {tf_device_name="/device:CPU:0"} : () -> !corert.ophandler + %cpu = "corert.create_cpu_op_handler"(%fallback) : (!corert.ophandler) -> !corert.ophandler + corert.register_op_handler %fallback "tf" + corert.register_op_handler %cpu "cpu" + tfrt.return +} + +// CHECK: --- Running 'transfer_to_host' +func.func @transfer_to_host() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %rtfb_handler = "corert.create_runtime_fallback_op_handler"() {tf_device_name="/device:CPU:0"} : () -> !corert.ophandler + %cpu_device = "tfrt.get_device"(%ch0) {device_name="CPU:0"} : (!tfrt.chain) -> !tfrt.device + + %th0 = corert.executeop(%rtfb_handler) "tf.Const"() + { dtype = f32, value = dense<1.0> : tensor<1xf32> } : 1 + + %th1 = corert.executeop(%rtfb_handler) "tf.Exp" (%th0) { T = f32 } : 1 + + // TFRuntimeFallbackT->DHT + %th2_tensor_type = corert.get_dst_tensor_type %th1, %cpu_device + %th2 = corert.transfer %th1, %cpu_device, %th2_tensor_type + + // CHECK: DenseHostTensor dtype = f32, shape = [1], values = [2.718282e+00] + %ch1 = "corert.print_tensorhandle"(%th2, %ch0) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK: --- Running 'scalar_to_runtime_fallback' +func.func @scalar_to_runtime_fallback() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %cpu = corert.get_op_handler %ch0 "cpu" + %cpu_device = "tfrt.get_device"(%ch0) {device_name="CPU:0"} : (!tfrt.chain) -> !tfrt.device + + %th1 = corert.executeop(%cpu) "tfrt_test.create_from_scalar"() + {shape = [2: i64, 2: i64], value = 1: i32} : 1 + + // ScalarHT->TFRuntimeFallbackT + %th2_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "RuntimeFallback" } : () -> !tfrt.tensor_type + %th2 = corert.transfer %th1, %cpu_device, %th2_tensor_type + + // TFRuntimeFallbackT->DHT + %th3_tensor_type = corert.get_dst_tensor_type %th2, %cpu_device + %th3 = corert.transfer %th2, %cpu_device, %th3_tensor_type + + // CHECK: DenseHostTensor dtype = i32, shape = [2, 2], values = [1, 1, 1, 1] + %ch1 = "corert.print_tensorhandle"(%th3, %ch0) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch1 : !tfrt.chain +} + +// CHECK: --- Running 'dht_to_runtime_fallback' +func.func @dht_to_runtime_fallback() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %cpu = corert.get_op_handler %ch0 "cpu" + %cpu_device = "tfrt.get_device"(%ch0) {device_name="CPU:0"} : (!tfrt.chain) -> !tfrt.device + + %th1 = corert.executeop(%cpu) "tfrt_test.create_dense_tensor"() + { shape = [2: i64, 2: i64], values = [42 : i32] } : 1 + + // DHT->TFRuntimeFallbackT + %th2_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "RuntimeFallback" } : () -> !tfrt.tensor_type + %th2 = corert.transfer %th1, %cpu_device, %th2_tensor_type + + // TFRuntimeFallbackT->DHT + %th3_tensor_type = corert.get_dst_tensor_type %th2, %cpu_device + %th3 = corert.transfer %th2, %cpu_device, %th3_tensor_type + + // CHECK: DenseHostTensor dtype = i32, shape = [2, 2], values = [42, 42, 42, 42] + %ch1 = "corert.print_tensorhandle"(%th3, %ch0) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + // DHT->TFKernelFallbackT + %th4_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "KernelFallback" } : () -> !tfrt.tensor_type + %th4 = corert.transfer %th1, %cpu_device, %th4_tensor_type + + // TFKernelFallbackT->TFRuntimeFallbackT + %th5_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "RuntimeFallback" } : () -> !tfrt.tensor_type + %th5 = corert.transfer %th4, %cpu_device, %th5_tensor_type + + // TFRuntimeFallbackT->TFKernelFallbackT + %th6_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "KernelFallback" } : () -> !tfrt.tensor_type + %th6 = corert.transfer %th5, %cpu_device, %th6_tensor_type + + // TFKernelFallbackT->DHT + %th7_tensor_type = corert.get_dst_tensor_type %th6, %cpu_device + %th7 = corert.transfer %th6, %cpu_device, %th7_tensor_type + + // CHECK: DenseHostTensor dtype = i32, shape = [2, 2], values = [42, 42, 42, 42] + %ch2 = "corert.print_tensorhandle"(%th7, %ch1) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch2 : !tfrt.chain +} + +// CHECK: --- Running 'sht_to_fallback' +func.func @sht_to_fallback() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %cpu = corert.get_op_handler %ch0 "cpu" + %cpu_device = "tfrt.get_device"(%ch0) {device_name="CPU:0"} : (!tfrt.chain) -> !tfrt.device + + %th1 = corert.const_string_tensor {shape = [2], value = ["uiuc", "berkeley"]} + + // SHT->TFRuntimeFallbackT + %th2_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "RuntimeFallback" } : () -> !tfrt.tensor_type + %th2 = corert.transfer %th1, %cpu_device, %th2_tensor_type + + // TFRuntimeFallbackT->SHT + %th3_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "StringHost" } : () -> !tfrt.tensor_type + %th3 = corert.transfer %th2, %cpu_device, %th3_tensor_type + + // CHECK: StringHostTensor shape = [2], values = ["uiuc", "berkeley"] + %ch1 = "corert.print_tensorhandle"(%th3, %ch0) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + // SHT->TFKernelFallbackT + %th4_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "KernelFallback" } : () -> !tfrt.tensor_type + %th4 = corert.transfer %th1, %cpu_device, %th4_tensor_type + + // TFKernelFallbackT->TFRuntimeFallbackT + %th5_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "RuntimeFallback" } : () -> !tfrt.tensor_type + %th5 = corert.transfer %th4, %cpu_device, %th5_tensor_type + + // TFRuntimeFallbackT->TFKernelFallbackT + %th6_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "KernelFallback" } : () -> !tfrt.tensor_type + %th6 = corert.transfer %th5, %cpu_device, %th6_tensor_type + + // TFKernelFallbackT->SHT + %th7_tensor_type = "tfrt_test.get_static_tensor_type"() + { tensor_type = "StringHost" } : () -> !tfrt.tensor_type + %th7 = corert.transfer %th6, %cpu_device, %th7_tensor_type + + // CHECK: StringHostTensor shape = [2], values = ["uiuc", "berkeley"] + %ch2 = "corert.print_tensorhandle"(%th7, %ch1) + : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch2 : !tfrt.chain +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_compat_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_compat_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..339adf1ac362e9d50eae4ea774ecd64d67e72bc2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_compat_test.cc @@ -0,0 +1,150 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/runtime_fallback/kernel/kernel_fallback_compat_request_state.h" +#include "tensorflow/core/runtime_fallback/util/fallback_test_util.h" +#include "tensorflow/core/tfrt/fallback/op_kernel_runner.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tensorflow/core/tfrt/utils/thread_pool.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_executor/bef_file.h" // from @tf_runtime +#include "tfrt/core_runtime/core_runtime.h" // from @tf_runtime +#include "tfrt/host_context/async_value.h" // from @tf_runtime +#include "tfrt/host_context/chain.h" // from @tf_runtime +#include "tfrt/host_context/function.h" // from @tf_runtime +#include "tfrt/host_context/host_context.h" // from @tf_runtime +#include "tfrt/host_context/resource_context.h" // from @tf_runtime +#include "tfrt/support/ref_count.h" // from @tf_runtime +#include "tfrt/tracing/tracing.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +// Creates a BEF file with a program that runs tfrt_fallback.batch_function with +// a empty function forwarding inputs or outputs. +// +// TODO(b/175648326): Move the function below to the common test utilities for +// BEF. +std::pair> CreateBefFile( + absl::string_view file_name, tfrt::HostContext* host) { + std::string file_path = GetDataDependencyFilepath(absl::StrCat( + "tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/", file_name)); + std::string data; + CHECK_OK(ReadFileToString(Env::Default(), file_path, &data)); + + tfrt::BefBuffer bef_buffer(data.begin(), data.end()); + + auto bef_file = tfrt::BEFFile::Open(bef_buffer, host->GetKernelRegistry(), + host->diag_handler(), host->allocator()); + CHECK(bef_file); + return std::make_pair(std::move(bef_buffer), std::move(bef_file)); +} + +TEST(KernelFallbackCompatTest, CreateOp) { + tfrt::tracing::SetTracingLevel(tfrt::tracing::TracingLevel::Debug); + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/4); + auto* host = runtime->core_runtime()->GetHostContext(); + auto pair = CreateBefFile("create_op.mlir.bef", host); + auto& bef_file = pair.second; + + tfrt::ResourceContext resource_ctx; + auto exec_ctx = tfd::CreateFallbackTestExecutionContext(host, &resource_ctx); + + auto chain = tfrt::GetReadyChain(); + + auto* func = bef_file->GetFunction("init"); + ASSERT_TRUE(func != nullptr); + + std::vector> results; + results.resize(1); + + func->Execute(exec_ctx, {chain.GetAsyncValue()}, results); + host->Await(results); + + ASSERT_FALSE(results[0]->IsError()) << results[0]->GetError().message(); + + auto* fallback_request_state = + exec_ctx.request_ctx() + ->GetDataIfExists(); + + ASSERT_TRUE(fallback_request_state != nullptr); + + auto* rendezvous = fallback_request_state->rendezvous(); + ASSERT_TRUE(rendezvous != nullptr); + + auto* runner_table = fallback_request_state->runner_table(); + + // TODO(tfrt-devs): Create a special key type for OpKernelRunnerCache instead + // of using tfrt::Location. The key should be generated by higher-level + // applications such as compiler, and it is higher-level applications' + // responsibility to make sure the key is unique in an OpKernelRunnerCache + // instance. + auto* kernel_runner_add = runner_table->Get(0); + ASSERT_TRUE(kernel_runner_add); + + auto* kernel_runner_flatmap = runner_table->Get(1); + ASSERT_TRUE(kernel_runner_flatmap); +} + +TEST(KernelFallbackCompatTest, CustomThreadPool) { + auto runtime = + tensorflow::tfrt_stub::Runtime::Create(/*num_inter_op_threads=*/4); + auto* host = runtime->core_runtime()->GetHostContext(); + + auto pair = CreateBefFile("custom_thread_pool.mlir.bef", host); + auto& bef_file = pair.second; + + tfrt::ResourceContext resource_ctx; + + tensorflow::tfrt_stub::TfThreadPool thread_pool(/*name=*/"test", + /*num_threads=*/1); + + auto exec_ctx = tfd::CreateFallbackTestExecutionContext(host, &resource_ctx, + &thread_pool); + + auto chain = tfrt::GetReadyChain(); + + auto* init_func = bef_file->GetFunction("init"); + ASSERT_TRUE(init_func != nullptr); + + std::vector> results; + results.resize(1); + + init_func->Execute(exec_ctx, {chain.GetAsyncValue()}, results); + host->Await(results); + ASSERT_FALSE(results[0]->IsError()) << results[0]->GetError().message(); + + auto* run_func = bef_file->GetFunction("run"); + ASSERT_TRUE(run_func != nullptr); + + results.clear(); + results.resize(2); + run_func->Execute(exec_ctx, {chain.GetAsyncValue()}, results); + host->Await(results); + ASSERT_FALSE(results[0]->IsError()) << results[0]->GetError().message(); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_op_handler.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_op_handler.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d992b52788b706da626e2b38096b82f265c8df40 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/kernel_fallback_op_handler.mlir @@ -0,0 +1,31 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor | FileCheck %s + + +// CHECK-LABEL: --- Not running 'register_op_handler' because it has arguments. +func.func @register_op_handler(%ch0: !tfrt.chain) -> !tfrt.chain { + %op_handler = "corert.create_kernel_fallback_op_handler"() : () -> !corert.ophandler + %ch = corert.register_op_handler %op_handler "tfkernel0" + tfrt.return %ch : !tfrt.chain +} + +// CHECK-LABEL: --- Not running 'get_op_handler' because it has arguments. +func.func @get_op_handler(%ch0: !tfrt.chain) -> !tfrt.chain { + %tfkernel0 = corert.get_op_handler %ch0 "tfkernel0" + tfrt.return %ch0 : !tfrt.chain +} + +// CHECK-LABEL: --- Not running 'failed_get_op_handler' because it has arguments. +func.func @failed_get_op_handler(%ch0: !tfrt.chain) -> !tfrt.chain { + // expected-error @+1 {{runtime error: op_handler not found}} + %tf0 = corert.get_op_handler %ch0 "tfkernel0" + tfrt.return %ch0 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_op_handler_kernels' +func.func @test_op_handler_kernels() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %ch1 = tfrt.call @failed_get_op_handler(%ch0) : (!tfrt.chain) -> !tfrt.chain + %ch2 = tfrt.call @register_op_handler(%ch1) : (!tfrt.chain) -> !tfrt.chain + %ch3 = tfrt.call @get_op_handler(%ch2) : (!tfrt.chain) -> !tfrt.chain + tfrt.return %ch3 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/mnist.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/mnist.mlir new file mode 100644 index 0000000000000000000000000000000000000000..9d7e09d88f3ccaadc8bc6cd5038384b470a5e0bb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/mnist.mlir @@ -0,0 +1,196 @@ +// RUN: tfrt_translate -mlir-to-bef %s | tf_bef_executor | FileCheck %s + +// Initialize all the variables used in the model by creating variable handles +// and assigning the variables with the correct values. This function takes an +// input chain and returns an output chain and a list of variable handles that +// have been value-initialized. The returned tfd.tf_tensors are TF tensors that +// contain tensorflow::ResourceHandle. +func.func @mnist_init_variables(%start_c : !tfrt.chain) -> ( + !tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor) { + + // Init w1. + %init_w1_c, %w1_h = "tfd.delegate_kernel"(%start_c) { + _name = "VarHandleOp", attr0_name = "container", attr0_value = "string$", + attr1_name = "shared_name", attr1_value = "string$w1", + attr2_name = "dtype", attr2_value = "tfdtype$DT_INT32", + attr3_name = "shape", attr3_value = "tfshape$[2,2]" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %const_w1_c, %w1_const_tensor = "tfd.delegate_kernel"(%init_w1_c) { + _name = "Const", attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32", + attr1_name = "value", + attr1_value = "tftensor$dtype:DT_INT32 tensor_shape { dim { size: 2 } dim { size: 2 }} int_val: 1 int_val: 1 int_val: 1 int_val: 1" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %assignv_w1_c = "tfd.delegate_kernel"(%const_w1_c, %w1_h, %w1_const_tensor) { + _name = "AssignVariableOp", + attr0_name = "dtype",attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain) + + // Init b1. + %init_b1_c, %b1_h = "tfd.delegate_kernel"(%assignv_w1_c) { + _name = "VarHandleOp", attr0_name = "container", attr0_value = "string$", + attr1_name = "shared_name", attr1_value = "string$b1", + attr2_name = "dtype", attr2_value = "tfdtype$DT_INT32", + attr3_name = "shape", attr3_value = "tfshape$[2,2]" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %const_b1_c, %b1_const_tensor = "tfd.delegate_kernel"(%init_b1_c) { + _name = "Const", attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32", + attr1_name = "value", + attr1_value = "tftensor$dtype:DT_INT32 tensor_shape { dim { size: 2 } dim { size: 2 }} int_val: 1 int_val: 1 int_val: 1 int_val: 1" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %assignv_b1_c = "tfd.delegate_kernel"(%const_b1_c, %b1_h, %b1_const_tensor) { + _name = "AssignVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain) + + // Init w2. + %init_w2_c, %w2_h = "tfd.delegate_kernel"(%assignv_b1_c) { + _name = "VarHandleOp", attr0_name = "container", attr0_value = "string$", + attr1_name = "shared_name", attr1_value = "string$w2", + attr2_name = "dtype", attr2_value = "tfdtype$DT_INT32", + attr3_name = "shape", attr3_value = "tfshape$[2,2]" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %const_w2_c, %w2_const_tensor = "tfd.delegate_kernel"(%init_w2_c) { + _name = "Const", attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32", + attr1_name = "value", + attr1_value = "tftensor$dtype:DT_INT32 tensor_shape { dim { size: 2 } dim { size: 2 }} int_val: 1 int_val: 1 int_val: 1 int_val: 1" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %assignv_w2_c = "tfd.delegate_kernel"(%const_w2_c, %w2_h, %w2_const_tensor) { + _name = "AssignVariableOp", + attr0_name = "dtype",attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain) + + // Init b2. + %init_b2_c, %b2_h = "tfd.delegate_kernel"(%assignv_w2_c) { + _name = "VarHandleOp", attr0_name = "container", attr0_value = "string$", + attr1_name = "shared_name", attr1_value = "string$b2", + attr2_name = "dtype", attr2_value = "tfdtype$DT_INT32", + attr3_name = "shape", attr3_value = "tfshape$[2,2]" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %const_b2_c, %b2_const_tensor = "tfd.delegate_kernel"(%init_b2_c) { + _name = "Const", attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32", + attr1_name = "value", + attr1_value = "tftensor$dtype:DT_INT32 tensor_shape { dim { size: 2 } dim { size: 2 }} int_val: 1 int_val: 1 int_val: 1 int_val: 1" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + %assignv_b2_c = "tfd.delegate_kernel"(%const_b2_c, %b2_h, %b2_const_tensor) { + _name = "AssignVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain) + + tfrt.return %assignv_b2_c, %w1_h, %b1_h, %w2_h, %b2_h : !tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor +} + +// This function represents one forward pass on the model. It takes an input +// chain and the variable handles for all model variables, and returns an output +// chain and the result tensor value. +func.func @inference_call( + %start_c : !tfrt.chain, %inputx_tensor : !tfd.tf_tensor, + %w1_h : !tfd.tf_tensor, %b1_h : !tfd.tf_tensor, %w2_h : !tfd.tf_tensor, + %b2_h : !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) { + + // Read all variables into tensors. + %readv_c1, %w1_tensor = "tfd.delegate_kernel"(%start_c, %w1_h) { + _name = "ReadVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + %readv_c2, %b1_tensor = "tfd.delegate_kernel"(%readv_c1, %b1_h) { + _name = "ReadVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + %readv_c3, %w2_tensor = "tfd.delegate_kernel"(%readv_c2, %w2_h) { + _name = "ReadVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + %readv_c4, %b2_tensor = "tfd.delegate_kernel"(%readv_c3, %b2_h) { + _name = "ReadVariableOp", + attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // Compute the model forward pass. + %matmul0_out_c, %matmul0_tensor = "tfd.delegate_kernel"( + %readv_c4, %inputx_tensor, %w1_tensor) { + _name = "MatMul", + attr1_name = "transpose_a", attr1_value = "bool$false", + attr2_name = "transpose_b", attr2_value = "bool$false" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> ( + !tfrt.chain, !tfd.tf_tensor) + + %add0_out_c, %add0_tensor = "tfd.delegate_kernel"( + %matmul0_out_c, %matmul0_tensor, %b1_tensor) { + _name = "AddV2" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> ( + !tfrt.chain, !tfd.tf_tensor) + + %relu0_out_c, %relu0_tensor = "tfd.delegate_kernel"( + %add0_out_c, %add0_tensor) { + _name = "Relu" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + %matmul1_out_c, %matmul1_tensor = "tfd.delegate_kernel"( + %relu0_out_c, %relu0_tensor, %w2_tensor) { + _name = "MatMul", + attr1_name = "transpose_a", attr1_value = "bool$false", + attr2_name = "transpose_b", attr2_value = "bool$false" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> ( + !tfrt.chain, !tfd.tf_tensor) + + %add1_out_c, %add1_tensor = "tfd.delegate_kernel"( + %matmul1_out_c, %matmul1_tensor, %b2_tensor) { + _name = "AddV2" + } : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> ( + !tfrt.chain, !tfd.tf_tensor) + + %identity_out_c, %identity_tensor = "tfd.delegate_kernel"( + %add1_out_c, %add1_tensor) { + _name = "Identity" + } : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + tfrt.return %identity_out_c, %identity_tensor : !tfrt.chain, !tfd.tf_tensor +} + +// This is the main driver function. It creates the EagerContext, initializes +// all variables and runs one forward pass on the model. +// CHECK: --- Running 'mnist_delegate_test' +func.func @mnist_delegate_test() { + %start_c = tfrt.new.chain + + // Init eager context. + %context_init_c = "tfd.init_eager_context"(%start_c) + : (!tfrt.chain) -> !tfrt.chain + + // Initialize all variables. + %init_var_c, %w1_h, %b1_h, %w2_h, %b2_h + = tfrt.call @mnist_init_variables(%context_init_c) : (!tfrt.chain) -> ( + !tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor, + !tfd.tf_tensor) + + // Get input data. + %get_input_c, %inputx_tensor = "tfd.delegate_kernel"(%init_var_c) { + _name = "Const", attr0_name = "dtype", attr0_value = "tfdtype$DT_INT32", + attr1_name = "value", + attr1_value = "tftensor$dtype:DT_INT32 tensor_shape { dim { size: 2 } dim { size: 2 }} int_val: 1 int_val: 1 int_val: 1 int_val: 1" + } : (!tfrt.chain) -> (!tfrt.chain, !tfd.tf_tensor) + + // Make one inference call. + %inference_call_out_c, %inference_call_out_tensor + = tfrt.call @inference_call( + %get_input_c, %inputx_tensor, %w1_h, %b1_h, %w2_h, %b2_h) : ( + !tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor, !tfd.tf_tensor, + !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // CHECK: shape = [2, 2], values = [7, 7, 7, 7] + %print_c = "tfd.print_tft"(%inference_call_out_tensor, %inference_call_out_c) + : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/runtime_fallback_op_handler.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/runtime_fallback_op_handler.mlir new file mode 100644 index 0000000000000000000000000000000000000000..b66a90d2108b9c0c9595b728b97471a420261590 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/runtime_fallback_op_handler.mlir @@ -0,0 +1,37 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor | FileCheck %s + + +// CHECK-LABEL: --- Not running 'register_runtime_fallback_op_handler_chain' because it has arguments. +func.func @register_runtime_fallback_op_handler_chain(%ch0: !tfrt.chain) -> !tfrt.chain { + %runtime_fallback = "corert.create_runtime_fallback_op_handler"() {tf_device_name="/device:CPU:0"} : () -> !corert.ophandler + %ch = corert.register_op_handler %runtime_fallback "tf0" + tfrt.return %ch : !tfrt.chain +} + +// CHECK-LABEL: --- Not running 'get_runtime_fallback_op_handler' because it has arguments. +func.func @get_runtime_fallback_op_handler(%ch0: !tfrt.chain) -> !tfrt.chain { + %tf0 = corert.get_op_handler %ch0 "tf0" + + %th = corert.executeop(%tf0) "tf.Const"() + { dtype = f32, value = dense<1.0> : tensor<2x2xf32> } : 1 + + %ch2 = "corert.print_tensorhandle" (%th, %ch0) : (!corert.tensorhandle, !tfrt.chain) -> !tfrt.chain + tfrt.return %ch2 : !tfrt.chain +} + +// CHECK-LABEL: --- Not running 'failed_runtime_fallback_get_op_handler' because it has arguments. +func.func @failed_runtime_fallback_get_op_handler(%ch0: !tfrt.chain) -> !tfrt.chain { + // expected-error @+1 {{runtime error: op_handler not found}} + %tf0 = corert.get_op_handler %ch0 "tf0" + tfrt.return %ch0 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_runtime_fallback_op_handler_chain_kernels' +func.func @test_runtime_fallback_op_handler_chain_kernels() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %ch1 = tfrt.call @failed_runtime_fallback_get_op_handler(%ch0) : (!tfrt.chain) -> !tfrt.chain + %ch2 = tfrt.call @register_runtime_fallback_op_handler_chain(%ch1) : (!tfrt.chain) -> !tfrt.chain + // CHECK: RuntimeFallbackTensor dtype = float, shape = [2, 2], values = [1.000000e+00, 1.000000e+00, 1.000000e+00, 1.000000e+00] + %ch3 = tfrt.call @get_runtime_fallback_op_handler(%ch2) : (!tfrt.chain) -> !tfrt.chain + tfrt.return %ch3 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_delegate.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_delegate.mlir new file mode 100644 index 0000000000000000000000000000000000000000..e7f6090651f0e1afba97f10d7ae6c0c3fb09a55f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_delegate.mlir @@ -0,0 +1,125 @@ +// RUN: tfrt_translate -mlir-to-bef %s | tf_bef_executor | FileCheck %s + +// CHECK: --- Running 'matmul_delegate_test' +func.func @matmul_delegate_test() { + %c0 = tfrt.new.chain + + // Create 2x2 dht with value 1 + %dht_a = tfrt_dht.create_uninitialized_tensor.i32.2 [2 : i64, 2 : i64] + %c1 = tfrt_dht.fill_tensor_with_constant.i32 %dht_a, %c0 1 : i32 + + // Create 2x2 dht with value 2 + %dht_b = tfrt_dht.create_uninitialized_tensor.i32.2 [2 : i64, 2 : i64] + %c2 = tfrt_dht.fill_tensor_with_constant.i32 %dht_b, %c0 2 : i32 + + // Convert dht to tf tensor + %tft_a, %c3 = "tfd.move_dht_to_tft"(%dht_a, %c1) + : (!tfrt_tensor.tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) + %tft_b, %c4 = "tfd.move_dht_to_tft"(%dht_b, %c2) + : (!tfrt_tensor.tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) + + // Print legacy TF tensors + // CHECK: shape = [2, 2], values = [1, 1, 1, 1] + %cc0 = "tfd.print_tft"(%tft_a, %c0) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + // CHECK: shape = [2, 2], values = [2, 2, 2, 2] + %cc1 = "tfd.print_tft"(%tft_b, %cc0) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + // Create TF eager context + %c6 = "tfd.init_eager_context"(%c0): (!tfrt.chain) -> !tfrt.chain + + // Delegate to tf.matmul + %c7, %tft_x = "tfd.delegate_kernel"(%c6, %tft_a, %tft_b) {op_name = "MatMul"} + : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // CHECK: shape = [2, 2], values = [4, 4, 4, 4] + %cc2 = "tfd.print_tft"(%tft_x, %cc1) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + // Delegate to tf.matmul by using %tft_x from the previous delegation as input + %c8, %tft_y = "tfd.delegate_kernel"(%c6, %tft_x, %tft_b) {op_name = "MatMul"} + : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // Print legacy TF tensors + // CHECK: shape = [2, 2], values = [16, 16, 16, 16] + %cc3 = "tfd.print_tft"(%tft_y, %cc2) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + // Convert tf tensor back to dht + %dht_c, %c9 = "tfd.convert_tft_to_dht"(%tft_x, %cc2) + : (!tfd.tf_tensor, !tfrt.chain) -> (!tfrt_tensor.tensor, !tfrt.chain) + %dht_d, %c10 = "tfd.convert_tft_to_dht"(%tft_y, %cc3) + : (!tfd.tf_tensor, !tfrt.chain) -> (!tfrt_tensor.tensor, !tfrt.chain) + + // Print the result dht + // CHECK: shape = [2, 2], values = [4, 4, 4, 4] + %cc4 = tfrt_dht.print_tensor %dht_c, %cc3 + // CHECK: shape = [2, 2], values = [16, 16, 16, 16] + %cc5 = tfrt_dht.print_tensor %dht_d, %cc4 + + tfrt.return +} + +// CHECK: --- Running 'bad_op_name_test' +func.func @bad_op_name_test() { + %c0 = tfrt.new.chain + + // Create 2x2 dht with value 2 + %dht_a = tfrt_dht.create_uninitialized_tensor.i32.2 [2 : i64, 2 : i64] + %c1 = tfrt_dht.fill_tensor_with_constant.i32 %dht_a, %c0 2 : i32 + + // Convert dht to tf tensor + %tft_a, %c2 = "tfd.move_dht_to_tft"(%dht_a, %c1) + : (!tfrt_tensor.tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) + + // Create TF eager context + %c3 = "tfd.init_eager_context"(%c0): (!tfrt.chain) -> !tfrt.chain + // expected-error @+1 {{runtime error: 'BadOp'}} + %c4, %tft_x = "tfd.delegate_kernel"(%c3, %tft_a) {op_name = "BadOp"} + : (!tfrt.chain, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + tfrt.return +} + +// CHECK: --- Running 'addn_delegate_test' +func.func @addn_delegate_test() { + %c0 = tfrt.new.chain + + // Create scalar dht with value 1 + %dht_a = tfrt_dht.create_uninitialized_tensor.i32.0 [] + %c1 = tfrt_dht.fill_tensor_with_constant.i32 %dht_a, %c0 1 : i32 + + // Create scalar dht with value 2 + %dht_b = tfrt_dht.create_uninitialized_tensor.i32.0 [] + %c2 = tfrt_dht.fill_tensor_with_constant.i32 %dht_b, %c0 2 : i32 + + // Convert dht to tf tensor + %tft_a, %c3 = "tfd.move_dht_to_tft"(%dht_a, %c1) + : (!tfrt_tensor.tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) + %tft_b, %c4 = "tfd.move_dht_to_tft"(%dht_b, %c2) + : (!tfrt_tensor.tensor, !tfrt.chain) -> (!tfd.tf_tensor, !tfrt.chain) + + // Print legacy TF tensors + // CHECK: shape = [], values = [1] + %cc0 = "tfd.print_tft"(%tft_a, %c0) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + // CHECK: shape = [], values = [2] + %cc1 = "tfd.print_tft"(%tft_b, %cc0) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + // Create TF eager context + %c6 = "tfd.init_eager_context"(%c0): (!tfrt.chain) -> !tfrt.chain + + // Delegate to AddN + %c7, %tft_x = "tfd.delegate_kernel"(%c6, %tft_a, %tft_b) {_op_name = "AddN", attr0_name = "N", attr0_value = "int$2", attr1_name = "T", attr1_value = "tfdtype$DT_INT32"} + : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // CHECK: shape = [], values = [3] + %cc2 = "tfd.print_tft"(%tft_x, %cc1) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + // Delegate to AddN by using %tft_x from the previous delegation as input + %c8, %tft_y = "tfd.delegate_kernel"(%c6, %tft_x, %tft_b) {_op_name = "AddN", attr0_name = "N", attr0_value = "int$2", attr1_name = "T", attr1_value = "tfdtype$DT_INT32"} + : (!tfrt.chain, !tfd.tf_tensor, !tfd.tf_tensor) -> (!tfrt.chain, !tfd.tf_tensor) + + // Print legacy TF tensors + // CHECK: shape = [], values = [5] + %cc3 = "tfd.print_tft"(%tft_y, %cc2) : (!tfd.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3769d47e659a8238d4f760b8c60f9546cfb3e8e3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops.mlir @@ -0,0 +1,181 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd --test_init_function=register_op_handlers 2>&1 | FileCheck %s + +func.func @register_op_handlers() -> (!tfrt.chain, !tfrt.chain, !tfrt.chain) { + %fallback = "corert.create_kernel_fallback_op_handler"() : () -> !corert.ophandler + %cpu = "corert.create_cpu_op_handler"(%fallback) : (!corert.ophandler) -> !corert.ophandler + %ch0 = corert.register_op_handler %fallback "tfkernel" + %cpu_identity = "tfrt_test.identity"(%ch0, %cpu) : (!tfrt.chain, !corert.ophandler) -> !corert.ophandler + %ch1 = corert.register_op_handler %cpu_identity "cpu" + %ch2 = "corert.add_kernel_fallback_implicit_conversions" (%cpu_identity) : (!corert.ophandler) -> !tfrt.chain + tfrt.return %ch0, %ch1, %ch2 : !tfrt.chain, !tfrt.chain, !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_tf_random_uniform' +func.func @test_tf_random_uniform() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(0) device("CPU:0") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(1) device("CPU:0") "tf.RandomUniform"() {T = i64, dtype = f32} num_args(1) + + %ch3, %shape = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("CPU:0") "tf.Const"() + { dtype = i64, value = dense<[2,2]> : tensor<2xi64> } : 1 + %random_uniform = tfrt_fallback_async.executeop key(1) cost(100) device("CPU:0") "tf.RandomUniform"(%shape) {T = i64, dtype = f32} : 1 + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch4 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_tf_vars' +func.func @test_tf_vars() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(2) device("CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<2x2xf32> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(3) device("CPU:0") "tf.VarHandleOp" () { dtype = f32, shape = #corert.shape<2x2>, container = "c", shared_name = "v0"} num_args(0) + %ch3 = tfrt_fallback_async.createop(%ch2) key(4) device("CPU:0") "tf.AssignVariableOp" () { dtype = f32 } num_args(2) + %ch4 = tfrt_fallback_async.createop(%ch3) key(5) device("CPU:0") "tf.ReadVariableOp" () { dtype = f32 } num_args(1) + + %ch5, %val = tfrt_fallback_async.executeop.seq(%ch4) key(2) cost(100) device("CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<2x2xf32> } : 1 + %ch6, %varh = tfrt_fallback_async.executeop.seq(%ch4) key(3) cost(100) device("CPU:0") "tf.VarHandleOp" () { dtype = f32, shape = #corert.shape<2x2>, container = "c", shared_name = "v0"} : 1 + %ch7 = tfrt_fallback_async.executeop.seq(%ch6) key(4) cost(100) device("CPU:0") "tf.AssignVariableOp" (%varh, %val) { dtype = f32 } : 0 + %ch8, %val2 = tfrt_fallback_async.executeop.seq(%ch7) key(5) cost(100) device("CPU:0") "tf.ReadVariableOp" (%varh) { dtype = f32 } : 1 + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch9 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_parse_example_v2' +func.func @test_parse_example_v2() -> !tfrt.chain { + %ch = tfrt.new.chain + + %serialized_th = corert.const_string_tensor {shape = [1], value = [""]} + %names_th = corert.const_string_tensor {shape = [0], value = []} + %dense_keys_th = corert.const_string_tensor {shape = [7], value = ["has_login_page_feature", "num_terms_inside_postform", "num_terms_outside_postform", "num_terms_outside_postform_without_bp", "query_params_contains_url", "title_with_login_phase", "url_contains_login_terms"]} + + + %ch0 = tfrt_fallback_async.createop(%ch) key(6) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} num_args(0) + %ch1 = tfrt_fallback_async.createop(%ch0) key(7) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor} num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(8) device("CPU:0") "tf.ParseExampleV2"() + {Tdense = [i64, i64, i64, i64, i64, i64, i64], dense_shapes = [#corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>], num_sparse = 0 : i64, ragged_split_types = [], ragged_value_types = [], sparse_types = []} + num_args(12) + + %ch3, %zero = tfrt_fallback_async.executeop.seq(%ch2) key(6) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} : 1 + %ch4, %one = tfrt_fallback_async.executeop.seq(%ch2) key(7) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor} : 1 + + %serialized, %names, %dense_keys = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %serialized_th, %names_th, %dense_keys_th + {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + %dense_values:7 = tfrt_fallback_async.executeop key(8) cost(100) device("CPU:0") + "tf.ParseExampleV2"(%serialized, %names, %names, %dense_keys, %names, %zero, %one, %zero, %one, %zero, %one, %zero) + {Tdense = [i64, i64, i64, i64, i64, i64, i64], dense_shapes = [#corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>], num_sparse = 0 : i64, ragged_split_types = [], ragged_value_types = [], sparse_types = []} : 7 + + // CHECK: Tensor + // CHECK: Tensor + // CHECK: Tensor + // CHECK: Tensor + // CHECK: Tensor + // CHECK: Tensor + // CHECK: Tensor + %ch5 = "tfrt_fallback_async.print_tensor"(%dense_values#0, %ch4) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch6 = "tfrt_fallback_async.print_tensor"(%dense_values#1, %ch5) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch7 = "tfrt_fallback_async.print_tensor"(%dense_values#2, %ch6) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch8 = "tfrt_fallback_async.print_tensor"(%dense_values#3, %ch7) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch9 = "tfrt_fallback_async.print_tensor"(%dense_values#4, %ch8) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch10 = "tfrt_fallback_async.print_tensor"(%dense_values#5, %ch9) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + %ch11 = "tfrt_fallback_async.print_tensor"(%dense_values#6, %ch10) + : (!tfrt_fallback.tf_tensor, !tfrt.chain) -> !tfrt.chain + + tfrt.return %ch11 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_tile' +func.func @test_tile() -> !tfrt.chain { + %ch = tfrt.new.chain + + %input_th = corert.const_string_tensor {shape = [1, 2], value = ["", ""]} + %multiples_th = corert.const_dense_tensor dense<[7,3]> : tensor<2xi32> + + %input, %multiples = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %input_th, %multiples_th {_tfrt_cost = 1 : i64, device = "CPU:0"} + : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch0 = tfrt_fallback_async.createop(%ch) key(9) device("CPU:0") "tf.Tile"() {T = !corert.string, Tmultiples = i32} num_args(2) + %ch1, %output = tfrt_fallback_async.executeop.seq(%ch0) key(9) cost(100) device("CPU:0") "tf.Tile"(%input, %multiples) {T = !corert.string, Tmultiples = i32} : 1 + + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch2 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'tfrt_test_benchmark_lifetime_regression_test' +func.func @tfrt_test_benchmark_lifetime_regression_test() { + %ch_0 = tfrt.new.chain + %cpu = corert.get_op_handler %ch_0 "cpu" + + %tensor_0 = corert.executeop(%cpu) "tfrt_test.create_dense_tensor"() { shape = [1 : i64], values = [1.0 : f32] } : 1 + %tensor_1 = corert.executeop(%cpu) "tf.VarHandleOp" (){ dtype = f32, shape = #corert.shape<1>, container = "c", shared_name = "v_1" } : 1 + %ch_1 = corert.executeop.seq(%cpu, %ch_0) "tf.AssignVariableOp" (%tensor_1, %tensor_0) { dtype = f32 } : 0 + + tfrt_test.benchmark "tfrt_test_benchmark_lifetime_regression_test"( + %ch_0 : !tfrt.chain, + %ch_1 : !tfrt.chain, + %tensor_1 : !corert.tensorhandle, + %cpu : !corert.ophandler + ) + duration_secs = 1, max_count = 1, num_warmup_runs = 0 + { + %ch_3_1, %tensor_2 = corert.executeop.seq(%cpu, %ch_1) "tf.ReadVariableOp" (%tensor_1) { dtype = f32 } : 1 + + tfrt.return %ch_0 : !tfrt.chain + } + tfrt.return +} + +// CHECK-LABEL: --- Running 'test_quantized' +func.func @test_quantized() -> !tfrt.chain { + %ch0 = tfrt.new.chain + + %ch1 = tfrt_fallback_async.createop(%ch0) key(2) device("CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<1x1xf32> } num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(3) device("CPU:0") "tf.QuantizeV2" () { T = !corert.quint8} num_args(3) + %ch3 = tfrt_fallback_async.createop(%ch2) key(4) device("CPU:0") "tf.QuantizeV2" () { T = !corert.quint16} num_args(3) + %ch4 = tfrt_fallback_async.createop(%ch3) key(5) device("CPU:0") "tf.QuantizeV2" () { T = !corert.qint8} num_args(3) + %ch5 = tfrt_fallback_async.createop(%ch4) key(6) device("CPU:0") "tf.QuantizeV2" () { T = !corert.qint16} num_args(3) + %ch6 = tfrt_fallback_async.createop(%ch5) key(7) device("CPU:0") "tf.QuantizeV2" () { T = !corert.qint32} num_args(3) + + %ch7, %val = tfrt_fallback_async.executeop.seq(%ch6) key(2) cost(100) device("CPU:0") "tf.Const" () { dtype = f32, value = dense<1.0> : tensor<1x1xf32> } : 1 + %ch8, %val1:3 = tfrt_fallback_async.executeop.seq(%ch7) key(3) cost(100) device("CPU:0") "tf.QuantizeV2" (%val, %val, %val) { T = !corert.quint8} : 3 + %ch9, %val2:3 = tfrt_fallback_async.executeop.seq(%ch8) key(4) cost(100) device("CPU:0") "tf.QuantizeV2" (%val, %val, %val) { T = !corert.quint16} : 3 + %ch10, %val3:3 = tfrt_fallback_async.executeop.seq(%ch9) key(5) cost(100) device("CPU:0") "tf.QuantizeV2" (%val, %val, %val) { T = !corert.qint8} : 3 + %ch11, %val4:3 = tfrt_fallback_async.executeop.seq(%ch10) key(6) cost(100) device("CPU:0") "tf.QuantizeV2" (%val, %val, %val) { T = !corert.qint16} : 3 + %ch12, %val5:3 = tfrt_fallback_async.executeop.seq(%ch11) key(7) cost(100) device("CPU:0") "tf.QuantizeV2" (%val, %val, %val) { T = !corert.qint32} : 3 + + // CHECK: Tensor !tfrt.chain + // CHECK: Tensor !tfrt.chain + // CHECK: Tensor !tfrt.chain + // CHECK: Tensor !tfrt.chain + // CHECK: Tensor !tfrt.chain + + tfrt.return %ch17 : !tfrt.chain +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops_error.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops_error.mlir new file mode 100644 index 0000000000000000000000000000000000000000..0766f8cc2eb7e67cc76a7bed0d3b266cb6ae39cc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tf_ops_error.mlir @@ -0,0 +1,60 @@ +// RUN: tfrt_fallback_translate -mlir-to-bef %s | tf_bef_executor --work_queue_type=mstd --test_init_function=register_op_handlers 2>&1 | FileCheck %s --dump-input=fail + +func.func @register_op_handlers() { + %fallback = "corert.create_kernel_fallback_op_handler"() : () -> !corert.ophandler + corert.register_op_handler %fallback "tfkernel" + tfrt.return +} + +// CHECK-LABEL: --- Running 'test_parse_example_v2_error' +func.func @test_parse_example_v2_error() -> !tfrt_fallback.tf_tensor { + %serialized_th = corert.const_string_tensor {shape = [2, 1], value = ["", ""]} + %names_th = corert.const_string_tensor {shape = [0], value = []} + %dense_keys_th = corert.const_string_tensor {shape = [7], value = ["has_login_page_feature", "num_terms_inside_postform", "num_terms_outside_postform", "num_terms_outside_postform_without_bp", "query_params_contains_url", "title_with_login_phase", "url_contains_login_terms"]} + + %ch = tfrt.new.chain + %ch0 = tfrt_fallback_async.createop(%ch) key(0) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} num_args(0) + %ch1 = tfrt_fallback_async.createop(%ch0) key(1) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor} num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(2) device("CPU:0") "tf.ParseExampleV2"() + {Tdense = [i64, i64, i64, i64, i64, i64, i64], dense_shapes = [#corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>], num_sparse = 0 : i64, ragged_split_types = [], ragged_value_types = [], sparse_types = []} + num_args(12) + + %ch3, %zero = tfrt_fallback_async.executeop.seq(%ch2) key(0) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<0> : tensor} : 1 + %ch4, %one = tfrt_fallback_async.executeop.seq(%ch2) key(1) cost(100) device("CPU:0") "tf.Const"() {dtype = i64, value = dense<1> : tensor} : 1 + + %serialized, %names, %dense_keys = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %serialized_th, %names_th, %dense_keys_th {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + // expected-error @+1 {{Expected serialized to be a scalar or vector, got shape: [2,1]}} + %dense_values:7 = tfrt_fallback_async.executeop key(2) cost(100) device("CPU:0") + "tf.ParseExampleV2"(%serialized, %names, %names, %dense_keys, %names, %zero, %one, %zero, %one, %zero, %one, %zero) + {Tdense = [i64, i64, i64, i64, i64, i64, i64], dense_shapes = [#corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>, #corert.shape<>], num_sparse = 0 : i64, ragged_split_types = [], ragged_value_types = [], sparse_types = []} : 7 + + tfrt.return %dense_values#0: !tfrt_fallback.tf_tensor +} + +// CHECK-LABEL: --- Running 'test_assign_variable_error' +func.func @test_assign_variable_error() -> !tfrt.chain { + %ch0 = tfrt.new.chain + %th0 = corert.const_dense_tensor dense<1> : tensor + %th1 = corert.const_dense_tensor dense<1.0> : tensor + %0, %1 = tfrt_fallback_async.corert_tensorhandle_to_fallback_tensor %th0, %th1 {_tfrt_cost = 1 : i64, device = "CPU:0"} : (!corert.tensorhandle, !corert.tensorhandle) -> (!tfrt_fallback.tf_tensor, !tfrt_fallback.tf_tensor) + + %ch1 = tfrt_fallback_async.createop(%ch0) key(3) device("CPU:0") "tf.VarHandleOp"() {container = "", dtype = f32, shape = #corert.shape<>, shared_name = "x"} num_args(0) + %ch2 = tfrt_fallback_async.createop(%ch1) key(4) device("CPU:0") "tf.AssignVariableOp"() {dtype = f32} num_args(2) + %ch3 = tfrt_fallback_async.createop(%ch2) key(5) device("CPU:0") "tf.AssignVariableOp"() {dtype = i32} num_args(2) + + %ch4, %2 = tfrt_fallback_async.executeop.seq(%ch3) key(3) cost(100) device("CPU:0") "tf.VarHandleOp"() {container = "", dtype = f32, shape = #corert.shape<>, shared_name = "x"} : 1 + %ch5 = tfrt_fallback_async.executeop.seq(%ch4) key(4) cost(100) device("CPU:0") "tf.AssignVariableOp"(%2, %1) {dtype = f32} + // expected-error @+1 {{Trying to assign variable with wrong dtype. Expected float got int32}} + %ch6 = tfrt_fallback_async.executeop.seq(%ch5) key(5) cost(100) device("CPU:0") "tf.AssignVariableOp"(%2, %0) {dtype = i32} + tfrt.return %ch6 : !tfrt.chain +} + +// CHECK-LABEL: --- Running 'test_ref_args' +func.func @test_ref_args() { + %ch0 = tfrt.new.chain + // expected-error @+1 {{Unsupported ref args in VariableV2}} + %ch1 = tfrt_fallback_async.createop(%ch0) key(6) device("CPU:0") "tf.VariableV2"() {container = "", dtype = f32, shape = #corert.shape<>, shared_name = "x"} num_args(0) + %ch2, %0 = tfrt_fallback_async.executeop.seq(%ch1) key(6) cost(100) device("CPU:0") "tf.VariableV2"() {container = "", dtype = f32, shape = #corert.shape<>, shared_name = "x"} : 1 + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tfrt_forwarding.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tfrt_forwarding.mlir new file mode 100644 index 0000000000000000000000000000000000000000..2adaeaef88da2f33eb57b2e8ce8e62d93cf2eaee --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/tfrt_fallback/tfrt_forwarding.mlir @@ -0,0 +1,50 @@ +// RUN: tfrt_translate --mlir-to-bef %s | tf_bef_executor | FileCheck %s + +// CHECK: --- Running 'scalar_add_tfrt_forwarding_test' +func.func @scalar_add_tfrt_forwarding_test() { + %one = tfrt.constant.i32 1 + %two = tfrt.constant.i32 2 + + %tft_a = "tfd.constant_tensor"(%one) : (i32) -> !tfd.tensor + %tft_b = "tfd.constant_tensor"(%two) : (i32) -> !tfd.tensor + + %tft_c = "tfd.forward_kernel"(%tft_a, %tft_b) {op_name = "ScalarAdd"}: (!tfd.tensor, !tfd.tensor) -> !tfd.tensor + + // CHECK: tensor=Tensor + "tfd.print_tensor"(%tft_c) : (!tfd.tensor) -> () + + tfrt.return +} + +// CHECK: --- Running 'failing_kernel_tfrt_forwarding_test' +func.func @failing_kernel_tfrt_forwarding_test() -> !tfd.tensor { + %tft = "tfd.forward_kernel"() {op_name = "FailingKernel"}: () -> !tfd.tensor // expected-error {{runtime error: OP_REQUIRES failed at filename:999 : Internal: TFRT forwarding error!}} + + tfrt.return %tft : !tfd.tensor +} +// CHECK-NEXT: 'failing_kernel_tfrt_forwarding_test' returned <> + +// CHECK: --- Running 'missing_kernel_tfrt_forwarding_test' +func.func @missing_kernel_tfrt_forwarding_test() -> !tfd.tensor { + %tft = "tfd.forward_kernel"() {op_name = "MissingKernel"}: () -> !tfd.tensor // expected-error {{runtime error: Not found: Could not find kernel MissingKernel in the registry.}} + + tfrt.return %tft : !tfd.tensor +} +// CHECK-NEXT: 'missing_kernel_tfrt_forwarding_test' returned <> + + +// CHECK: --- Running 'bool_attr_tfrt_forwarding_test' +func.func @bool_attr_tfrt_forwarding_test() { + + // Note that op_name is prefixed with underscore because op_name must be the + // first attributes when all attributes are sorted by name (b/140896071). + %tft_t = "tfd.forward_kernel"() {_op_name = "KernelWithBoolAttr", attr1_name = "testattr", attr1_value = "bool$true"}: () -> !tfd.tensor + %tft_f = "tfd.forward_kernel"() {_op_name = "KernelWithBoolAttr", attr1_name = "testattr", attr1_value = "bool$false"}: () -> !tfd.tensor + + // CHECK: tensor=Tensor + "tfd.print_tensor"(%tft_t) : (!tfd.tensor) -> () + // CHECK: tensor=Tensor + "tfd.print_tensor"(%tft_f) : (!tfd.tensor) -> () + + tfrt.return +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_fallback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_fallback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..c2842d3535975620076370f7023a8638352c4689 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_fallback.mlir @@ -0,0 +1,82 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-executor-to-tfrt-pipeline="target-gpu=true" -tfrt-lower-tf-savedmodel=hoist-invariant-ops=true %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +func.func private @xla_func_0(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: [[INPUT_0:%.*]] = gpurt.transfer_to_device + // CHECK: [[VAR_0:%.*]] = gpurt.maybe_transfer_variable + // CHECK: tfrt_fallback_async.executeop.seq{{.*}}"tf.XlaLaunch"([[INPUT_0]], [[VAR_0]]) + // CHECK-SAME: {function = "xla_func_0"} + // CHECK: gpurt.transfer_from_device + %2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_0, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + +// Check the case when there are multiple XLA clusters. + +func.func private @xla_func_1(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1 : tensor<1x3xf32> +} + +func.func private @xla_func_2(%arg0: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg0) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @multi_clusters +func.func @multi_clusters(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: [[INPUT_0:%.*]] = gpurt.transfer_to_device + // CHECK: [[VAR_0:%.*]] = gpurt.maybe_transfer_variable + // CHECK: tfrt_fallback_async.executeop.seq{{.*}}"tf.XlaLaunch"([[INPUT_0]], [[VAR_0]]) + // CHECK-SAME: {function = "xla_func_1"} + // CHECK: [[RESULT_1:%.*]] = gpurt.transfer_from_device + %2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_1, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + + // The output of the above XLA cluster is consumed by the below XLA cluster. + // Currently, the output is first transferred back to CPU and then + // transferred to GPU again, which is unnecessary. + // TODO(b/262280565): Remove unnecessary data transfers when there are + // multiple XLA clusters. + // CHECK: [[INPUT_1:%.*]] = gpurt.transfer_to_device [[RESULT_1]] + // CHECK: tfrt_fallback_async.executeop.seq{{.*}}"tf.XlaLaunch"([[INPUT_1]]) + // CHECK-SAME: {function = "xla_func_2"} + // CHECK: gpurt.transfer_from_device + %3 = "tf.XlaLaunch"(%2) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_2, operandSegmentSizes = array} : (tensor<1x3xf32>) -> tensor<*xf32> + + func.return %3 : tensor<*xf32> +} + + +// Check that unused outputs of the XLA cluster are not transferred. + +func.func private @xla_func_3(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> (tensor<1x3xf32>, tensor<1x3xf32>) attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + %2 = "tf.DIV"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1, %2 : tensor<1x3xf32>, tensor<1x3xf32> +} + +// CHECK-LABEL: func @skip_unused_output +func.func @skip_unused_output(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: [[INPUT_0:%.*]] = gpurt.transfer_to_device + // CHECK: [[VAR_0:%.*]] = gpurt.maybe_transfer_variable + // CHECK: tfrt_fallback_async.executeop.seq{{.*}}"tf.XlaLaunch"([[INPUT_0]], [[VAR_0]]) + // CHECK-SAME: {function = "xla_func_3"} + // Since only one output of the XlaLaunch is used, there is only one data transfer. + // CHECK: gpurt.transfer_from_device + // CHECK-NOT: gpurt.transfer_from_device + %2:2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_3, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> (tensor<*xf32>, tensor<*xf32>) + func.return %2#0 : tensor<*xf32> +} + + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_lowering.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_lowering.mlir new file mode 100644 index 0000000000000000000000000000000000000000..f9a072c67e01ae33259e6ca25f5030d2b0041a78 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_launch_lowering.mlir @@ -0,0 +1,18 @@ +// RUN: tf-tfrt-opt -split-input-file -tf-executor-to-tfrt-pipeline="target-gpu=true use-gpu-compile-and-execute-op=true" -tfrt-lower-tf-savedmodel=hoist-invariant-ops=true %s | FileCheck %s --dump-input=fail --dump-input-filter=all + +func.func private @xla_func_0(%arg0: tensor<1x3xf32>, %arg1: tensor<1x3xf32>) -> tensor<1x3xf32> attributes {tf._XlaMustCompile = true, tf._noinline = true, tf._original_func_name = "should_not_be_used"} { + %1 = "tf.AddV2"(%arg0, %arg1) : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<1x3xf32> + func.return %1 : tensor<1x3xf32> +} + +// CHECK-LABEL: func @main +func.func @main(%arg0: tensor<1x3xf32>) -> tensor<*xf32> attributes {tf.entry_function = {control_outputs = "", inputs = "input:0", outputs = "output:0"}} { + %0 = "tf.VarHandleOp"() {device = "/device:CPU:0", container = "", shared_name = "variable"} : () -> tensor>> + %1 = "tf.ReadVariableOp"(%0) {device = "/device:CPU:0"} : (tensor>>) -> tensor<1x3xf32> + // CHECK: gpurt.compile_and_execute + // CHECK-SAME: func_name = "xla_func_0" + // CHECK-SAME: resource_indices = [1] + %2 = "tf.XlaLaunch"(%arg0, %1) {_noinline = true, _xla_compile_device_type = "GPU", device = "/device:GPU:0", function = @xla_func_0, operandSegmentSizes = array} : (tensor<1x3xf32>, tensor<1x3xf32>) -> tensor<*xf32> + func.return %2 : tensor<*xf32> +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_rewrite.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_rewrite.mlir new file mode 100644 index 0000000000000000000000000000000000000000..3d466bf5523db90c1447f32052dbba2daf6f1c90 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tests/xla_rewrite.mlir @@ -0,0 +1,45 @@ +// RUN: tf-tfrt-opt -tfrt-xla-rewrite %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: xla_launch +func.func @xla_launch(%arg: tensor, %v0: tensor<*x!tf_type.resource>, %v1: tensor<*x!tf_type.resource>) -> (tensor) { + %c0 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + %c1 = "tf.Const"() {device = "/device:CPU:0", value = dense<0> : tensor} : () -> tensor + + // CHECK: tf.XlaLaunchV2 + // CHECK-SAME: constants = [0, 3] + // CHECK-SAME: resources = [2, 4] + %r0 = "tf.StatefulPartitionedCall"(%c0, %arg, %v0, %c1, %v1) + {_XlaMustCompile = true, config = "", config_proto = "", + device = "/device:CPU:0", executor_type = "", f = @callee} + : (tensor, tensor, tensor<*x!tf_type.resource>, tensor, tensor<*x!tf_type.resource>) -> tensor + + // CHECK-NOT: tf.XlaLaunchV2 + // CHECK: tf.StatefulPartitionedCall + %r1 = "tf.StatefulPartitionedCall"(%c0, %r0, %v0, %c1, %v1) + {_XlaMustCompile = false, config = "", config_proto = "", + device = "/device:CPU:0", executor_type = "", f = @callee} + : (tensor, tensor, tensor<*x!tf_type.resource>, tensor, tensor<*x!tf_type.resource>) -> tensor + + // CHECK-NOT: tf.XlaLaunchV2 + // CHECK: tf.StatefulPartitionedCall + %r2 = "tf.StatefulPartitionedCall"(%c0, %r1, %v0, %c1, %v1) + {_XlaMustCompile = true, config = "", config_proto = "", + device = "/device:GPU:0", executor_type = "", f = @callee} + : (tensor, tensor, tensor<*x!tf_type.resource>, tensor, tensor<*x!tf_type.resource>) -> tensor + + // CHECK: tf.XlaLaunchV2 + // CHECK-SAME: constants = [0, 3] + // CHECK-SAME: resources = [2, 4] + %r3 = "tf.PartitionedCall"(%c0, %r2, %v0, %c1, %v1) {_XlaMustCompile = true, config = "", config_proto = "", + device = "/device:CPU:0", executor_type = "", f = @callee} + : (tensor, tensor, tensor<*x!tf_type.resource>, tensor, tensor<*x!tf_type.resource>) -> tensor + + func.return %r3 : tensor +} + +func.func @callee(%c0: tensor, %arg: tensor, %v0: tensor<*x!tf_type.resource>, %c1: tensor, %v1: tensor<*x!tf_type.resource>) -> (tensor) { + %0 = "tf.ReadVariableOp"(%v0) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor + %1 = "tf.ReadVariableOp"(%v1) {device = ""} : (tensor<*x!tf_type.resource>) -> tensor + %r = "tf.AddN"(%c0, %c1, %arg, %0, %1) : (tensor, tensor, tensor, tensor, tensor) -> tensor + func.return %r : tensor +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tf-tfrt-opt.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tf-tfrt-opt.cc new file mode 100644 index 0000000000000000000000000000000000000000..95f2d36ba4b496c25bb3a01ec815df8aa09a54c8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tf-tfrt-opt.cc @@ -0,0 +1,79 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/InitAllPasses.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Tools/mlir-opt/MlirOptMain.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "stablehlo/dialect/Register.h" // from @stablehlo +#include "tensorflow/compiler/mlir/init_mlir.h" +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "xla/mlir_hlo/mhlo/IR/hlo_ops.h" +#include "tfrt/init_tfrt_dialects.h" // from @tf_runtime + +int main(int argc, char **argv) { + tensorflow::InitMlir y(&argc, &argv); + + mlir::registerAllPasses(); + mlir::registerInlinerPass(); + + mlir::registerTensorFlowPasses(); + mlir::TFDevice::registerTensorFlowDevicePasses(); + + tensorflow::mlrt_compiler::RegisterMlrtPasses(); + tensorflow::ifrt_serving::RegisterTfIfrtPasses(); + + mlir::PassRegistration([]() { + return std::make_unique(); + }); + + mlir::DialectRegistry registry; + mlir::registerAllDialects(registry); + mlir::RegisterAllTensorFlowDialects(registry); + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + registry.insert(); + tensorflow::RegisterTPUDialects(®istry); + tensorflow::RegisterGpuDialects(®istry); + mlir::stablehlo::registerAllDialects(registry); + + tfrt::RegisterTFRTDialects(registry); + tensorflow::tfrt_compiler::RegisterTPULowerClusterToRuntimeOpsPassPipeline(); + tensorflow::tfrt_compiler:: + RegisterNonTPULowerClusterToRuntimeOpsPassPipeline(); + + return failed( + mlir::MlirOptMain(argc, argv, "TensorFlow TFRT pass driver\n", registry)); +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.cc new file mode 100644 index 0000000000000000000000000000000000000000..1ef360943775562763fa68a4a852a5ecf297f860 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.cc @@ -0,0 +1,29 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.h" + +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_sync.h" + +namespace tensorflow { +namespace tfd { +void RegisterTfrtFallbackDialect(mlir::DialectRegistry ®istry) { + registry.insert(); + registry.insert(); + registry.insert(); +} +} // namespace tfd +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.h new file mode 100644 index 0000000000000000000000000000000000000000..803a554511073e170038185ae9057cf83eb55c75 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.h @@ -0,0 +1,33 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements TFRuntimeFallback tensor conversion function for +// converting to host tensor. + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TFRT_FALLBACK_REGISTRATION_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TFRT_FALLBACK_REGISTRATION_H_ + +#include "mlir/IR/Dialect.h" // from @llvm-project + +namespace tensorflow { +namespace tfd { + +// Register conversion functions for TFRuntimeFallbackTensors. +void RegisterTfrtFallbackDialect(mlir::DialectRegistry ®istry); + +} // namespace tfd +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TFRT_FALLBACK_REGISTRATION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_translate_registration.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_translate_registration.cc new file mode 100644 index 0000000000000000000000000000000000000000..7afcec8d87ead74c2840bd886cda31252eb53f62 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tfrt_fallback_translate_registration.cc @@ -0,0 +1,30 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Tools/mlir-translate/Translation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/tfrt_fallback_registration.h" +#include "tfrt/bef_converter/mlir_to_bef_translate.h" // from @tf_runtime +#include "tfrt/init_tfrt_dialects.h" // from @tf_runtime + +static mlir::TranslateFromMLIRRegistration registration( + "mlir-to-bef", "translate MLIR to BEF", tfrt::MLIRToBEFTranslate, + [](mlir::DialectRegistry& registry) { + mlir::RegisterAllTensorFlowDialects(registry); + tfrt::RegisterTFRTDialects(registry); + tfrt::RegisterTFRTCompiledDialects(registry); + tensorflow::tfd::RegisterTfrtFallbackDialect(registry); + }); diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tools/tfrt_translate/static_registration.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tools/tfrt_translate/static_registration.cc new file mode 100644 index 0000000000000000000000000000000000000000..3789b37e227362a816244c8be7c0bb54cde9c85b --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/tools/tfrt_translate/static_registration.cc @@ -0,0 +1,35 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Tools/mlir-translate/Translation.h" +#include "tfrt/bef_converter/bef_to_mlir_translate.h" // from @tf_runtime +#include "tfrt/bef_converter/mlir_to_bef_translate.h" // from @tf_runtime +#include "tfrt/init_tfrt_dialects.h" // from @tf_runtime + +static mlir::TranslateFromMLIRRegistration mlir_to_bef_registration( + "mlir-to-bef", "translate MLIR to BEF", tfrt::MLIRToBEFTranslate, + [](mlir::DialectRegistry ®istry) { + tfrt::RegisterTFRTDialects(registry); + tfrt::RegisterTFRTCompiledDialects(registry); + }); + +static mlir::TranslateToMLIRRegistration bef_to_mlir_registration( + "bef-to-mlir", "translate BEF to MLIR", + [](llvm::SourceMgr &source_mgr, mlir::MLIRContext *context) { + mlir::DialectRegistry registry; + context->appendDialectRegistry(registry); + return tfrt::BEFToMLIRTranslate(source_mgr, context); + }); diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..f2a29dca9d5e5ed4e7101df625ebc1f51e3400d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.cc @@ -0,0 +1,192 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h" + +#include "mlir/IR/Attributes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OperationSupport.h" +#include "mlir/IR/Types.h" +#include "mlir/Support/LLVM.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tfrt/core_runtime/opdefs/attributes.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +mlir::TypeAttr ConvertTypeAttribute(mlir::TypeAttr type_attr, + mlir::Builder& builder) { + auto type = type_attr.getValue(); + + if (IsSupportedTfrtNumericDType(type)) return type_attr; + + // For TF custom types, we convert it to custom corert types. + if (mlir::isa(type)) + return mlir::TypeAttr::get( + tfrt::corert::StringType::get(builder.getContext())); + + if (mlir::isa(type)) + return mlir::TypeAttr::get( + tfrt::corert::ResourceType::get(builder.getContext())); + + if (mlir::isa(type)) + return mlir::TypeAttr::get( + tfrt::corert::VariantType::get(builder.getContext())); + + if (mlir::isa(type)) { + return mlir::TypeAttr::get( + tfrt::corert::Quint8Type::get(builder.getContext())); + } + + if (mlir::isa(type)) { + return mlir::TypeAttr::get( + tfrt::corert::Quint16Type::get(builder.getContext())); + } + + if (mlir::isa(type)) { + return mlir::TypeAttr::get( + tfrt::corert::Qint8Type::get(builder.getContext())); + } + + if (mlir::isa(type)) { + return mlir::TypeAttr::get( + tfrt::corert::Qint16Type::get(builder.getContext())); + } + + if (mlir::isa(type)) { + return mlir::TypeAttr::get( + tfrt::corert::Qint32Type::get(builder.getContext())); + } + + // Return invalid results to emit error for unsupported types. + return {}; +} + +mlir::Attribute ConvertAttribute(mlir::Attribute attr, mlir::Builder& builder) { + // The supported attributes here should be kept consistent with + // //third_party/tf_runtime/include/tfrt/core_runtime/op_attr_type.h + // + // Currently, not all tensorflow data types are supported. Unranked shape + // attributes are not supported yet. + + // Return directly if the attribute is already supported. + if (mlir::isa(attr)) + return attr; + + // For type attributes, we convert non-standard MLIR types to corresponding + // corert types. + if (auto type_attr = mlir::dyn_cast(attr)) { + if (auto shape_type = + mlir::dyn_cast(type_attr.getValue())) { + if (!shape_type.hasRank()) + return tfrt::corert::ShapeAttr::get(builder.getContext()); + + return tfrt::corert::ShapeAttr::get(builder.getContext(), + shape_type.getShape()); + } + + return ConvertTypeAttribute(type_attr, builder); + } + + // Convert the attribute to the corresponding format in TFRT dialect if + // needed. + if (auto shape_attr = mlir::dyn_cast(attr)) { + if (!shape_attr.hasRank()) + return tfrt::corert::ShapeAttr::get(builder.getContext()); + return tfrt::corert::ShapeAttr::get(builder.getContext(), + shape_attr.getShape()); + } + + // For arrays, we recursively convert the elements. + if (auto array_attr = mlir::dyn_cast(attr)) { + llvm::SmallVector attrs; + attrs.reserve(array_attr.size()); + for (auto attr : array_attr) { + auto converted = ConvertAttribute(attr, builder); + if (!converted) return {}; + attrs.push_back(converted); + } + return builder.getArrayAttr(attrs); + } + + return {}; +} + +} // namespace + +bool IsSupportedTfrtNumericDType(mlir::Type type) { + // Most of the tensorflow data types (eg. f32, i64) are supported and they + // are standard MLIR types that need no conversion here. + if (type.isBF16() || type.isF16() || type.isF32() || type.isF64() || + type.isInteger(1) || type.isInteger(8) || type.isInteger(16) || + type.isInteger(32) || type.isInteger(64) || type.isUnsignedInteger(8) || + type.isUnsignedInteger(16) || type.isUnsignedInteger(32) || + type.isUnsignedInteger(64)) + return true; + + if (auto complex_type = mlir::dyn_cast(type)) { + auto element_type = complex_type.getElementType(); + if (element_type.isF32() || element_type.isF64()) return true; + } + + return false; +} + +// TODO(chky): attributes "_output_shapes" should be removed by any tool that +// generates TF MLIR dialect, as they are not used by CoreRuntime. Remove this +// filtering logic once unused attributes are cleaned up in the upper layer. +bool IsUnusedTfrtAttribute(llvm::StringRef name) { + // NOTE: attributes "f.*" are function attribute related and + // are added during importing graph to MLIR TF Executor dialect. These + // attributes are not actually used by TF ops with function attributes. + // TODO(b/180399811): Re-evaluate the usage of these attributes. + static const char* const kUnusedAttributes[] = { + "_output_shapes", + "resultSegmentSizes", + "operandSegmentSizes", + }; + + for (auto attr : kUnusedAttributes) { + if (name == attr) { + return true; + } + } + + return name.contains("f."); +} + +mlir::ArrayAttr CreateTfrtOpAttrs(llvm::ArrayRef attrs, + mlir::Builder& builder) { + llvm::SmallVector attr_array; + for (auto key_and_value : attrs) { + if (!IsUnusedTfrtAttribute(key_and_value.getName())) { + auto converted = ConvertAttribute(key_and_value.getValue(), builder); + if (!converted) return {}; + + mlir::StringAttr key = + builder.getStringAttr(key_and_value.getName().strref()); + attr_array.push_back(builder.getArrayAttr({key, converted})); + } + } + return builder.getArrayAttr(attr_array); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..84d73c898411eacf11781c3700777df20c56728a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h @@ -0,0 +1,42 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_ATTR_LOWERING_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_ATTR_LOWERING_UTILS_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project + +namespace tensorflow { + +// TODO(chky): attributes "_output_shapes" should be removed by any tool that +// generates TF MLIR dialect, as they are not used by CoreRuntime. Remove this +// filtering logic once unused attributes are cleaned up in the upper layer. +bool IsUnusedTfrtAttribute(llvm::StringRef name); + +// Create a single attribute that contains the named attribute lists. It is an +// array of pairs. The key must be a string attribute, and the value can be +// any attribute that is supported by CoreRuntime. +mlir::ArrayAttr CreateTfrtOpAttrs(llvm::ArrayRef attrs, + mlir::Builder& builder); + +bool IsSupportedTfrtNumericDType(mlir::Type type); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_ATTR_LOWERING_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.cc new file mode 100644 index 0000000000000000000000000000000000000000..d9e9d68811573dd38c89fa1b9ace28d79861cbf9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.cc @@ -0,0 +1,229 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/corert_converter.h" + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Attributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OperationSupport.h" +#include "mlir/IR/Types.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Support/LLVM.h" +#include "mlir/Transforms/DialectConversion.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Interfaces/DerivedAttributeOpInterface.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tfrt/basic_kernels/opdefs/basic_kernels.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tensorflow { + +CoreRTConverter::CoreRTConverter( + mlir::MLIRContext *context, + const mlir::TF::SideEffectAnalysis::Info *side_effect_analysis) + : builder_(context), side_effect_analysis_(*side_effect_analysis) { + addConversion([](tfrt::compiler::ChainType type) { return type; }); + addConversion([](tfrt::corert::OpHandlerType type) { return type; }); + addConversion([](tfrt::corert::TensorHandleType type) { return type; }); + addConversion([=](mlir::TensorType type) -> std::optional { + // Ref types are not supported in both compiler and runtime. + if (mlir::isa(type.getElementType())) + return std::nullopt; + return tensor_handle_type(); + }); + addConversion([=](mlir::Type type) -> std::optional { + if (type == builder_.getI1Type()) return type; + return std::nullopt; + }); +} + +void CoreRTConverter::MaterializeDerivedAttributes(mlir::Operation *op) { + if (auto interface = llvm::dyn_cast(op)) { + auto derived_attrs = interface.materializeDerivedAttributes(); + for (auto named_attr : derived_attrs) { + op->setAttr(named_attr.getName(), named_attr.getValue()); + } + } +} + +mlir::ArrayAttr CoreRTConverter::CreateOpFuncAttrs( + const mlir::SymbolTable &symbol_table, ArrayRef attrs, + llvm::SmallVector *func_attr_keys, + bool use_mlir_func_name) { + llvm::SmallVector attr_array; + for (auto key_and_value : attrs) { + auto attr_key = key_and_value.getName(); + auto attr_value = key_and_value.getValue(); + if (!IsUnusedTfrtAttribute(attr_key) && + mlir::isa(attr_value)) { + auto func_attr = mlir::dyn_cast(attr_value); + auto converted = CanonicalizeTensorflowFunctionName( + symbol_table, func_attr.getValue().str(), use_mlir_func_name); + if (!converted) return {}; + + mlir::StringAttr key = builder_.getStringAttr(attr_key.strref()); + attr_array.push_back( + builder_.getArrayAttr({key, builder_.getStringAttr(*converted)})); + + // Remove the attribute to avoid being converted again. + func_attr_keys->push_back(attr_key); + } + } + return builder_.getArrayAttr(attr_array); +} + +// TODO(chky): Add support for multiple device instances. +std::optional CoreRTConverter::ParseDeviceName( + llvm::StringRef device_name) const { + std::string tf_device_name = device_name.str(); + + if (tf_device_name.empty()) { + return std::nullopt; + } + + ParseDeviceNameResult result; + result.device_name = tf_device_name; + + // Parse the device name in format of the current tensorflow. + DeviceNameUtils::ParsedName parsed_name; + if (!DeviceNameUtils::ParseFullName(result.device_name, &parsed_name)) { + return std::nullopt; + } + if (!parsed_name.has_type) { + return std::nullopt; + } + result.device_type = parsed_name.type; + + result.op_handler_name = tf_device_name; + + return result; +} + +std::optional CoreRTConverter::ParseDeviceName( + mlir::Operation *op) const { + auto device_attr = op->getAttr("device"); + if (!device_attr) { + return std::nullopt; + } + + auto parsed_device_name = + ParseDeviceName(mlir::cast(device_attr).getValue()); + if (!parsed_device_name) op->emitWarning("failed to parse device name."); + return parsed_device_name; +} + +mlir::Value CoreRTConverter::ConvertOpHandler( + mlir::Operation *op, llvm::StringRef op_handler_name, + ConversionPatternRewriter *rewriter) { + auto iter = op_handler_by_name_.find(op_handler_name); + if (iter != op_handler_by_name_.end()) return iter->second; + + mlir::Block *block = op->getBlock(); + ConversionPatternRewriter::InsertionGuard insertion_guard(*rewriter); + rewriter->setInsertionPointToStart(block); + + func::FuncOp func_op = op->getParentOfType(); + mlir::Value in_chain = func_op.getArgument(0); + auto get_op_handler_op = tfrt::corert::GetOpHandler::create( + *rewriter, block->getParent()->getLoc(), op_handler_type(), in_chain, + op_handler_name); + op_handler_by_name_[op_handler_name] = get_op_handler_op.getResult(); + return get_op_handler_op.getResult(); +} + +mlir::Value CoreRTConverter::GetDistributedContext( + mlir::Operation *op, mlir::ConversionPatternRewriter *rewriter) { + mlir::func::FuncOp func_op = op->getParentOfType(); + auto iter = distributed_context_by_func_.find(func_op.getOperation()); + if (iter != distributed_context_by_func_.end()) { + return iter->second; + } + return mlir::Value(); +} + +mlir::Value CoreRTConverter::GetRemoteChainManager( + mlir::Operation *op, mlir::ConversionPatternRewriter *rewriter) { + mlir::func::FuncOp func_op = op->getParentOfType(); + auto iter = remote_chain_mgr_by_func_.find(func_op.getOperation()); + if (iter != remote_chain_mgr_by_func_.end()) { + return iter->second; + } + return mlir::Value(); +} + +mlir::Value CoreRTConverter::GetLocalSideEffectChain( + mlir::Operation *op, mlir::ConversionPatternRewriter *rewriter) { + auto func_op = op->getParentOfType(); + + llvm::SmallVector predecessors; + if (llvm::isa(op)) { + auto sinks = side_effect_analysis_.ControlSinks(); + predecessors.assign(sinks.begin(), sinks.end()); + } else { + predecessors = side_effect_analysis_.DirectControlPredecessors(op); + } + + llvm::SmallVector chains; + for (auto *pred : predecessors) { + // TODO(chky): ReadVariableOp is removed in the pass and not converted. + // Ideally, every side-effecting op should be converted to a + // tfrt_fallback.executeop.seq op. The special rewrite logic of + // ReadVariableOp should be done in a previous pass. + if (auto chain = local_side_effect_chains_.lookup(pred)) + chains.push_back(chain); + } + + // If there is no side-effect predecessor, then the input side-effect chain + // is used. + if (chains.empty()) return func_op.getArgument(0); + + if (chains.size() == 1) return chains[0]; + + // If there are multiple side-effect predecessors, insert a merge_chains + // kernel and return the merged chain. + ConversionPatternRewriter::InsertionGuard insertion_guard(*rewriter); + rewriter->setInsertionPoint(op); + return tfrt::compiler::MergeChainsOp::create(*rewriter, op->getLoc(), + chain_type(), chains); +} + +mlir::Value CoreRTConverter::GetTaskHandle( + mlir::Operation *op, StringRef task_name, + mlir::ConversionPatternRewriter *rewriter) { + mlir::func::FuncOp func_op = op->getParentOfType(); + llvm::StringMap &task_handle_by_name = + task_handles_by_func_[func_op.getOperation()]; + auto iter = task_handle_by_name.find(task_name); + if (iter != task_handle_by_name.end()) { + return iter->second; + } + + return mlir::Value(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.h new file mode 100644 index 0000000000000000000000000000000000000000..578e02017bfbd3936a40d4dbaeaa7c27dfc8488a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/corert_converter.h @@ -0,0 +1,175 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_CORERT_CONVERTER_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_CORERT_CONVERTER_H_ + +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tensorflow { + +struct ParseDeviceNameResult { + std::string device_type; + std::string device_name; + std::string op_handler_name; +}; + +// A helper class for converting CoreRT types and attributes. +class CoreRTConverter : public mlir::TypeConverter { + public: + CoreRTConverter( + mlir::MLIRContext *context, + const mlir::TF::SideEffectAnalysis::Info *side_effect_analysis); + // Materialize all derived attributes. Note that this is only needed by + // CoreRT ops and fallback ops. + void MaterializeDerivedAttributes(mlir::Operation *op); + + // Similar to CreateOpAttrs, create a single attribute that contains the + // named attribute lists, which is an array of pairs, with keys and values + // both being string attributes. The values represent function names. + // This method also populates a vector of attribute keys to be removed. + // If `use_mlir_func_name` is true, the function name given by MLIR will be + // used, which could be different from the original function name in the graph + // function library. This is used when the original function has been changed + // by lowering passes, and hence it needs to be exported to function library + // for runtime to use. + mlir::ArrayAttr CreateOpFuncAttrs( + const mlir::SymbolTable &symbol_table, + llvm::ArrayRef attrs, + llvm::SmallVector *func_attr_keys, + bool use_mlir_func_name = false); + + // Parse the device name of `op` to TFRT's device name. For example, "/CPU:0" + // will be parsed as "cpu". Return None if no device is assigned. + std::optional ParseDeviceName( + llvm::StringRef device_name) const; + std::optional ParseDeviceName( + mlir::Operation *op) const; + + // Convert the device name in a TF op to a op_handler value produced by the + // corresponding GetOpHandler in the current block. If there does not exist + // one, insert a GetOpHandler to the beginning of the block and return the + // device value. + mlir::Value ConvertOpHandler(mlir::Operation *op, llvm::StringRef device_name, + mlir::ConversionPatternRewriter *rewriter); + + // Get a DistributedContext value to be used by the given op. The + // DistributedContext value should be shared by all operations in the body + // of the same FuncOp. If there does not exist one, return a null Value. + mlir::Value GetDistributedContext(mlir::Operation *op, + mlir::ConversionPatternRewriter *rewriter); + + // Get a RemoteChainManager value to be used by the given op. The + // RemoteChainManager value should be shared by all operations in the body + // of the same FuncOp. If there does not exist one, return a null Value. + mlir::Value GetRemoteChainManager(mlir::Operation *op, + mlir::ConversionPatternRewriter *rewriter); + + // Get a TaskHandle value with the given task name. If the TaskHandle value + // has already been created for the given task name within the same FuncOp, + // return this TaskHandle value. Otherwise, return a null Value. + mlir::Value GetTaskHandle(mlir::Operation *op, StringRef task_name, + mlir::ConversionPatternRewriter *rewriter); + + // Any local operation which uses any result of the `op` should depend on the + // given `chain`. + void RegisterLocalSideEffectChain(mlir::Operation *op, mlir::Value chain) { + local_side_effect_chains_[op] = chain; + } + + // Return a local chain for side effects for `op`. If there are multiple + // chains, a merge_chains kernel will be inserted and the merged chain will be + // returned. + mlir::Value GetLocalSideEffectChain( + mlir::Operation *op, mlir::ConversionPatternRewriter *rewriter); + + mlir::Type op_handler_type() { + return builder_.getType<::tfrt::corert::OpHandlerType>(); + } + + mlir::Type tensor_handle_type() { + return builder_.getType<::tfrt::corert::TensorHandleType>(); + } + + mlir::Type chain_type() { + return builder_.getType<::tfrt::compiler::ChainType>(); + } + + mlir::Builder &builder() { return builder_; } + + private: + // TODO(chky): attributes "_output_shapes" should be removed by any tool that + // generates TF MLIR dialect, as they are not used by CoreRuntime. Remove this + // filtering logic once unused attributes are cleaned up in the upper layer. + bool IsUnusedAttribute(llvm::StringRef name) const { + // NOTE: attributes "f.*" are function attribute related and + // are added during importing graph to MLIR TF Executor dialect. These + // attributes are not actually used by TF ops with function attributes. + // TODO(b/180399811): Re-evaluate the usage of these attributes. + static const char *const kUnusedAttributes[] = { + "_output_shapes", + "result_segment_sizes", + "operand_segment_sizes", + }; + + for (auto attr : kUnusedAttributes) { + if (name == attr) { + return true; + } + } + + return name.contains("f."); + } + + // Returns the converted attribute in TFRT dialect. If the conversion fails, + // returns a null attribute instead. + mlir::Attribute ConvertAttribute(mlir::Attribute attr); + + mlir::TypeAttr ConvertTypeAttribute(mlir::TypeAttr type_attr); + + mlir::Builder builder_; + + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis_; + + llvm::DenseMap local_side_effect_chains_; + llvm::DenseMap distributed_context_by_func_; + llvm::DenseMap remote_chain_mgr_by_func_; + llvm::DenseMap> + task_handles_by_func_; + llvm::StringMap op_handler_by_name_; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_CORERT_CONVERTER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/cross_device_transfer.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/cross_device_transfer.cc new file mode 100644 index 0000000000000000000000000000000000000000..6f0554752ff9bedf2b04458164ce6042b6dfef7e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/cross_device_transfer.cc @@ -0,0 +1,201 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass inserts corert.transfer op to make sure any argument of any op is +// on the same device of the op itself. + +#include +#include +#include + +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/core/util/device_name_utils.h" +#include "tfrt/basic_kernels/opdefs/basic_kernels.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tensorflow { + +namespace { + +using DeviceNameUtils = ::tensorflow::DeviceNameUtils; + +constexpr const char *kDeviceAttr = "device"; +constexpr const char *kTFRTDeviceAttr = "tfrt.device"; +// TODO(b/175480458): Do not assign default device once every op in the TF +// dialect has the device attribute. +constexpr const char *kDefaultDevice = + "/job:localhost/replica:0/task:0/device:CPU:0"; + +// This method canonicalizes the device name so that we can use string +// comparison to see if two devices are the same. It does the following +// transformations: +// 1) Set device ID to 0 if device ID is not already specified. +// 2) Change the device type to uppercase string. +static std::string CanonicalizeDeviceName(const std::string &device) { + if (device.empty()) return kDefaultDevice; + + DeviceNameUtils::ParsedName parsed_name; + if (!device.empty() && device.at(0) == '/') { + DeviceNameUtils::ParseFullName(device, &parsed_name); + } else { + DeviceNameUtils::ParseFullName("/device:" + device, &parsed_name); + } + + if (!parsed_name.has_id) { + parsed_name.has_id = true; + parsed_name.id = 0; + } + + if (parsed_name.type == "cpu") + parsed_name.type = "CPU"; + else if (parsed_name.type == "gpu") + parsed_name.type = "GPU"; + else if (parsed_name.type == "tpu") + parsed_name.type = "TPU"; + return DeviceNameUtils::ParsedNameToString(parsed_name); +} + +// Return the device of the given operation. +static std::string GetDevice(Operation *op) { + std::string device = ""; + if (StringAttr device_attr = op->getAttrOfType(kDeviceAttr)) { + device = device_attr.getValue().str(); + } else if (auto execute_op = llvm::dyn_cast(op)) { + SmallVector, 4> attrs; + execute_op.getOpAttrs(&attrs); + for (std::pair entry : attrs) { + if (entry.first == kDeviceAttr && mlir::isa(entry.second)) { + device = mlir::cast(entry.second).getValue().str(); + break; + } + } + } + + return CanonicalizeDeviceName(device); +} + +// Return the device of the given value. +static std::string GetDevice(mlir::Value value, func::FuncOp parent_func_op) { + std::string device = ""; + if (BlockArgument block_arg = mlir::dyn_cast(value)) { + if (StringAttr device_attr = parent_func_op.getArgAttrOfType( + block_arg.getArgNumber(), kTFRTDeviceAttr)) { + device = device_attr.getValue().str(); + } + } else { + device = GetDevice(value.getDefiningOp()); + } + + return CanonicalizeDeviceName(device); +} + +struct CrossDeviceTransferPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CrossDeviceTransferPass) + + void runOnOperation() override; + + llvm::StringRef getArgument() const final { + return "tfrt-cross-device-transfer"; + } + + llvm::StringRef getDescription() const final { + return "This pass inserts corert.transfer op to make sure any argument of " + "any op is on the same device of the op itself."; + } +}; + +void CrossDeviceTransferPass::runOnOperation() { + func::FuncOp func_op = getOperation(); + llvm::DenseMap> + transferred_value_by_value_and_device; + + func_op.getBody().walk([&](Operation *op) { + if (op->hasTrait()) return WalkResult::advance(); + // Do not transfer the argument of corert.transfer op. + if (llvm::isa(op)) return WalkResult::advance(); + + OpBuilder builder(op); + std::string dst_device = GetDevice(op); + mlir::Type tensor_type_type = + builder.getType<::tfrt::compiler::TensorTypeType>(); + mlir::Type device_type = builder.getType<::tfrt::compiler::DeviceType>(); + + for (mlir::Value arg : op->getOperands()) { + // Do not transfer non-TensorHandle values. + if (!mlir::isa(arg.getType())) continue; + + // Do not transfer the result of corert.transfer op. + if (OpResult op_result = mlir::dyn_cast(arg)) { + Operation *defining_op = arg.getDefiningOp(); + if (llvm::isa(defining_op)) continue; + } + + std::string src_device = GetDevice(arg, func_op); + + if (DeviceNameUtils::LocalName(src_device) == + DeviceNameUtils::LocalName(dst_device)) + continue; + + // Re-use the value already transferred to the given device. + llvm::StringMap &transferred_value_by_device = + transferred_value_by_value_and_device[arg]; + auto iter = transferred_value_by_device.find(dst_device); + if (iter != transferred_value_by_device.end()) { + op->replaceUsesOfWith(arg, iter->second); + continue; + } + + mlir::Value chain_in = func_op.getArgument(0); + auto get_device_op = tfrt::compiler::GetDeviceOp::create( + builder, op->getLoc(), device_type, chain_in, dst_device); + auto get_tensor_type_op = tfrt::corert::GetDstTensorTypeOp::create( + builder, op->getLoc(), tensor_type_type, arg, + get_device_op.getResult()); + auto transfer_op = tfrt::corert::TransferOp::create( + builder, op->getLoc(), arg.getType(), arg, get_device_op.getResult(), + get_tensor_type_op.getResult()); + mlir::Value new_arg = transfer_op.getResult(); + transferred_value_by_device[dst_device] = new_arg; + op->replaceUsesOfWith(arg, new_arg); + } + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> CreateCrossDeviceTransferPass() { + return std::make_unique(); +} + +static PassRegistration pass; + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_batch_function.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_batch_function.cc new file mode 100644 index 0000000000000000000000000000000000000000..e06451cce9c6ac5a720118d366a78dfbdfc5406c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_batch_function.cc @@ -0,0 +1,177 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +using ::mlir::ArrayRef; +using ::mlir::ModuleOp; +using ::mlir::Operation; +using ::mlir::SymbolTable; +using ::mlir::SymbolTableCollection; +using ::mlir::SymbolUserMap; + +// This only includes some preliminary checks as this is a short term solution. +bool AreEquivalent(mlir::func::FuncOp& lhs, mlir::func::FuncOp& rhs) { + if (lhs.getFunctionType() != rhs.getFunctionType()) return false; + + for (auto arg_pair : llvm::zip(lhs.getArguments(), rhs.getArguments())) { + auto& lhs_arg = std::get<0>(arg_pair); + auto& rhs_arg = std::get<1>(arg_pair); + if (lhs_arg.getType() != rhs_arg.getType()) return false; + } + + auto lhs_ops = lhs.getBody().getOps(); + auto rhs_ops = rhs.getBody().getOps(); + if (std::distance(lhs_ops.begin(), lhs_ops.end()) != + std::distance(rhs_ops.begin(), rhs_ops.end())) + return false; + + for (auto op_pair : llvm::zip(lhs_ops, rhs_ops)) { + auto& lhs_op = std::get<0>(op_pair); + auto& rhs_op = std::get<1>(op_pair); + if (lhs_op.getName() != rhs_op.getName()) return false; + if (lhs_op.getNumRegions() != rhs_op.getNumRegions()) return false; + if (lhs_op.getNumSuccessors() != rhs_op.getNumSuccessors()) return false; + if (!std::equal(lhs_op.getOperandTypes().begin(), + lhs_op.getOperandTypes().end(), + rhs_op.getOperandTypes().begin())) + return false; + if (!std::equal(lhs_op.getResultTypes().begin(), + lhs_op.getResultTypes().end(), + rhs_op.getResultTypes().begin())) + return false; + } + + return true; +} + +// Deduplicate the functions if all users are BatchFunctionOp and have the same +// shared_name. +// +// TODO(b/192463730): this is the short term solution and not needed anymore +// after the shape inference pass is revamped with ideal solution +// (b/192463730#comment11). +class DeduplicateFunctionsInovkedByBatchFunction + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + DeduplicateFunctionsInovkedByBatchFunction) + + private: + llvm::StringRef getArgument() const final { + return "tfrt-deduplicate-functions-invoked-by-batch-function"; + } + llvm::StringRef getDescription() const final { + return "Deduplicate the functions invoked by tf.BatchFunction with the " + "same shared_name"; + } + void runOnOperation() override { + if (failed(Run())) { + signalPassFailure(); + } + } + + mlir::LogicalResult Run(); +}; + +mlir::LogicalResult DeduplicateFunctionsInovkedByBatchFunction::Run() { + ModuleOp module = getOperation(); + SymbolTableCollection symbol_table_collection; + SymbolTable& symbol_table = symbol_table_collection.getSymbolTable(module); + SymbolUserMap symbol_users(symbol_table_collection, module); + + // Categorize the functions invoked by BatchFunctionOp by its shared_name. + llvm::StringMap> + shared_name_to_func_ops; + + for (auto func : + llvm::make_early_inc_range(module.getOps())) { + ArrayRef users = symbol_users.getUsers(func); + llvm::StringRef shared_name; + // Deduplicate the function only if all users are BatchFunctionOp and have + // the same shared_name + if (!users.empty() && llvm::all_of(users, [&shared_name](Operation* user) { + auto op = llvm::dyn_cast_or_null(user); + // User is not a BatchFunctionOp + if (!op) return false; + if (shared_name.empty()) { + shared_name = op.getSharedName(); + return true; + } + return shared_name == op.getSharedName(); + })) { + shared_name_to_func_ops[shared_name].push_back(func); + } + } + + for (auto& it : shared_name_to_func_ops) { + auto& func_ops = it.second; + mlir::func::FuncOp& func_op_to_keep = func_ops.front(); + for (mlir::func::FuncOp& func_op_to_remove : llvm::drop_begin(func_ops)) { + if (!AreEquivalent(func_op_to_keep, func_op_to_remove)) { + return func_op_to_remove.emitError( + "func_ops for BatchFunctionOp with the same shared name are " + "different"); + } + if (failed(SymbolTable::replaceAllSymbolUses( + func_op_to_remove, func_op_to_keep.getSymNameAttr(), module))) { + return func_op_to_remove.emitError("unable to replace the symbol use"); + } + symbol_table.erase(func_op_to_remove); + } + } + + return mlir::success(); +} +} // namespace + +std::unique_ptr> +CreateDeduplicateFunctionsInovkedByBatchFunctionPass() { + return std::make_unique(); +} + +static mlir::PassRegistration + register_pass; + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_if_result_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_if_result_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..40b3d0075a0b88f0a70e6864980f5f04dd6f0f7e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/deduplicate_if_result_pass.cc @@ -0,0 +1,232 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +struct ResultMapping { + bool has_duplicate = false; + llvm::SmallVector old_to_new; + llvm::SmallVector new_to_old; +}; + +ResultMapping FindDuplicateResultIndices(mlir::func::FuncOp func) { + llvm::SmallDenseMap results; + auto &block = func.getBody().front(); + + auto return_op = llvm::cast(block.getTerminator()); + + ResultMapping mapping; + mapping.old_to_new.reserve(return_op.getNumOperands()); + + int j = 0; + for (int i = 0; i < return_op.getNumOperands(); ++i) { + auto value = return_op.getOperand(i); + auto iter = results.find(value); + if (iter != results.end()) { + mapping.has_duplicate = true; + mapping.old_to_new.push_back(iter->second); + } else { + results[value] = j; + mapping.old_to_new.push_back(j++); + } + } + + mapping.new_to_old.resize(j); + for (int i = 0; i < mapping.old_to_new.size(); ++i) { + mapping.new_to_old[mapping.old_to_new[i]] = i; + } + + return mapping; +} + +mlir::func::FuncOp CreateBranchFunctionWithDeduplicatedResults( + mlir::OpBuilder &builder, mlir::Location loc, absl::string_view name, + mlir::func::FuncOp func, const ResultMapping &mapping) { + auto arg_types = func.getFunctionType().getInputs(); + auto result_types = func.getFunctionType().getResults(); + + llvm::SmallVector new_result_types; + new_result_types.reserve(mapping.new_to_old.size()); + + for (int i : mapping.new_to_old) { + new_result_types.push_back(result_types[i]); + } + + auto new_func_type = mlir::FunctionType::get(builder.getContext(), arg_types, + new_result_types); + + auto new_func = mlir::func::FuncOp::create(builder, loc, name, new_func_type); + new_func.setVisibility(mlir::func::FuncOp::Visibility::Private); + + mlir::OpBuilder::InsertionGuard guard(builder); + + // In the body of newly created function, we insert + // tf.PartitionedCall ops to call the original func. + auto *block = new_func.addEntryBlock(); + builder.setInsertionPointToStart(block); + auto empty_string_attr = builder.getStringAttr(""); + + llvm::SmallVector results; + results.reserve(new_func_type.getNumResults()); + + // Create the call op to the original func. The arguments are simply + // the arguments from the wrapper function. + auto call_op = mlir::TF::PartitionedCallOp::create( + builder, loc, result_types, block->getArguments(), /*args_attrs=*/nullptr, + /*res_attrs=*/nullptr, + mlir::FlatSymbolRefAttr::get(func.getSymNameAttr()), empty_string_attr, + empty_string_attr, empty_string_attr); + + for (int i : mapping.new_to_old) { + results.push_back(call_op.getResult(i)); + } + + mlir::func::ReturnOp::create(builder, loc, results); + + return new_func; +} + +void DeduplicateIfOps(mlir::ModuleOp module) { + mlir::SymbolTable symbol_table(module); + + llvm::DenseMap visited; + llvm::DenseMap new_funcs; + llvm::SmallVector candidates; + + module.walk([&](mlir::TF::IfOp op) { candidates.push_back(op); }); + + llvm::SmallVector to_erase; + + mlir::OpBuilder builder(module.getRegion()); + for (auto op : candidates) { + auto find_mapping = [&](mlir::func::FuncOp func) { + if (auto iter = visited.find(func); iter != visited.end()) { + return iter->second; + } + + auto mapping = FindDuplicateResultIndices(func); + visited[func] = mapping; + return mapping; + }; + + auto then_branch = + symbol_table.lookup(op.getThenBranch()); + auto else_branch = + symbol_table.lookup(op.getElseBranch()); + + auto then_mapping = find_mapping(then_branch); + auto else_mapping = find_mapping(else_branch); + + if (then_mapping.has_duplicate && + then_mapping.old_to_new == else_mapping.old_to_new) { + auto get_or_create = [&](mlir::func::FuncOp func, + const ResultMapping &mapping) { + auto iter = new_funcs.find(func); + if (iter != new_funcs.end()) { + return iter->second; + } + + auto new_func = CreateBranchFunctionWithDeduplicatedResults( + builder, op.getLoc(), + absl::StrCat(func.getSymName().str(), "/tfrt_dedup_results"), func, + mapping); + new_funcs[func] = new_func; + return new_func; + }; + auto new_then_func = get_or_create(then_branch, then_mapping); + auto new_else_func = get_or_create(else_branch, else_mapping); + + mlir::OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPoint(op); + + llvm::SmallVector new_result_types; + for (int i : then_mapping.new_to_old) { + new_result_types.push_back(op->getResult(i).getType()); + } + + auto new_if_op = mlir::TF::IfOp::create( + builder, op.getLoc(), new_result_types, op.getCond(), op.getInput(), + new_then_func.getSymName(), new_else_func.getSymName(), + op.getIsStateless()); + + DCHECK_EQ(then_mapping.old_to_new.size(), op.getNumResults()); + for (int i = 0; i < then_mapping.old_to_new.size(); ++i) { + op.getResult(i).replaceAllUsesWith( + new_if_op.getResult(then_mapping.old_to_new[i])); + } + to_erase.push_back(op); + } + } + + for (auto op : to_erase) { + op->erase(); + } +} + +class DeduplicateIfResultPass + : public mlir::PassWrapper> { + llvm::StringRef getArgument() const final { + return "tfrt-deduplicate-if-result"; + } + llvm::StringRef getDescription() const final { + return "Deduplicate the results of tf.If ops"; + } + + void runOnOperation() override { DeduplicateIfOps(getOperation()); } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(DeduplicateIfResultPass) +}; + +} // namespace + +std::unique_ptr> +CreateDeduplicateIfResultPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateDeduplicateIfResultPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.cc new file mode 100644 index 0000000000000000000000000000000000000000..19af749b92efa1b0e81bb964e963299826abb375 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.cc @@ -0,0 +1,162 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.h" + +#include + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime + +namespace tensorflow { +namespace tfrt_compiler { + +FallbackConverter::FallbackConverter(mlir::MLIRContext *context) + : builder_(context) { + addConversion([](tfrt::compiler::ChainType type) { return type; }); + addConversion([](tfrt::fallback::TFTensorType type) { return type; }); + addConversion([=](mlir::TensorType type) -> std::optional { + // Ref types are not supported in both compiler and runtime. + if (mlir::isa(type.getElementType())) { + return std::nullopt; + } + + return builder_.getType(); + }); + addConversion([=](mlir::Type type) -> std::optional { + if (type == builder_.getI1Type()) return type; + return std::nullopt; + }); +} + +mlir::Value ConvertCoreRTTensorHandleToFallbackTensor( + mlir::Location loc, llvm::StringRef device, mlir::Value value, + mlir::ConversionPatternRewriter &rewriter) { + if (mlir::isa(value.getType())) return value; + + if (!mlir::isa(value.getType())) return {}; + + mlir::OpBuilder::InsertionGuard guard(rewriter); + + if (device.ends_with("CPU:0") && !device.starts_with("/job:")) { + // Canonicalize CPU device name. This is needed as corert library only uses + // the default CPU device name (i.e. + // "/job:localhost/replica:0/task:0/device:CPU:0") and cannot recoganize + // other legal variants (e.g. "/device:CPU:0"). + // + // Note that we don't want to make change to the device name if it is + // already canonicalized by users. + // e.g. "/job:tpu_worker/replica:0/task:x/device:CPU:0". + // TODO(tfrt-devs): to make the canonicalization more robust we should + // introduce a util to check each component of the TF device name. + device = GetDefaultCpuDeviceName(); + } + + auto *def = value.getDefiningOp(); + if (def) { + rewriter.setInsertionPointAfter(def); + } else { + rewriter.setInsertionPointToStart(value.getParentBlock()); + } + + return tfrt::fallback_async::CoreRTTensorHandleToFallbackTensorOp::create( + rewriter, loc, rewriter.getType(), + value, device) + .getResult(0); +} + +mlir::Value ConvertFallbackTensorToCoreRTTensorHandle( + mlir::Location loc, mlir::Value value, + mlir::ConversionPatternRewriter &rewriter) { + if (mlir::isa(value.getType())) return value; + + if (!mlir::isa(value.getType())) return {}; + + // Use CPU device by default if no device is specified. + llvm::StringRef device = GetDefaultCpuDeviceName(); + if (auto *def = value.getDefiningOp()) { + if (auto device_attr = def->getAttrOfType("device")) { + // NOTE: The TPU_SYSTEM check is just a short term workaround. The long + // term solution should be checking the HostMemory annotation of the + // defining op (it should be defined in TF OpKernel). If HostMemory + // annotation is set for an output tensor, we should use CPU device here. + // TODO(b/200896904): Support HostMemory annotation. + if (!device_attr.getValue().ends_with("TPU_SYSTEM:0")) { + device = device_attr.getValue(); + } + } + } + + return tfrt::fallback_async::FallbackTensorToCoreRTTensorHandleOp::create( + rewriter, loc, rewriter.getType(), + value, device) + .getResult(0); +} + +mlir::LogicalResult ConvertCoreRTOperands( + mlir::Operation *op, mlir::ValueRange operands, + llvm::SmallVectorImpl *new_operands, + mlir::ConversionPatternRewriter &rewriter) { + mlir::OpBuilder::InsertionGuard guard(rewriter); + // Insert before the current op. + rewriter.setInsertionPoint(op); + + for (auto operand : operands) { + auto value = ConvertFallbackTensorToCoreRTTensorHandle(op->getLoc(), + operand, rewriter); + if (!value) { + return op->emitWarning("failed to convert to !corert.tensorhandle") + << operand.getType(); + } + + new_operands->push_back(value); + } + return success(); +} + +mlir::LogicalResult ConvertFallbackOperands( + mlir::Operation *op, llvm::StringRef device, mlir::ValueRange operands, + llvm::SmallVectorImpl *new_operands, + mlir::ConversionPatternRewriter &rewriter) { + for (auto operand : operands) { + if (!mlir::isa(operand.getType())) { + auto new_operand = ConvertCoreRTTensorHandleToFallbackTensor( + op->getLoc(), device, operand, rewriter); + if (!new_operand) + return op->emitWarning( + "failed to convert the operand to fallback tensor."); + new_operands->push_back(new_operand); + } else { + new_operands->push_back(operand); + } + } + return success(); +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.h new file mode 100644 index 0000000000000000000000000000000000000000..bb88130112c7bbdab8985a31fce4c61a1766092d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.h @@ -0,0 +1,96 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_FALLBACK_CONVERTER_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_FALLBACK_CONVERTER_H_ + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project + +namespace tensorflow { +namespace tfrt_compiler { + +inline llvm::StringRef GetDefaultCpuDeviceName() { + static constexpr char kCpuDeviceName[] = + "/job:localhost/replica:0/task:0/device:CPU:0"; + return kCpuDeviceName; +} + +class FallbackConverter : public mlir::TypeConverter { + public: + explicit FallbackConverter(mlir::MLIRContext *context); + + // Return the next dense key for fallback ops. The key is simply an array + // index so that in runtime, the fallback ops can be efficiently retrieved. + int64_t GetNextFallbackKey() const { return fallback_ops_.size(); } + + void RegisterFallbackOp(mlir::Operation *op) { fallback_ops_.push_back(op); } + + void ReplaceFallbackOp(int64_t key, mlir::Operation *op) { + fallback_ops_[key] = op; + } + + llvm::ArrayRef GetFallbackOps() const { + return fallback_ops_; + } + + private: + mlir::Builder builder_; + // Using a vector to keep fallback ops in order, and the key for a fallback op + // is its corresponding index here. + llvm::SmallVector fallback_ops_; +}; + +// Convert the `value` that is a !corert.tensorhandle to +// !tfrt_fallback.tf_tensor. If needed, tensor conversion kernels will be added. +// On error it returns nullptr. +mlir::Value ConvertCoreRTTensorHandleToFallbackTensor( + mlir::Location loc, llvm::StringRef device, mlir::Value value, + mlir::ConversionPatternRewriter &rewriter); + +// Convert the `value` that is a !tfrt_fallback.tf_tensor to +// !corert.tensorhandle. If needed, tensor conversion kernels will be added. On +// error it returns nullptr. +mlir::Value ConvertFallbackTensorToCoreRTTensorHandle( + mlir::Location loc, mlir::Value value, + mlir::ConversionPatternRewriter &rewriter); + +// Convert operands that might be !tfrt_fallback.tf_tensor for corert operations +// that take only !corert.tensorhandle. +mlir::LogicalResult ConvertCoreRTOperands( + mlir::Operation *op, mlir::ValueRange operands, + llvm::SmallVectorImpl *new_operands, + mlir::ConversionPatternRewriter &rewriter); + +// Convert operands that might be !corert.tensorhandle for fallback operations +// that take only !tfrt_fallback.tf_tensor. +mlir::LogicalResult ConvertFallbackOperands( + mlir::Operation *op, llvm::StringRef device, mlir::ValueRange operands, + llvm::SmallVectorImpl *new_operands, + mlir::ConversionPatternRewriter &rewriter); + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_FALLBACK_CONVERTER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fuse_tpu_compile_and_execute_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fuse_tpu_compile_and_execute_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..074115b03c35b5aa39ea503018f2c7e17ef6f3ea --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/fuse_tpu_compile_and_execute_ops.cc @@ -0,0 +1,282 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace tensorflow { +namespace { + +void RecursivelyMoveOp(mlir::TF::_TPUCompileMlirOp compile_op, + mlir::Operation *op_candidate, + llvm::SmallDenseSet *ops_to_move) { + if (!op_candidate || !ops_to_move->contains(op_candidate)) return; + // Move the parent first. + for (const auto &operand : op_candidate->getOperands()) { + RecursivelyMoveOp(compile_op, operand.getDefiningOp(), ops_to_move); + } + op_candidate->moveBefore(compile_op); + // Erase the op to avoid moving the common ancestor. + ops_to_move->erase(op_candidate); +} + +// Move exec_op's args defining ops before the compile op to group compile op +// and execute op. +void GroupCompileOpAndExecuteOp(mlir::func::FuncOp func, + mlir::TF::_TPUCompileMlirOp compile_op, + mlir::TF::TPUExecuteOp exec_op) { + // Collect the ops between compile op and execute op. + llvm::SmallDenseSet ops_to_move; + bool collect = false; + func.walk( + [&ops_to_move, &collect, &compile_op, &exec_op](mlir::Operation *op) { + if (collect) ops_to_move.insert(op); + if (op == compile_op.getOperation()) collect = true; + return op == exec_op.getOperation() ? mlir::WalkResult::interrupt() + : mlir::WalkResult::advance(); + }); + // Recursively move the defining op of the execute op argument in front of the + // compile op such that the compile op and execute op are grouped together. + for (const auto &operand : exec_op.getArgs()) { + RecursivelyMoveOp(compile_op, operand.getDefiningOp(), &ops_to_move); + } +} + +mlir::Value FindFullShapedOperand(mlir::TF::SplitOp split_op) { + auto operand = split_op.getOperand(1); + auto defining_op = operand.getDefiningOp(); + if (!defining_op) { + return operand; + } + if (auto parent_split_op = llvm::dyn_cast(defining_op)) + return FindFullShapedOperand(parent_split_op); + + return operand; +} + +bool MaybeFindUsedExecuteOp( + const llvm::SmallVector &exec_op_in_group, + mlir::TF::TPUExecuteOp &used_exec_op) { + // TODO(b/218763089): Here we assume the TPU output is replicated on all + // cores. Once the b/218763089 is fixed in MLIR Bridge that it properly + // supports output being sharded on TPU cores, then the results from all + // of the TPUExecuteOp are used. + bool found_used_exec_op = false; + for (auto exec_op : exec_op_in_group) { + if (!exec_op.use_empty()) { + if (found_used_exec_op) { + exec_op.emitOpError( + "More than 1 TPUExecuteOp has dependencies for the same " + "_TPUCompileMlirOp"); + return false; + } + used_exec_op = exec_op; + found_used_exec_op = true; + } + } + if (!exec_op_in_group.empty() && !found_used_exec_op) { + used_exec_op = exec_op_in_group[0]; + found_used_exec_op = true; + } + return found_used_exec_op; +} + +void FuseCompileAndExecuteOps( + mlir::TF::_TPUCompileMlirOp &compile_op, + const llvm::SmallVector &exec_op_in_group, + mlir::TF::TPUExecuteOp &used_exec_op, + llvm::SmallDenseMap< + mlir::TF::TPUExecuteOp, + llvm::SmallDenseMap> + &exec_to_static_shaped_operands_map, + mlir::OpBuilder &builder, mlir::MLIRContext *context) { + builder.setInsertionPointAfter(compile_op); + llvm::SmallVector output_types; + output_types.push_back(mlir::RankedTensorType::get( + {3}, builder.getType())); + output_types.insert(output_types.end(), used_exec_op.getResultTypes().begin(), + used_exec_op.getResultTypes().end()); + + llvm::SmallVector static_shaped_operand_indices_attr; + llvm::SmallVector static_shape_tensors; + llvm::SmallVector exec_op_args; + exec_op_args.resize(used_exec_op.getArgs().size()); + + auto &static_shaped_operands = + exec_to_static_shaped_operands_map[used_exec_op]; + for (int i = 0; i < used_exec_op.getArgs().size(); ++i) { + auto iter = static_shaped_operands.find(i); + if (iter != static_shaped_operands.end()) { + static_shaped_operand_indices_attr.push_back(iter->first); + static_shape_tensors.push_back(iter->second.getStaticShape()); + exec_op_args[i] = iter->second.getInput(); + // The first operand is the input tensor, while the second operand is + // the static shape tensor, hence the drop_back here. + iter->second->replaceAllUsesWith( + mlir::ValueRange({iter->second.getInput()})); + iter->second->erase(); + } else { + auto split_op = ::llvm::dyn_cast_or_null( + used_exec_op->getOperand(i).getDefiningOp()); + exec_op_args[i] = split_op ? FindFullShapedOperand(split_op) + : used_exec_op->getOperand(i); + } + } + + auto producer_name = + used_exec_op->getAttrOfType("_producer_name"); + if (!producer_name) producer_name = mlir::StringAttr::get(context, "default"); + auto compile_and_execute_op = mlir::TF::TPUCompileMlirAndExecuteOp::create( + builder, used_exec_op.getLoc(), output_types, exec_op_args, + static_shape_tensors, + builder.getI32ArrayAttr(static_shaped_operand_indices_attr), + compile_op.getMlirModule(), compile_op.getMetadata(), producer_name); + + for (auto exec_op : exec_op_in_group) { + exec_op.replaceAllUsesWith(compile_and_execute_op.getResults()); + assert(exec_op.use_empty()); + exec_op.erase(); + } + + for (auto program_result : compile_op.getProgram()) { + program_result.replaceAllUsesWith( + compile_and_execute_op.getRendezvousKeyBase()); + } + assert(compile_op.use_empty()); + compile_op.erase(); +} + +// This pass rewrites tf._TPUCompileMlirOp and tf.TPUExecuteOp into a single +// tf.TPUCompileMlirAndExecuteOp. Also it removes the unnecessary +// TPUCompileSucceededAssertOp. +class FuseTpuCompileAndExecutePass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FuseTpuCompileAndExecutePass) + + llvm::StringRef getArgument() const final { + return "tfrt-fuse-tpu-compile-and-execute-ops"; + } + llvm::StringRef getDescription() const final { + return "Fuse TPU Ops according to TFRT's requirements."; + } + + void runOnOperation() override { + auto func = getOperation(); + + // remove TPUCompileSucceededAssertOp + func.walk([&](mlir::Operation *op) { + if (llvm::isa(op)) { + op->erase(); + } + }); + + // A map from an exec op to a struct containing the static shape tensor from + // a SetDynamicDimensionBoundsOp and the operand index. + llvm::SmallDenseMap< + mlir::TF::TPUExecuteOp, + llvm::SmallDenseMap> + exec_to_static_shaped_operands_map; + + llvm::SmallVector tpu_execute_ops; + func.walk([&](mlir::Operation *op) { + if (auto exec_op = llvm::dyn_cast(op)) { + tpu_execute_ops.push_back(exec_op); + // Collect any operands to this tf.Execute op that are defined by a + // SetStaticDimensionBoundsOp along with the operand index. + for (const auto &operand : llvm::enumerate(exec_op.getOperands())) { + if (auto defining_op = + operand.value() + .getDefiningOp()) { + exec_to_static_shaped_operands_map[exec_op][operand.index()] = + defining_op; + } + } + } + }); + + mlir::OpBuilder builder(&func.getBody()); + + llvm::SmallDenseMap> + compile_and_execute_groups; + + for (auto exec_op : tpu_execute_ops) { + auto compile_cache_entry = exec_op.getKey(); + auto compile_op = ::llvm::dyn_cast( + compile_cache_entry.getDefiningOp()); + if (!compile_op) { + exec_op.emitOpError("could not get the _TPUCompileMlirOp"); + signalPassFailure(); + return; + } + + GroupCompileOpAndExecuteOp(func, compile_op, exec_op); + compile_and_execute_groups[compile_op].push_back(exec_op); + } + for (const auto &kv : compile_and_execute_groups) { + auto compile_op = kv.first; + const llvm::SmallVector &exec_op_in_group = + kv.second; + mlir::TF::TPUExecuteOp used_exec_op; + if (!MaybeFindUsedExecuteOp(exec_op_in_group, used_exec_op)) { + signalPassFailure(); + return; + } + + FuseCompileAndExecuteOps(compile_op, exec_op_in_group, used_exec_op, + exec_to_static_shaped_operands_map, builder, + &getContext()); + } + } +}; + +} // namespace + +namespace tfrt_compiler { + +std::unique_ptr> +CreateFuseTpuCompileAndExecutePass() { + return std::make_unique(); +} + +static mlir::PassRegistration + fuse_tpu_compile_and_execute_ops_pass; + +} // namespace tfrt_compiler + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.cc new file mode 100644 index 0000000000000000000000000000000000000000..5e3f34ae07e67483867428da8860b621a46d1f72 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.cc @@ -0,0 +1,36 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h" + +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h" + +namespace tensorflow { + +void RegisterGpuDialects(mlir::DialectRegistry *registry) { + registry->insert(); +} + +void AddGpuTargetDialectAndPatterns(mlir::MLIRContext *context, + mlir::ConversionTarget *target, + mlir::RewritePatternSet *patterns) { + target->addLegalDialect(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..ca8bd23994032d68cea8fc1b2c9522b07f49d00c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h @@ -0,0 +1,38 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_GPU_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_GPU_PASSES_H_ + +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Pass/PassOptions.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/IR/PatternMatch.h" // from @llvm-project + +namespace tensorflow { + +// Registers dialects used in TFRT GPU lowering. +void RegisterGpuDialects(mlir::DialectRegistry *registry); + +// Adds a target dialect and rewrite patterns for TFRT GPU lowering. +void AddGpuTargetDialectAndPatterns(mlir::MLIRContext *context, + mlir::ConversionTarget *target, + mlir::RewritePatternSet *patterns); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_GPU_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..7a879943ffaf1418c2d73b719a45358d35d23ae7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/BUILD @@ -0,0 +1,338 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load( + "@xla//xla/tsl/platform:build_config.bzl", + "tf_proto_library", +) +load("//tensorflow:tensorflow.bzl", "if_google", "tf_cc_test") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [":friends"], + licenses = ["notice"], +) + +package_group( + name = "friends", + packages = [ + "//tensorflow/compiler/mlir/tfrt/...", + "//tensorflow/core/tfrt/ifrt/...", + "//tensorflow/core/tfrt/saved_model/tests/...", + ] + if_google([ + "//learning/brain/tfrt/cpp_tests/...", + "//learning/serving/servables/tfrt/...", + "//learning/brain/tfrt/ifrt/...", + "//learning/brain/tfrt/tfrt_session/...", + # Allow visibility from the mlir language server. + "//learning/brain/mlir/mlir_lsp_server/...", + "//smartass/brain/util/...", + ]), +) + +gentbl_cc_library( + name = "pass_inc_gen", + tbl_outs = {"passes.h.inc": [ + "-gen-pass-decls", + "-name=TfrtIfrtServing", + ]}, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "passes.td", + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +cc_library( + name = "ifrt_constants", + srcs = [], + hdrs = ["ifrt_constants.h"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "ifrt_types", + srcs = [], + hdrs = ["ifrt_types.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + ], +) + +cc_library( + name = "tf_ifrt_passes", + srcs = [ + "lower_to_ifrt_restore_variable.cc", + "propagate_static_shapes.cc", + "rewrite_cluster_to_ifrt_call.cc", + "sink_variable_as_named_array.cc", + "tf_device_cleanup.cc", + "tf_identity_propagation.cc", + "tf_ifrt_passes.cc", + "tf_restore_merging.cc", + "tf_restore_pruning.cc", + "tf_restore_splitting.cc", + ], + hdrs = [ + "tf_ifrt_passes.h", + ], + textual_hdrs = ["passes.h.inc"], + deps = [ + ":ifrt_constants", + ":pass_inc_gen", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:convert_type", + "//tensorflow/compiler/mlir/tensorflow:device_util", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_types", + "//tensorflow/compiler/mlir/tensorflow:tpu_rewrite_device_util", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "//tensorflow/compiler/mlir/tensorflow/transforms:tensorflow_passes", + "//tensorflow/compiler/mlir/tensorflow/transforms/host_runtime:tpu_metadata_utils", + "//tensorflow/compiler/mlir/tfrt:tf_to_tfrt", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "//tensorflow/core:framework", + "//tensorflow/core/ir/types:Dialect", + "//tensorflow/core/platform:protobuf", + "//tensorflow/core/platform:random", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/tfrt/ifrt:ifrt_config_proto_cc", + "@com_google_absl//absl/base", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@tsl//tsl/platform:protobuf", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/service:computation_placer_hdr", + ], +) + +cc_library( + name = "tf2hlo", + srcs = ["tf2hlo.cc"], + hdrs = ["tf2hlo.h"], + deps = [ + ":ifrt_compilation_proto_cc", + ":ifrt_constants", + ":ifrt_types", + "//tensorflow/compiler/jit:xla_cpu_jit", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:serialize_mlir_module_utils", + "//tensorflow/compiler/mlir/tf2xla:compile_mlir_util", + "//tensorflow/compiler/mlir/tf2xla/api/v2:legalize_tf", + "//tensorflow/compiler/tf2xla:layout_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:lib_headers_for_pybind", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", + "//tensorflow/core/protobuf/tpu:topology_proto_cc", + "//tensorflow/core/tpu/kernels:tpu_compile_op_support", + "//tensorflow/core/tpu/kernels/xla:host_compute_ops", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@tsl//tsl/platform:fingerprint", + "@xla//xla:shape_util", + "@xla//xla:xla_data_proto_cc", + "@xla//xla/client:client_library", + "@xla//xla/hlo/translate/hlo_to_mhlo:hlo_to_mlir_hlo", + "@xla//xla/pjrt:pjrt_compiler", + "@xla//xla/python/ifrt", + "@xla//xla/service:computation_placer_hdr", + "@xla//xla/service:hlo_proto_cc", + "@xla//xla/stream_executor:platform_manager", + "@xla//xla/tsl/platform:errors", + ], +) + +cc_library( + name = "extract_callback", + srcs = ["extract_callback.cc"], + hdrs = ["extract_callback.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_a_m_inc_gen", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_n_z_inc_gen", + "//tensorflow/compiler/mlir/tensorflow:visitor", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + ], +) + +tf_cc_test( + name = "tf2hlo_test", + srcs = [ + "tf2hlo_test.cc", + ], + data = [ + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata", + ], + tags = ["no_oss"], + deps = [ + ":ifrt_types", + ":tf2hlo", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/core:test", + "//tensorflow/core/platform:resource_loader", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:RegisterAllDialects", # buildcleaner: keep + "@tsl//tsl/platform:protobuf", + "@xla//xla:debug_options_flags", + "@xla//xla:shape_util", + "@xla//xla/backends/cpu:target_machine_options", + "@xla//xla/pjrt:pjrt_compiler", + "@xla//xla/pjrt/plugin/xla_cpu:cpu_topology_description", + "@xla//xla/python/ifrt", + "@xla//xla/python/ifrt:mock", + "@xla//xla/python/ifrt:test_util", + "@xla//xla/python/pjrt_ifrt", + "@xla//xla/python/pjrt_ifrt:tfrt_cpu_client_test_lib", + "@xla//xla/service:computation_placer_hdr", + "@xla//xla/tsl/platform:statusor", + ], +) + +cc_library( + name = "ifrt_backend_compiler", + srcs = ["ifrt_backend_compiler.cc"], + hdrs = ["ifrt_backend_compiler.h"], + deps = [ + ":tf_ifrt_passes", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tensorflow:visitor", + "//tensorflow/compiler/mlir/tf2xla/api/v2:cluster_tf", + "//tensorflow/compiler/mlir/tfrt:backend_compiler", + "//tensorflow/compiler/mlir/tfrt:tpu_passes", + "//tensorflow/core/tfrt/ifrt:ifrt_executable_registry", + "//tensorflow/core/tfrt/ifrt:ifrt_model_context", + "//tensorflow/core/tfrt/ifrt:ifrt_serving_executable", + "//tensorflow/core/tfrt/runtime", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@tsl//tsl/profiler/lib:traceme", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_cc_test( + name = "ifrt_backend_compiler_test", + srcs = [ + "ifrt_backend_compiler_test.cc", + ], + data = [ + "//tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata", + ], + tags = ["no_oss"], + deps = [ + ":ifrt_backend_compiler", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/core:test", + "//tensorflow/core/platform:resource_loader", + "//tensorflow/core/tfrt/graph_executor:graph_execution_options", + "//tensorflow/core/tfrt/ifrt:ifrt_executable_registry", + "//tensorflow/core/tfrt/ifrt:ifrt_model_context", + "//tensorflow/core/tfrt/ifrt:ifrt_serving_core_selector", + "//tensorflow/core/tfrt/ifrt:sharding_utils", + "//tensorflow/core/tfrt/runtime", + "//tensorflow/core/tfrt/saved_model:saved_model_testutil", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:RegisterAllDialects", # buildcleaner: keep + "@tf_runtime//:hostcontext", + "@xla//xla/python/ifrt", + "@xla//xla/python/ifrt:test_util", + "@xla//xla/python/pjrt_ifrt:tfrt_cpu_client_test_lib", + "@xla//xla/tsl/framework/test_util:mock_serving_device_selector", + "@xla//xla/tsl/platform:env", + "@xla//xla/tsl/platform:statusor", + ], +) + +tf_proto_library( + name = "ifrt_compilation_proto", + srcs = ["ifrt_compilation.proto"], + protodeps = [ + "//tensorflow/compiler/tf2xla:host_compute_metadata_proto", + "@xla//xla/service:hlo_proto", + "//tensorflow/core:protos_all", + "//tensorflow/core/protobuf/tpu:compile_metadata_proto", + ], + deps = [ + "@xla//xla:xla_data_proto", + "@xla//xla/python/ifrt:layout_proto", + ], +) + +cc_library( + name = "pack_inputs_pass", + srcs = ["pack_inputs_pass.cc"], + hdrs = ["pack_inputs_pass.h"], + deps = [ + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@stablehlo//:stablehlo_ops", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.cc new file mode 100644 index 0000000000000000000000000000000000000000..1c94f4460d327c01dfd2de72ed99c905462b37a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.cc @@ -0,0 +1,81 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.h" + +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h.inc" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h.inc" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/visitor.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace tensorflow { +namespace ifrt_serving { + +absl::StatusOr> ExtractCallbackModule( + mlir::ModuleOp module, absl::string_view callback_key) { + // Find the entry function name first. + mlir::func::FuncOp callback_entry_func; + module.walk([&](mlir::func::FuncOp func) { + if (func.getSymName().str() == callback_key) { + callback_entry_func = func; + return mlir::WalkResult::skip(); + } + return mlir::WalkResult::advance(); + }); + + if (!callback_entry_func) { + return absl::NotFoundError( + absl::StrCat("Callback key ", callback_key, " not found")); + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module->getContext()); + auto entry_function_name = callback_entry_func.getSymName(); + auto submodule = mlir::TF::CreatePrunedModule(module, entry_function_name); + if (mlir::failed(submodule)) { + return diag_handler.ConsumeStatus(); + } + + // Remove the attribute inherited from saved model loading. They impose + // additional constraint on public functions that are not necessary. + submodule->get()->removeAttr("tf_saved_model.semantics"); + submodule->get().walk([&](mlir::func::FuncOp func) { + if (func.getSymName() == entry_function_name) { + func.setPublic(); + } + }); + return std::move(*submodule); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.h new file mode 100644 index 0000000000000000000000000000000000000000..4df001f67173ec89070cab5665e294584d1c4495 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/extract_callback.h @@ -0,0 +1,36 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_EXTRACT_CALLBACK_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_EXTRACT_CALLBACK_H_ + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/core/framework/types.pb.h" + +namespace tensorflow { +namespace ifrt_serving { + +// Extracts a module that consists of a public callback function in name of +// `callback_key` and all its reachables. +absl::StatusOr> ExtractCallbackModule( + mlir::ModuleOp module, absl::string_view callback_key); + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_EXTRACT_CALLBACK_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.cc new file mode 100644 index 0000000000000000000000000000000000000000..d589603c53743e4ef566024358e549746b0b53c0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.cc @@ -0,0 +1,235 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.h" + +#include +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Verifier.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/visitor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_executable_registry.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_model_context.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_serving_executable.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tsl/profiler/lib/traceme.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { +absl::StatusOr> +CompileAndRegisterIfrtPrograms(absl::string_view model_name, + mlir::ModuleOp module, + IfrtModelContext& ifrt_model_context) { + std::vector handles; + + // Compile Ifrt programs and register the executables. Outlined Ifrt + // programs are marked with `tfrt_ifrt_serving.program_id` attributes. + for (auto func : module.getOps()) { + int64_t program_id; + if (auto attr = func->getAttrOfType( + "tfrt_ifrt_serving.program_id")) { + program_id = attr.getInt(); + } else { + continue; + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module->getContext()); + auto entry_function_name = func.getSymName(); + auto submodule = mlir::TF::CreatePrunedModule(module, entry_function_name); + if (mlir::failed(submodule)) { + return diag_handler.ConsumeStatus(); + } + + // Remove the attribute inherited from saved model loading. They impose + // additional constraint on public functions that are not necessary. + submodule->get()->removeAttr("tf_saved_model.semantics"); + submodule->get().walk([&](mlir::func::FuncOp func) { + if (func.getSymName() == entry_function_name) { + func.setName("main"); + func.setSymName("main"); + func.setPublic(); + } + }); + // Remove the program id attribute from the submodule because they are not + // needed and will prevent us generating consistent cache key. + // program id is already in ifrt_call op's attribute and that part is not + // touched here. + submodule->get()->walk([](mlir::func::FuncOp func) { + func->removeAttr("tfrt_ifrt_serving.program_id"); + }); + + TF_ASSIGN_OR_RETURN( + auto executable, + IfrtServingExecutable::Create( + program_id, model_name, entry_function_name.str(), + *std::move(submodule), ifrt_model_context.GetClient(), + &ifrt_model_context.GetThreadPool(), + &ifrt_model_context.GetLoadedVariableRegistry(), + &ifrt_model_context.GetRestoreTensorRegistry(), + ifrt_model_context.checkpoint_loader_queue(), + ifrt_model_context.GetDeviceMgr(), + ifrt_model_context.GetShapeRepresentationFn(), + ifrt_model_context.GetIfrtServingCoreSelector(), + ifrt_model_context.GetCompilationEnvOrOverrides(), + ifrt_model_context.GetTfToHloCompiler(), + ifrt_model_context.GetPersistentCompilationCache(), + ifrt_model_context.GetH2DTransferExecutorFactory())); + + // Register the Ifrt program to `ServingExecutableRegistry` so that + // the client TF program can invoke them via `IfrtCall` op. + TF_ASSIGN_OR_RETURN(auto handle, ServingExecutableRegistry::Register( + program_id, std::move(executable))); + + handles.push_back(std::move(handle)); + } + + return handles; +} + +absl::Status CompileTensorflowForIfrtServing( + absl::string_view model_name, IfrtModelContext& ifrt_model_context, + mlir::ModuleOp module, bool enable_async_ifrt) { + tsl::profiler::TraceMe trace_me("CompileTensorflowForIfrtServing"); + mlir::Builder builder(module.getContext()); + + TF_RETURN_IF_ERROR(RunClusterToIfrtRuntimeOpsPassPipeline( + module, model_name, + ifrt_model_context.enable_propagate_static_shapes_pass(), + enable_async_ifrt)); + + TF_ASSIGN_OR_RETURN( + auto handles, + CompileAndRegisterIfrtPrograms(model_name, module, ifrt_model_context)); + + for (auto& handle : handles) { + ifrt_model_context.RegisterHandle(std::move(handle)); + } + + return absl::OkStatus(); +} + +} // namespace + +// Compile ifrt programs in TF dialect into ifrt executables. +// Remove ifrt programs afterwards. +absl::Status IfrtBackendCompiler::CompileTensorflow( + tensorflow::tfrt_stub::ModelRuntimeContext& model_context, + mlir::ModuleOp module) const { + auto ifrt_model_context = + model_context.resource_context().GetResource( + kIfrtModelContextName); + if (!ifrt_model_context.has_value()) { + return absl::InternalError( + "Failed to find model context for ifrt serving."); + } + + if ((*ifrt_model_context)->IsFrozen()) { + return absl::FailedPreconditionError( + "Cannot compile IFRT programs after the model is frozen. Please make " + "sure warmup covers all signatures by following go/tf-model-warmup."); + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module->getContext()); + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("ifrt_tpu_bct_conversion_before", module); + } + + TfrtTpuCompileOptions options; + options.disable_set_default_tpu_device_and_device_assignment_attributes = + compile_options_ + .disable_set_default_tpu_device_and_device_assignment_attributes; + options.support_multi_dims_sharding = true; + + if (tpu_compiler_ != nullptr) { + // Run backward compat pass so that we can use bridge to do clustering. + if (mlir::failed( + tpu_compiler_->RunTPUBackwardCompatConversion(module, options))) { + return diag_handler.Combine( + absl::InternalError("Failed to handle legacy TPU Ops")); + } + } + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("ifrt_tpu_bct_conversion_after", module); + } + + // Use bridge for cluster formation. + TF_RETURN_IF_ERROR(tensorflow::tf2xla::v2::RunFunctionTf2xlaClusteringBridge( + module, /*is_supported_by_replicated_brige*/ true, + /*is_in_fallback_enabled_mode=*/false)); + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("before_ifrt_outlining", module); + } + + // Extract TPU program for IFRT call. + TF_RETURN_IF_ERROR(CompileTensorflowForIfrtServing( + model_context.name(), **ifrt_model_context, module, + model_context.graph_execution_options() + .compile_options.enable_async_ifrt)); + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("after_ifrt_outlining", module); + } + + // IFRT program is no longer needed. + llvm::SmallVector to_erase; + for (auto func : module.getOps()) { + if (func->getAttr("tfrt_ifrt_serving.program_id")) { + to_erase.push_back(func); + } + } + for (auto func : to_erase) { + func->erase(); + } + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("after_ifrt_program_removal", module); + } + + if (mlir::failed(mlir::verify(module))) { + return diag_handler.ConsumeStatus(); + } + + return absl::OkStatus(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.h new file mode 100644 index 0000000000000000000000000000000000000000..6e7f5045f852c85b7357dc1ef1003093db2c7405 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.h @@ -0,0 +1,72 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_BACKEND_COMPILER_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_BACKEND_COMPILER_H_ + +#include +#include + +#include "absl/status/status.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/backend_compiler.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" + +namespace tensorflow { +namespace ifrt_serving { + +// Implements the custom backend compiler for IFRT based serving in TFRT. +class IfrtBackendCompiler : public tensorflow::BackendCompiler { + public: + struct Options { + // If true, disable running TFRTSetTPUDeviceAttrPass which set the default + // `tf.device` and `device_assignment` attributes. + // This is a server-level option for now. We can consider to make it a + // per-model option in the future. + bool disable_set_default_tpu_device_and_device_assignment_attributes = true; + }; + + explicit IfrtBackendCompiler( + std::unique_ptr tpu_compiler = nullptr) + : tpu_compiler_(std::move(tpu_compiler)) {} + + explicit IfrtBackendCompiler( + const Options& ifrt_backend_compile_options, + std::unique_ptr tpu_compiler = nullptr) + : tpu_compiler_(std::move(tpu_compiler)), + compile_options_(ifrt_backend_compile_options) {} + + void GetDependentDialects(mlir::DialectRegistry& registry) const override { + if (tpu_compiler_) { + tpu_compiler_->RegisterTPUDialects(®istry); + } + } + + // Rewrites the tensorflow graph in MLIR for IFRT serving. The methods + // extracts regions for IFRT execution on accelerator (e.g. TPU). + absl::Status CompileTensorflow( + tensorflow::tfrt_stub::ModelRuntimeContext& model_context, + mlir::ModuleOp module) const override; + + private: + std::unique_ptr tpu_compiler_; + Options compile_options_; +}; + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_BACKEND_COMPILER_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..2f612913a26eb2da9cf5ced846585b982fa9cb0e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler_test.cc @@ -0,0 +1,186 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_backend_compiler.h" + +#include +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "xla/python/ifrt/client.h" +#include "xla/python/ifrt/test_util.h" +#include "xla/tsl/framework/test_util/mock_serving_device_selector.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/env.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/tsl/platform/threadpool.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/tfrt/graph_executor/graph_execution_options.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_executable_registry.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_model_context.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_serving_core_selector.h" +#include "tensorflow/core/tfrt/ifrt/sharding_utils.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tensorflow/core/tfrt/saved_model/saved_model_testutil.h" +#include "tfrt/host_context/resource_context.h" // from @tf_runtime + +namespace tensorflow { +namespace ifrt_serving { + +tsl::thread::ThreadPool& GetThreadPool() { + constexpr int kMaxParallelism = 16; + static tsl::thread::ThreadPool* thread_pool = + new tsl::thread::ThreadPool(tsl::Env::Default(), tsl::ThreadOptions(), + "IfrtSharding", kMaxParallelism); + return *thread_pool; +} + +class IfrtBackendCompilerTest : public ::testing::Test { + protected: + void SetUp() override { + mlir::registerAllDialects(registry_); + mlir::RegisterAllTensorFlowDialects(registry_); + context_.appendDialectRegistry(registry_); + + // Create contexts required for the compiler execution. + TF_ASSERT_OK_AND_ASSIGN(client_, xla::ifrt::test_util::GetClient()); + + core_selector_ = std::make_unique( + &mock_serving_device_selector_, client_->addressable_device_count()); + h2d_transfer_executor_factory_ = + std::make_unique(); + + runtime_context_.resource_context().CreateResource( + "IfrtModelContext", client_, core_selector_.get(), &GetThreadPool(), + /*compilation_environment_proto=*/nullptr, + /*h2d_transfer_executor_factory=*/h2d_transfer_executor_factory_.get()); + } + + void verifyModules() { + absl::MutexLock l(ServingExecutableRegistry::mu_); + for (const auto& [_, executable] : + *ServingExecutableRegistry::executables_) { + absl::MutexLock l(executable->mutex_); + executable->module_->walk([](mlir::func::FuncOp func) { + ASSERT_FALSE(func->hasAttr("tfrt_ifrt_serving.program_id")); + }); + } + } + + mlir::DialectRegistry registry_; + mlir::MLIRContext context_; + std::shared_ptr client_; + + std::unique_ptr runtime_ = + tensorflow::tfrt_stub::DefaultTfrtRuntime(/*num_threads=*/1); + tensorflow::tfrt_stub::GraphExecutionOptions graph_execution_options_ = + tensorflow::tfrt_stub::GraphExecutionOptions(runtime_.get()); + tfrt::ResourceContext resource_context_; + tensorflow::tfrt_stub::ModelRuntimeContext runtime_context_ = + tensorflow::tfrt_stub::ModelRuntimeContext( + &graph_execution_options_, /*export_dir=*/"", &resource_context_); + + tsl::test_util::MockServingDeviceSelector mock_serving_device_selector_; + std::unique_ptr core_selector_; + std::unique_ptr h2d_transfer_executor_factory_; + IfrtBackendCompiler compiler_; +}; + +namespace { +using ::testing::HasSubstr; + +struct IfrtBackendCompilerTestParams { + std::string mlir_file_name; +}; + +class IfrtBackendCompilerParameterizedTest + : public IfrtBackendCompilerTest, + public ::testing::WithParamInterface {}; + +TEST_P(IfrtBackendCompilerParameterizedTest, CompilesOk) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/", GetParam().mlir_file_name)); + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, &context_); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK( + compiler_.CompileTensorflow(runtime_context_, mlir_module.get())); + verifyModules(); +} + +INSTANTIATE_TEST_SUITE_P(IfrtBackendCompilerParameterizedTest, + IfrtBackendCompilerParameterizedTest, + ::testing::ValuesIn({ + {.mlir_file_name = "ifrt_cluster.mlir"}, + {.mlir_file_name = "restore_with_reference.mlir"}, + })); + +TEST_F(IfrtBackendCompilerTest, CompileShallFailAfterModelIsFrozen) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/ifrt_cluster.mlir")); + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, &context_); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK( + compiler_.CompileTensorflow(runtime_context_, mlir_module.get())); + + std::optional ifrt_model_context = + runtime_context_.resource_context().GetResource( + "IfrtModelContext"); + ASSERT_TRUE(ifrt_model_context.has_value()); + + TF_ASSERT_OK((*ifrt_model_context)->Freeze()); + + mlir::OwningOpRef another_mlir_module = + mlir::parseSourceFile(mlir_module_path, &context_); + + EXPECT_THAT( + compiler_.CompileTensorflow(runtime_context_, another_mlir_module.get()), + absl_testing::StatusIs( + absl::StatusCode::kFailedPrecondition, + HasSubstr("Cannot compile IFRT programs after the model is frozen"))); +} + +} // namespace +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_compilation.proto b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_compilation.proto new file mode 100644 index 0000000000000000000000000000000000000000..3fab6bfa0b21f45dd505688089390e6da35e66f8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_compilation.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package tensorflow.ifrt_serving; + +import "tensorflow/compiler/tf2xla/host_compute_metadata.proto"; +import "xla/python/ifrt/layout.proto"; +import "xla/service/hlo.proto"; +import "xla/xla_data.proto"; +import "tensorflow/core/framework/types.proto"; +import "tensorflow/core/protobuf/tpu/compile_metadata.proto"; + +message Tf2HLOResultProto { + xla.HloModuleProto hlo_module_proto = 1; + tensorflow.tpu.TPUCompileMetadataProto compile_metadata = 2; + tensorflow.tf2xla.HostComputeMetadata host_compute_metadata = 3; + repeated xla.ShapeProto xla_input_shapes = 4; +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..3a1660f4910e751e502e6630f46d620540247f85 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h @@ -0,0 +1,40 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_CONSTANTS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_CONSTANTS_H_ + +#include "absl/strings/string_view.h" + +namespace tensorflow { +namespace ifrt_serving { + +// Attribute name of a text TpuCompileMetadataProto. Note that the text proto is +// not backward compatible and shall not be serialized. +inline constexpr absl::string_view kMetadataTextAttrName = + "__tpu_compile_metadata_text"; + +// Name of a variable as loaded IFRT array . +inline constexpr absl::string_view kVariableArrayNameAttr = + "__variable_array_name"; + +// Attribute of a text `VariableDeviceShardingConfigProto`. +inline constexpr absl::string_view kVariableShardingConfigTextAttr = + "__variable_sharding_config_text"; + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_CONSTANTS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h new file mode 100644 index 0000000000000000000000000000000000000000..b0c1e167e3b9746c0285752bf7c8f153fd7a9552 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h @@ -0,0 +1,61 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_TYPES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_TYPES_H_ + +#include + +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" + +namespace tensorflow { +namespace ifrt_serving { + +// TODO - b/445480506: Refactor this struct to a class so that accesses are +// always through accessors. +struct DtypeAndShape { + tensorflow::DataType dtype; + tensorflow::TensorShape shape; + // If available, use static shape (the upper bound of the actual input shapes) + // for compilation and caching. + std::optional static_shape; + + bool operator==(const DtypeAndShape& other) const { + return dtype == other.dtype && shape == other.shape && + static_shape == other.static_shape; + } + + // Returns the shape to be used for compilation. If static shape is + // available, use static shape, so that the same executable can be reused for + // all input shapes with the same static shape. + tensorflow::TensorShape& GetMutableShapeForCompilation() { + // Avoid `static_shape.value_or(shape)`. Since `value_or` returns by value, + // using it here would return a reference to a temporary object, leading to + // a dangling reference. + return static_shape.has_value() ? *static_shape : shape; + } + const tensorflow::TensorShape& GetShapeForCompilation() const { + // Avoid `static_shape.value_or(shape)`. Since `value_or` returns by value, + // using it here would return a reference to a temporary object, leading to + // a dangling reference. + return static_shape.has_value() ? *static_shape : shape; + } +}; + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_IFRT_TYPES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/lower_to_ifrt_restore_variable.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/lower_to_ifrt_restore_variable.cc new file mode 100644 index 0000000000000000000000000000000000000000..82d50cd2798860401b254e90602c1e2dbb4fc52e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/lower_to_ifrt_restore_variable.cc @@ -0,0 +1,597 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/ir/types/dialect.h" +#include "tensorflow/core/platform/protobuf.h" // IWYU pragma: keep +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_config.pb.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_LOWERTOIFRTRESTOREVARIABLEPASS +#define GEN_PASS_DECL_LOWERTOIFRTRESTOREVARIABLEPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +class LowerToIfrtRestoreVariablePass + : public impl::LowerToIfrtRestoreVariablePassBase< + LowerToIfrtRestoreVariablePass> { + public: + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + + std::vector restore_ops; + module.walk([&](mlir::TF::RestoreV2Op restore_op) { + restore_ops.push_back(restore_op); + }); + + for (const auto& restore_op : restore_ops) { + if (mlir::failed(RewriteRestore(restore_op))) { + return signalPassFailure(); + } + } + } + + private: + // Holds information about a user of a restored tensor from RestoreV2Op. + struct RestoreOpUser { + // The chain of ops from RestoreV2Op's user. If var_handle_op is + // not null, this chain ends with AssignVariableOp and will be deleted. + // Otherwise, restored_tensor_to_return will be set, and this chain + // ends with an op that uses the restored tensor, which should be returned + // by IfrtRestoreVariableOp. + std::vector op_chain_from_restore_output; + + // The name of the restored tensor. + mlir::StringRef tensor_name; + + // The attributes of VarHandleOp associated with the AssignVariableOp. + mlir::Type var_handle_type; + mlir::StringAttr container; + mlir::StringAttr shared_name; + bool truncate_in_cast = + false; // value of the truncate attribute in the CastOp. + // Default to false if CastOp is not present. + + // The restored tensor that is not assigned to a variable and should be + // returned by IfrtRestoreVariableOp. + std::optional restored_tensor_to_return; + + // The VarHandleOp associated with the AssignVariableOp, if it exists in + // the same block or function. + mlir::TF::VarHandleOp original_var_handle_op; + }; + + // Groups parameters being collected for IfrtRestoreVariableOp. + struct RestorationParams { + llvm::SmallVectorImpl& tensor_names; + llvm::SmallVectorImpl& shape_and_slices; + llvm::SmallVectorImpl& dtypes; + llvm::SmallVectorImpl& truncate_in_cast; + std::vector& var_handle_values; + }; + + // Holds information about an intercepted Read-Assign chain. + struct InterceptedChain { + mlir::TF::AssignVariableOp assign_op; + mlir::TF::VarHandleOp target_handle; + bool truncate = false; + std::vector op_chain; + }; + + mlir::LogicalResult ValidateThenUpdateUser( + mlir::Value source_tensor, mlir::Operation* user, + mlir::StringRef tensor_name, + std::vector& restored_tensor_users) { + // Traverses the user chain of a restored tensor, validates it, and collects + // user information. + RestoreOpUser restored_tensor_user; + restored_tensor_user.tensor_name = tensor_name; + for (;;) { + restored_tensor_user.op_chain_from_restore_output.push_back(user); + // The user chain must consist of zero or more `TF::CastOp`s followed by + // a `TF::AssignVariableOp`. + if (auto cast_op = llvm::dyn_cast(user)) { + // The chain can contain intermediate `CastOp`s. + if (!cast_op.getResult().hasOneUse()) { + return cast_op->emitOpError() + << " has more than one use in the restore user chain"; + } + restored_tensor_user.truncate_in_cast = + restored_tensor_user.truncate_in_cast || cast_op.getTruncate(); + // Move to the next operation in the user chain. + user = *cast_op.getResult().getUsers().begin(); + } else if (auto assign_variable_op = + llvm::dyn_cast(user)) { + // The chain must end with an `AssignVariableOp`. + if (auto var_handle_op = llvm::dyn_cast( + assign_variable_op.getResource().getDefiningOp())) { + // The AssignVariableOp must be associated with a VarHandleOp. + restored_tensor_user.var_handle_type = var_handle_op.getType(); + restored_tensor_user.container = var_handle_op.getContainerAttr(); + restored_tensor_user.shared_name = var_handle_op.getSharedNameAttr(); + restored_tensor_user.original_var_handle_op = var_handle_op; + break; + } else { + return assign_variable_op->emitOpError() + << "does not have any associated VarHandle"; + } + } else { + // If the restored tensor is not assigned to a variable, it is returned + // by the IfrtRestoreVariableOp. A VarHandleOp is created to track the + // restored tensor. + mlir::OpBuilder builder(user); + + mlir::Type resource_type = mlir::RankedTensorType::get( + /*shape=*/{}, /*element_type=*/mlir::tf_type::ResourceType::get( + {mlir::cast(source_tensor.getType())}, + builder.getContext())); + restored_tensor_user.var_handle_type = resource_type; + // In TensorFlow, variables are identified by a (container, + // shared_name) pair. An empty string for `container` is valid and + // indicates that the default container should be used. For restored + // tensors that are returned as outputs instead of being assigned to + // variables, we create a logical VarHandleOp with an empty container + // and use `tensor_name` as `shared_name`. + restored_tensor_user.container = builder.getStringAttr(""); + restored_tensor_user.shared_name = builder.getStringAttr(tensor_name); + + restored_tensor_user.restored_tensor_to_return = source_tensor; + break; + } + } + + // If the user chain is valid, add the collected information. + restored_tensor_users.push_back(restored_tensor_user); + return mlir::success(); + } + + // Traces and validates an operation chain starting from a `ReadVariableOp` + // user. Returns the chain if it's a valid pattern (optional `Cast`s, + // `Slice`s, or `Identity`s followed by an `AssignVariableOp`). + std::optional TraceReadChain(mlir::Operation* read_user) { + InterceptedChain chain; + mlir::Operation* current = read_user; + while (true) { + if (auto cast_op = llvm::dyn_cast(current)) { + chain.truncate |= cast_op.getTruncate(); + chain.op_chain.push_back(cast_op); + if (!cast_op.getResult().hasOneUse()) return std::nullopt; + current = *cast_op.getResult().getUsers().begin(); + } else if (auto slice_op = llvm::dyn_cast(current)) { + chain.op_chain.push_back(slice_op); + if (!slice_op.getOutput().hasOneUse()) return std::nullopt; + current = *slice_op.getOutput().getUsers().begin(); + } else if (auto identity_op = + llvm::dyn_cast(current)) { + chain.op_chain.push_back(identity_op); + if (!identity_op.getOutput().hasOneUse()) return std::nullopt; + current = *identity_op.getOutput().getUsers().begin(); + } else { + break; + } + } + + if (auto assign_op = llvm::dyn_cast(current)) { + if (auto target_handle = llvm::dyn_cast( + assign_op.getResource().getDefiningOp())) { + chain.assign_op = assign_op; + chain.target_handle = target_handle; + chain.op_chain.push_back(assign_op); + return chain; + } + } + return std::nullopt; + } + + // Processes a validly intercepted chain: updates restoration parameters and + // manages `VarHandleOp`s. + void ProcessInterceptedChain(InterceptedChain& chain, + mlir::StringRef tensor_name, + mlir::StringRef slice, mlir::Attribute dtype, + RestorationParams& params, + mlir::OpBuilder& handle_builder, + mlir::Operation* restore_op) { + params.tensor_names.push_back(tensor_name.str()); + params.shape_and_slices.push_back(slice.str()); + params.dtypes.push_back(dtype); + params.truncate_in_cast.push_back(chain.truncate); + + mlir::Value handle_result; + if (chain.target_handle->getBlock() == restore_op->getBlock()) { + handle_result = chain.target_handle.getResult(); + // Since we are reusing an op that originally appeared later, we need to + // move it before the new `IfrtRestoreVariableOp` to avoid dominance + // violations. + chain.target_handle->moveBefore(restore_op); + } else { + auto derived_local_handle = handle_builder.create( + restore_op->getLoc(), chain.target_handle.getType(), + chain.target_handle.getContainerAttr(), + chain.target_handle.getSharedNameAttr()); + handle_result = derived_local_handle.getResult(); + } + params.var_handle_values.push_back(handle_result); + } + + // Identifies and handles cases where a restored variable is read and + // subsequently assigned to another variable. For example, if VarA is restored + // from a checkpoint, and VarB is initialized with VarA's value (e.g., via + // ReadVariable(VarA) -> AssignVariable(VarB)), then VarB should also be + // treated as being restored from the checkpoint. This function detects such + // patterns, adds VarB to list of tensors to be restored by + // IfrtRestoreVariableOp, and marks the intermediate ReadVariableOp and + // AssignVariableOp chain for deletion. + // Returns intercepted chains if all users of a `ReadVariableOp` lead to + // assignments. + std::optional> GetInterceptedChains( + mlir::TF::ReadVariableOp read_op) { + std::vector intercepted_chains; + // Follow the user chain for each user of the `ReadVariableOp`. + for (mlir::Operation* read_user : read_op.getResult().getUsers()) { + if (std::optional intercepted_chain = + TraceReadChain(read_user)) { + intercepted_chains.push_back(std::move(*intercepted_chain)); + } else { + return std::nullopt; + } + } + return intercepted_chains; + } + + // Identifies and handles cases where a restored variable is read and + // subsequently assigned to another variable. For example, if VarA is restored + // from a checkpoint, and VarB is initialized with VarA's value (e.g., via + // ReadVariable(VarA) -> AssignVariable(VarB)), then VarB should also be + // treated as being restored from the checkpoint. This function detects such + // patterns, adds VarB to list of tensors to be restored by + // IfrtRestoreVariableOp, and marks the intermediate ReadVariableOp and + // AssignVariableOp chain for deletion. + void InterceptReadAssignVariable( + const llvm::StringMap>& + shared_name_to_reads, + mlir::StringRef tensor_name, mlir::StringRef slice, mlir::Attribute dtype, + RestoreOpUser& restored_tensor_user, mlir::OpBuilder& handle_builder, + mlir::Operation* restore_op, RestorationParams& params, + llvm::SmallSetVector& ops_to_erase) { + auto it = + shared_name_to_reads.find(restored_tensor_user.shared_name.getValue()); + // If no `ReadVariableOp` reads the variable associated with + // `restored_tensor_user`, then there is no read-assign chain to intercept. + if (it == shared_name_to_reads.end()) { + return; + } + + auto add_to_erase = [&](mlir::Operation* op) { ops_to_erase.insert(op); }; + + // Iterate through all `ReadVariableOp`s that read the variable. + for (mlir::TF::ReadVariableOp read_op : it->second) { + // If all users of this specific `ReadVariableOp` instance were + // successfully mapped to `AssignVariableOp` chains, we can erase the + // `ReadVariableOp` and all intermediate ops in these chains. + if (auto intercepted_chains = GetInterceptedChains(read_op)) { + if (intercepted_chains->empty()) continue; + + add_to_erase(read_op); + for (auto& chain : *intercepted_chains) { + ProcessInterceptedChain(chain, tensor_name, slice, dtype, params, + handle_builder, restore_op); + for (mlir::Operation* op : chain.op_chain) { + add_to_erase(op); + } + } + } + } + } + + // Creates a TF::ConstOp with a 1D tensor of strings from the given values. + mlir::Value CreateStringTensorConst( + mlir::OpBuilder& builder, mlir::Location loc, + const llvm::SmallVector& values) { + llvm::SmallVector ref_values; + for (const auto& value : values) { + ref_values.push_back(value); + } + auto type = mlir::RankedTensorType::get( + {static_cast(values.size())}, + mlir::tf_type::StringType::get(builder.getContext())); + + return builder.create( + loc, mlir::DenseStringElementsAttr::get(type, ref_values)); + } + + mlir::LogicalResult RewriteRestore(mlir::TF::RestoreV2Op restore_op) { + // Find and validate all users of the RestoreV2Op's output tensors. + + mlir::DenseStringElementsAttr tensor_names_attr; + if (!matchPattern(restore_op.getTensorNames(), + m_Constant(&tensor_names_attr))) { + return restore_op.emitOpError("expects tensor_names to be a constant"); + } + auto name_values = tensor_names_attr.getValues(); + llvm::SmallVector tensor_names_vec(name_values.begin(), + name_values.end()); + + if (restore_op.getTensors().size() != tensor_names_vec.size()) { + return restore_op.emitOpError(absl::StrCat( + "expects the number of tensors and tensor names to be the same, got ", + restore_op.getTensors().size(), " tensors and ", + tensor_names_vec.size(), " tensor names.")); + } + + // The RestoreV2 op contains shape and slice specifications for each tensor + // to be restored. These are expected to be provided as a constant string + // tensor. + mlir::DenseStringElementsAttr shape_and_slices_attr; + if (!matchPattern(restore_op.getShapeAndSlices(), + m_Constant(&shape_and_slices_attr))) { + return restore_op.emitOpError( + "expects shape_and_slices to be a constant"); + } + // Extract shape and slice strings into a vector. + // This vector is used later to associate each restored tensor with its + // corresponding shape and slice specification when creating the + // IfrtRestoreVariableOp. + auto slice_values = shape_and_slices_attr.getValues(); + llvm::SmallVector shape_and_slices_vec( + slice_values.begin(), slice_values.end()); + + std::vector restored_tensor_users; + for (int i = 0; i < restore_op.getTensors().size(); ++i) { + mlir::Value out_tensor = restore_op.getTensors()[i]; + mlir::StringRef tensor_name = tensor_names_vec[i]; + for (mlir::Operation* user : out_tensor.getUsers()) { + if (mlir::failed(ValidateThenUpdateUser(out_tensor, user, tensor_name, + restored_tensor_users))) { + return mlir::failure(); + } + } + } + + // Check that each tensor has exactly one valid user chain. + if (restored_tensor_users.size() != restore_op.getTensors().size()) { + return restore_op->emitOpError() + << "expects " << restore_op.getTensors().size() + << " valid users, but got " << restored_tensor_users.size(); + } + + // The transformation needs to identify cases where a restored variable is + // read and then assigned to another variable (e.g., RestoreV2 -> + // AssignVariable(var1), ReadVariable(var1) -> AssignVariable(var2)). + // In such cases, `var2` should also be treated as a restored variable + // initialized from the checkpoint. To achieve this, we first map all + // variable shared names to their VarHandleOps and all ReadVariableOps + // that read them. This allows us to trace reads of restored variables and + // check if they are part of an assignment to another variable. + llvm::StringMap shared_name_to_handle; + llvm::StringMap> shared_name_to_reads; + restore_op->getParentOfType().walk([&](mlir::Operation* + op) { + if (auto var_handle = llvm::dyn_cast(op)) { + shared_name_to_handle[var_handle.getSharedName()] = var_handle; + } else if (auto read_op = llvm::dyn_cast(op)) { + if (auto var_handle = + read_op.getResource().getDefiningOp()) { + shared_name_to_reads[var_handle.getSharedName()].push_back(read_op); + } + } + }); + + // Collect tensor dtypes for the new op. + std::vector dtypes; + for (const auto& dtype : restore_op.getDtypes()) { + dtypes.push_back(mlir::TypeAttr::get(dtype)); + } + + // The following vectors collect information for each tensor that will be + // restored by the new IfrtRestoreVariableOp. This includes tensors + // directly restored from RestoreV2Op, as well as variables initialized + // by reading other restored variables (e.g., ReadVariable(restored_var) -> + // AssignVariable(new_var)). This information is used to construct the + // arguments for the IfrtRestoreVariableOp that replaces RestoreV2Op and + // its user chains. + llvm::SmallVector new_tensor_names; + llvm::SmallVector new_shape_and_slices; + llvm::SmallVector new_dtypes; + llvm::SmallVector new_truncate_in_cast; + + std::vector var_handle_values; + + RestorationParams params{new_tensor_names, new_shape_and_slices, new_dtypes, + new_truncate_in_cast, var_handle_values}; + + llvm::SmallVector result_types; + llvm::DenseMap returned_tensors_indices; + llvm::SmallVector returned_tensor_names; + + // The following data structures are used to manage operations that need to + // be erased after the IfrtRestoreVariableOp is created. This includes + // chains of operations from RestoreV2Op to AssignVariableOp, as well as + // ReadVariableOp -> AssignVariableOp chains that are optimized away by + // InterceptReadAssignVariable. + llvm::SmallSetVector ops_to_erase; + // Helper function to add an operation to ops_to_erase, ensuring no + // duplicates are added. + auto add_to_erase = [&](mlir::Operation* op) { ops_to_erase.insert(op); }; + // OpBuilder for creating new VarHandleOps before the RestoreV2Op. + mlir::OpBuilder handle_builder(restore_op); + + for (int i = 0; i < restored_tensor_users.size(); ++i) { + RestoreOpUser& restored_tensor_user = restored_tensor_users[i]; + mlir::StringRef tensor_name = tensor_names_vec[i]; + mlir::StringRef slice = shape_and_slices_vec[i]; + mlir::Attribute dtype = dtypes[i]; + + params.tensor_names.push_back(tensor_name.str()); + params.shape_and_slices.push_back(slice.str()); + params.dtypes.push_back(dtype); + params.truncate_in_cast.push_back(restored_tensor_user.truncate_in_cast); + + // Reuse the original VarHandleOp if it exists and is in the same block as + // the RestoreV2Op. This avoids redundant VarHandleOps in the IR and + // makes the output cleaner. + mlir::Value handle_result; + if (restored_tensor_user.original_var_handle_op && + restored_tensor_user.original_var_handle_op->getBlock() == + restore_op->getBlock()) { + handle_result = restored_tensor_user.original_var_handle_op.getResult(); + // Since we are reusing an op that originally appeared later, we need to + // move it before the new IfrtRestoreVariableOp to avoid dominance + // violations. + restored_tensor_user.original_var_handle_op->moveBefore(restore_op); + } else { + mlir::TF::VarHandleOp local_handle = + handle_builder.create( + restore_op->getLoc(), restored_tensor_user.var_handle_type, + restored_tensor_user.container, + restored_tensor_user.shared_name); + handle_result = local_handle.getResult(); + } + params.var_handle_values.push_back(handle_result); + + if (!restored_tensor_user.restored_tensor_to_return.has_value()) { + // This tensor is assigned to a variable. + // Delete the path from the RestoreV2Op to the AssignVariableOp. + for (mlir::Operation* op : + restored_tensor_user.op_chain_from_restore_output) { + add_to_erase(op); + } + + // Intercept usages where the restored variable is assigned to a var + // handle 1, and then read from varhandle1 and assigned to var handle 2. + InterceptReadAssignVariable(shared_name_to_reads, tensor_name, slice, + dtype, restored_tensor_user, handle_builder, + restore_op, params, ops_to_erase); + } else { + // This tensor is not assigned to a variable and should be returned by + // IfrtRestoreVariableOp. + mlir::Value tensor = *restored_tensor_user.restored_tensor_to_return; + if (returned_tensors_indices.find(tensor) == + returned_tensors_indices.end()) { + returned_tensors_indices.insert({tensor, result_types.size()}); + result_types.push_back(tensor.getType()); + returned_tensor_names.push_back(restored_tensor_user.tensor_name); + } + } + } + + for (mlir::Operation* op : llvm::reverse(ops_to_erase)) { + if (!op->use_empty()) { + return op->emitOpError() << "is expected to be erased but has uses"; + } + op->erase(); + } + + mlir::OpBuilder builder(restore_op); + + // If tensor_names or shape_and_slices have been modified (e.g., due to + // intercepted read-assign variables), create new constant tensors. + // Otherwise, reuse the original tensors from RestoreV2Op to avoid + // redundancy. + mlir::Value new_tensor_names_op; + if (new_tensor_names.size() == tensor_names_vec.size() && + std::equal(new_tensor_names.begin(), new_tensor_names.end(), + tensor_names_vec.begin())) { + new_tensor_names_op = restore_op.getTensorNames(); + } else { + new_tensor_names_op = CreateStringTensorConst( + builder, restore_op->getLoc(), new_tensor_names); + } + mlir::Value new_shape_and_slices_op; + if (new_shape_and_slices.size() == shape_and_slices_vec.size() && + std::equal(new_shape_and_slices.begin(), new_shape_and_slices.end(), + shape_and_slices_vec.begin())) { + new_shape_and_slices_op = restore_op.getShapeAndSlices(); + } else { + new_shape_and_slices_op = CreateStringTensorConst( + builder, restore_op->getLoc(), new_shape_and_slices); + } + + auto ifrt_restore_variable_op = mlir::TF::IfrtRestoreVariableOp::create( + builder, restore_op->getLoc(), result_types, restore_op.getPrefix(), + new_tensor_names_op, new_shape_and_slices_op, var_handle_values, + builder.getArrayAttr(new_dtypes), + builder.getDenseBoolArrayAttr(new_truncate_in_cast), + builder.getStrArrayAttr(returned_tensor_names)); + + // Replace the original restored tensors with the results of the new + // IfrtRestoreVariableOp. + llvm::SmallPtrSet replaced_tensors; + for (auto& restored_tensor_user : restored_tensor_users) { + if (restored_tensor_user.restored_tensor_to_return.has_value()) { + mlir::Value tensor = *restored_tensor_user.restored_tensor_to_return; + // Replace all uses of the original tensor with the new op's result. + // The `replaced_tensors` set ensures that we only do this once per + // tensor. + if (replaced_tensors.insert(tensor).second) { + tensor.replaceAllUsesWith(ifrt_restore_variable_op.getResult( + returned_tensors_indices[tensor])); + } + } + } + + // Finally, erase the original RestoreV2Op. + if (!restore_op->use_empty()) { + return restore_op->emitOpError() << "failed to identify all users" + "associated with this RestoreV2Op."; + } else { + restore_op.erase(); + } + + return mlir::success(); + } +}; + +} // namespace + +std::unique_ptr> +CreateLowerToIfrtRestoreVariablePass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..72e7f98ed2082a82aab56ea83774ce4cb55e4c88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.cc @@ -0,0 +1,238 @@ +/* Copyright 2026 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.h" + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "stablehlo/dialect/StablehloOps.h" // from @stablehlo + +namespace tensorflow { +namespace ifrt_serving { + +PackInputsPass::PackInputsPass(llvm::ArrayRef slices) + : slices_(slices.begin(), slices.end()) {} + +// TODO(b/445201291): This pass currently has a caveat of losing the original +// user inputs' layout. We should look into this if it turns out to be a +// problem. +void PackInputsPass::runOnOperation() { + mlir::ModuleOp module = getOperation(); + auto main_func = module.lookupSymbol("main"); + if (!main_func || main_func.isPrivate() || main_func.isExternal()) { + module.emitError("Expected a public function 'main'."); + signalPassFailure(); + return; + } + + // If command line option is provided, use it to populate slices_ + if (!slices_flat_list_.empty()) { + if (slices_flat_list_.size() % 3 != 0) { + module.emitError( + "The 'slices' option must be a flat list of integers " + "with a length multiple of 3."); + signalPassFailure(); + return; + } + slices_.clear(); + for (size_t i = 0; i < slices_flat_list_.size(); i += 3) { + slices_.push_back({static_cast(slices_flat_list_[i]), + slices_flat_list_[i + 1], slices_flat_list_[i + 2]}); + } + } + + if (slices_.empty()) { + return; + } + + llvm::SmallVector small_arg_indices; + int64_t total_small_size = 0; + + for (const auto& slice : slices_) { + if (slice.arg_index >= main_func.getNumArguments()) { + module->emitError(absl::StrCat("Slice arg_index ", slice.arg_index, + " is out of range.")); + return signalPassFailure(); + } + + if (llvm::is_contained(small_arg_indices, slice.arg_index)) { + module->emitError( + absl::StrCat("Duplicate slice arg_index: ", slice.arg_index)); + return signalPassFailure(); + } + + mlir::BlockArgument arg = main_func.getArgument(slice.arg_index); + auto tensor_type = llvm::dyn_cast(arg.getType()); + if (!tensor_type) { + module->emitError(absl::StrCat("Argument ", slice.arg_index, + " is not a ranked tensor type.")); + return signalPassFailure(); + } + if (!tensor_type.hasStaticShape()) { + module->emitError(absl::StrCat("Argument ", slice.arg_index, + " does not have static shape.")); + return signalPassFailure(); + } + + int64_t bitwidth = tensor_type.getElementType().getIntOrFloatBitWidth(); + if (bitwidth < 8 || bitwidth % 8 != 0) { + module->emitError( + absl::StrCat("Argument ", slice.arg_index, + " has invalid element bit width: ", bitwidth)); + return signalPassFailure(); + } + + int64_t expected_size = tensor_type.getNumElements() * (bitwidth / 8); + if (slice.size != expected_size) { + module->emitError(absl::StrCat( + "Slice size ", slice.size, " for argument ", slice.arg_index, + " does not match expected byte size ", expected_size)); + return signalPassFailure(); + } + + small_arg_indices.push_back(slice.arg_index); + total_small_size = std::max(total_small_size, slice.start + slice.size); + } + + // Check for overlapping slices + for (size_t i = 0; i < slices_.size(); ++i) { + for (size_t j = i + 1; j < slices_.size(); ++j) { + int64_t start_i = slices_[i].start; + int64_t end_i = start_i + slices_[i].size; + int64_t start_j = slices_[j].start; + int64_t end_j = start_j + slices_[j].size; + if (std::max(start_i, start_j) < std::min(end_i, end_j)) { + module->emitError(absl::StrCat("Slices for argument ", + slices_[i].arg_index, " and ", + slices_[j].arg_index, " overlap.")); + return signalPassFailure(); + } + } + } + + mlir::OpBuilder builder(&getContext()); + mlir::RankedTensorType combined_arg_type = + mlir::RankedTensorType::get({total_small_size}, builder.getI8Type()); + + mlir::Block& entry_block = main_func.front(); + mlir::BlockArgument combined_arg = + entry_block.addArgument(combined_arg_type, main_func.getLoc()); + + builder.setInsertionPointToStart(&entry_block); + for (const auto& slice : slices_) { + mlir::BlockArgument arg = entry_block.getArgument(slice.arg_index); + llvm::SmallVector start_indices = {slice.start}; + llvm::SmallVector limit_indices = {slice.start + slice.size}; + llvm::SmallVector strides = {1}; + auto slice_op = builder.create( + arg.getLoc(), combined_arg, builder.getDenseI64ArrayAttr(start_indices), + builder.getDenseI64ArrayAttr(limit_indices), + builder.getDenseI64ArrayAttr(strides)); + + auto tensor_type = llvm::dyn_cast(arg.getType()); + int64_t targetEltBitWidth = + tensor_type.getElementType().getIntOrFloatBitWidth(); + + // If the original type is one byte, we can just do a reshape. If it is + // larger than one byte, we need to do a reshape and a bitcast to convert + // to the original type. For example, if the original type is F32 of shape + // (x, y), we need to reshape to {x, y, 4} as F32 is 4 bytes, and then + // bitcast to F32 with shape (x, y). + if (targetEltBitWidth == 8) { + auto reshape_op = builder.create( + arg.getLoc(), arg.getType(), slice_op.getResult()); + arg.replaceAllUsesWith(reshape_op.getResult()); + } else { + llvm::SmallVector intermediate_shape( + tensor_type.getShape().begin(), tensor_type.getShape().end()); + intermediate_shape.push_back(targetEltBitWidth / 8); + auto intermediate_type = + mlir::RankedTensorType::get(intermediate_shape, builder.getI8Type()); + + auto reshape_op = builder.create( + arg.getLoc(), intermediate_type, slice_op.getResult()); + + auto bitcast_op = builder.create( + arg.getLoc(), arg.getType(), reshape_op.getResult()); + + arg.replaceAllUsesWith(bitcast_op.getResult()); + } + } + + // Erase arguments + llvm::BitVector small_args_to_erase_bv(entry_block.getNumArguments()); + for (unsigned index : small_arg_indices) { + small_args_to_erase_bv.set(index); + } + entry_block.eraseArguments(small_args_to_erase_bv); + + // Set new function type and attributes + mlir::FunctionType old_func_type = main_func.getFunctionType(); + llvm::SmallVector new_input_types; + llvm::SmallVector new_arg_attrs; + for (unsigned i = 0; i < old_func_type.getNumInputs(); ++i) { + if (!llvm::is_contained(small_arg_indices, i)) { + new_input_types.push_back(old_func_type.getInput(i)); + new_arg_attrs.push_back(main_func.getArgAttrDict(i)); + } + } + new_input_types.push_back(combined_arg_type); + new_arg_attrs.push_back(nullptr); + + auto new_func_type = mlir::FunctionType::get(&getContext(), new_input_types, + old_func_type.getResults()); + main_func.setFunctionType(new_func_type); + main_func.setAllArgAttrs(new_arg_attrs); +} + +void PackInputsPass::getDependentDialects( + mlir::DialectRegistry& registry) const { + registry.insert(); +} + +mlir::StringRef PackInputsPass::getArgument() const { return "pack-inputs"; } + +mlir::StringRef PackInputsPass::getDescription() const { + return "Pack specified tensor inputs of main function into a single i8 " + "tensor."; +} + +std::unique_ptr> CreatePackInputsPass( + llvm::ArrayRef slices) { + return std::make_unique(slices); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..e83ae78c06438ad7bb7f779fdfaed7de269eef88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/pack_inputs_pass.h @@ -0,0 +1,80 @@ +/* Copyright 2026 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_PACK_INPUTS_PASS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_PACK_INPUTS_PASS_H_ + +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project + +namespace mlir { +class DialectRegistry; +} // namespace mlir + +namespace tensorflow { +namespace ifrt_serving { + +struct SliceInfo { + unsigned arg_index; + int64_t start; + int64_t size; +}; + +// This pass packs specified tensor inputs of main function into a single i8 +// tensor buffer using explicit slice coordinates. +class PackInputsPass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PackInputsPass) + + PackInputsPass() = default; + PackInputsPass(const PackInputsPass& other) + : PassWrapper(other), slices_(other.slices_) {} + explicit PackInputsPass(llvm::ArrayRef slices); + + void runOnOperation() final; + + void getDependentDialects(mlir::DialectRegistry& registry) const override; + + mlir::StringRef getArgument() const override; + mlir::StringRef getDescription() const override; + + private: + std::vector slices_; + ListOption slices_flat_list_{ + *this, "slices", + llvm::cl::desc( + "Flat list of integers specifying {arg_index, start, size} " + "for each slice to be packed.")}; +}; + +// Creates an instance of the PackInputsPass. +std::unique_ptr> CreatePackInputsPass( + llvm::ArrayRef slices = {}); + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_PACK_INPUTS_PASS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.td b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.td new file mode 100644 index 0000000000000000000000000000000000000000..5f7c0499b2757cd7b12bf940b13b33e733ca11f2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.td @@ -0,0 +1,158 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +include "mlir/Pass/PassBase.td" + + +def RewriteClusterToIfrtCallPass: Pass<"rewrite-cluster-to-ifrt-call", "mlir::ModuleOp"> { + let summary = "Lowers tf_device.cluster_func to tf.ifrt_call"; + + let description = [{ + This pass converts the clustered tpu program into an IFRT program and rewrites + tf_device.cluster_fun to tf.ifrt_call. + }]; + + let dependentDialects = ["mlir::tf_device::TensorFlowDeviceDialect"]; + + let constructor = "CreateRewriteClusterToIfrtCallPass()"; + let options = [ + Option<"enable_async_ifrt", "enable-async-ifrt", "bool", "false", + "Enable async IFRT execution."> + ]; + } + +def PropagateStaticShapesPass : Pass<"propagate-static-shapes", "mlir::ModuleOp"> { + let summary = "Propagates static shapes from `tf.SetStaticDimensionBounds` to `tf.IfrtCall`"; + + let description = [{ + This pass propagates static shape information from `tf.SetStaticDimensionBounds` + to `tf.IfrtCall` operands and the arguments of its callee function. + `tf.SetStaticDimensionBounds` is then replaced with its input. + }]; + + let constructor = "CreatePropagateStaticShapesPass()"; +} + +def SinkVariableAsNamedArrayPass: Pass<"sink-variable-as-named-array", "mlir::ModuleOp"> { + let summary = "Sink variable tensor for tpu device as named IFRT array for tf.IfrtCall"; + + let description = [{ + This pass sinks variable tensor argument to `tf.IfrtCall` as variable_arg_indices + and variable_names attributes and also lowers `tf.ReadVariableOp` to + `tf.IfrtLoadVariableOp`. + + The runtime ensures that `tf.IfrtCall` kernel can bind the IFRT array by + its name as input to the TPU program. + + }]; + + let constructor = "CreateSinkVariableAsNamedArrayPass()"; +} + +def LowerToIfrtRestoreVariablePass: Pass<"lower-to-ifrt-restore-variable", "mlir::ModuleOp"> { + let summary = "Lowers tf.RestoreV2 to tf.IfrtRestoreVariable"; + + let description = [{ + This pass rewrites `RestoreV2Op` and `AssignVariableOp` to `IfrtRestoreVariableOp`. + + This pass is ran so that the runtime may run restore operation asynchronously + such that the expensive resotre operation can be parallerized with other + model compilation operation. + }]; + + let constructor = "CreateLowerToIfrtRestoreVariablePass()"; + } + +def TfRestorePruningPass + : Pass<"tf-restore-pruning", "mlir::func::FuncOp"> { + let summary = "Prune unused`tf.RestoreV2` ops"; + + let description = [{ + This pass prune unused `tf.RestoreV2` op. A typical use case is to combine + `TfRestoreSplittingPass`, this pass and `TfRestoreMergingPass` in sequence + so that the un-used restored tensors are not read into host memory. + }]; + + let constructor = "CreateTfRestorePruningPass()"; +} + + +def TfRestoreSplittingPass + : Pass<"tf-restore-splitting", "mlir::func::FuncOp"> { + let summary = "Splits `tf.RestoreV2` ops"; + + let description = [{ + This pass splits each `tf.RestoreV2` op so that one restore op handles one + variable only. This pass can split restore ops only if the tensor names and + the shape/slices arguments are constants, which is usually the case. + + Splitting monolithic restore ops into per-tensor restore ops makes it easier + to shard SavedModel initialization across multiple clusters. + }]; + + let constructor = "CreateTfRestoreSplittingPass()"; +} + +def TfRestoreMergingPass : Pass<"tf-restore-merging", "mlir::func::FuncOp"> { + let summary = "Merges `tf.RestoreV2` ops"; + + let description = [{ + This pass merges multiple `tf.RestoreV2` ops into one `tf.RestoreV2` op + using variadic results. The current implementation merges restore ops only + if they have the same `prefix` and have constant tensor names and + shape/slice arguments. + + This pass is run in order to undo `tf-restore-splitting` after cluster + formation and reduce the op dispatch overhead. + }]; + + let constructor = "CreateTfRestoreMergingPass()"; +} + +def TfIdentityPropagationPass + : Pass<"tf-identity-propagation", "mlir::func::FuncOp"> { + let summary = "Propagates inputs of no-op identity ops to their outputs"; + + let description = [{ + This pass finds identity ops that are no-op and propagates their inputs + directly to outputs so that identity ops can be skipped. + + One example of identity ops that are not no-op is identity ops with XLA + sharding annotation. Since some models use identity ops with `_XlaSharding` + attributes to change output sharding, this pass doesn't propagate the inputs + of such identity ops in order to preserve the sharding changes. + + This pass is useful to make sure that ineffective identity ops don't affect + the graph partitioning. For example, in a pipelined model, if there is a CPU + identity op between two TPU computation stages (which sometimes happens + because TensorFlow inserts it), this will unnecessarily route the + intermediate tensors through the CPU device. By forwarding the inputs of the + identity op directly to its outputs, we can avoid such inefficiency. + }]; + + let constructor = "CreateTfIdentityPropagationPass()"; +} + +def TfDeviceCleanupPass : Pass<"tf-device-cleanup", "mlir::func::FuncOp"> { + let summary = "Cleans up device attributes from all ops"; + + let description = [{ + This pass removes `device` attributes from all TF ops. Some Serving + doesn't rely on `device` attributes from SavedModel. + }]; + + let constructor = "CreateTfDeviceCleanupPass()"; +} + diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/propagate_static_shapes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/propagate_static_shapes.cc new file mode 100644 index 0000000000000000000000000000000000000000..a3cacfad87eda5724c92643661c9e5a3039b18f9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/propagate_static_shapes.cc @@ -0,0 +1,332 @@ +/* Copyright 2026 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/ValueRange.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_type.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +using mlir::TF::AsyncIfrtCallOp; +using mlir::TF::IfrtCallOp; +using mlir::TF::SetStaticDimensionBoundsOp; + +// Identifies operands of `IfrtCallOp` or `AsyncIfrtCallOp` that are defined by +// `SetStaticDimensionBoundsOp` and returns a map from operand index to the +// defining op. +template +llvm::SmallDenseMap +GetArgIdxToStaticShapeOpMap(OpT ifrt_call) { + llvm::SmallDenseMap + arg_idx_to_static_shape_op; + for (const auto& [i, arg] : llvm::enumerate(ifrt_call.getArgs())) { + mlir::Value cur_arg = arg; + // We only check `IdentityOp` (not `IdentityNOp`) because + // `SetStaticDimensionBoundsOp` returns a single tensor. + while (auto identity = cur_arg.getDefiningOp()) { + cur_arg = identity.getInput(); + } + if (auto static_shape_op = + cur_arg.getDefiningOp()) { + VLOG(2) << "Found SetStaticDimensionBoundsOp for arg " << i; + arg_idx_to_static_shape_op[i] = static_shape_op; + } + } + return arg_idx_to_static_shape_op; +} + +// Replaces `old_op` with a new op with `updated_args` and +// `static_shape_args`. Returns the new op. +template +OpT UpdateIfrtCallOp(OpT old_op, + const llvm::SmallVector& updated_args, + const llvm::SmallVector& static_shape_args, + mlir::OpBuilder& builder) { + // Clone to a new op. + builder.setInsertionPoint(old_op); + auto new_op = mlir::cast(builder.clone(*old_op)); + // Wire up inputs. + llvm::SmallVector new_operands; + new_operands.append(updated_args.begin(), updated_args.end()); + new_operands.append(static_shape_args.begin(), static_shape_args.end()); + new_op->setOperands(new_operands); + // This attr is to delimit the boundary between original arguments and static + // shape arguments. + new_op->setAttr("operandSegmentSizes", + builder.getDenseI32ArrayAttr( + {static_cast(updated_args.size()), + static_cast(static_shape_args.size())})); + // Wire up outputs. + old_op.replaceAllUsesWith(new_op.getResults()); + // Get rid of the old op. + old_op.erase(); + return new_op; +} + +// Updates TPUCompileMetadataProto in function attributes to include new +// arguments for static shapes. Each static shape is added as a new parameter +// argument with replicated sharding. +mlir::LogicalResult UpdateTpuCompileMetadata( + mlir::func::FuncOp func_op, + const llvm::SmallVector& static_shape_args) { + // Read the old metadata from the func op. + auto metadata_attr = + func_op->getAttrOfType(kMetadataTextAttrName); + if (!metadata_attr) return mlir::success(); + tensorflow::tpu::TPUCompileMetadataProto metadata; + if (!tsl::protobuf::TextFormat::ParseFromString( + metadata_attr.getValue().str(), &metadata)) { + return func_op.emitError() << "Failed to parse " << kMetadataTextAttrName; + } + // Update metadata proto to include static shape args. + for (const auto& static_shape_arg : static_shape_args) { + auto* new_compile_arg = metadata.add_args(); + // NOTE: tf2hlo.cc strictly requires PARAMETER kind for all args. + new_compile_arg->set_kind( + tensorflow::tpu::TPUCompileMetadataProto::Arg::PARAMETER); + tensorflow::DataType dtype; + if (const auto status = + tensorflow::ConvertToDataType(static_shape_arg.getType(), &dtype); + !status.ok()) { + return func_op.emitError() << "Failed to convert static shape type to " + "TensorFlow DataType: " + << status.message(); + } + new_compile_arg->set_dtype(dtype); + tensorflow::ConvertTypeToTensorShape(static_shape_arg.getType()) + .AsProto(new_compile_arg->mutable_shape()); + new_compile_arg->mutable_sharding()->set_type(xla::OpSharding::REPLICATED); + } + // Write the new metadata to the func op. + std::string new_metadata_str; + if (!tsl::protobuf::TextFormat::PrintToString(metadata, &new_metadata_str)) { + return func_op.emitError() + << "Failed to serialize " << kMetadataTextAttrName; + } + func_op->setAttr( + kMetadataTextAttrName, + mlir::StringAttr::get(func_op.getContext(), new_metadata_str)); + return mlir::success(); +} + +// Updates the `FuncOp` identified by `program_id` to incorporate +// `static_shape_args`. This includes setting `tf._static_shape_arg` attributes +// on original arguments, updating the function signature, adding block +// arguments, and updating the `TPUCompileMetadataProto`. +template +mlir::LogicalResult UpdateCalleeFuncOp( + mlir::ModuleOp module, OpT ifrt_call, + const llvm::SmallVector& static_shape_args, + const llvm::SmallDenseMap& arg_idx_to_static_shape_idx) { + // Find the callee `FuncOp` by program ID. + mlir::func::FuncOp func_op; + // NOTE: IFRT program functions are generated by + // `RewriteClusterToIfrtCallPass` and inserted into the module's symbol table, + // so they are always top-level operations. + for (auto f : module.getOps()) { + auto id_attr = + f->getAttrOfType("tfrt_ifrt_serving.program_id"); + // NOTE: Cast is used because this ID is uint64_t in `IfrtCallOp` but + // signless (aka int64_t in MLIR's sense) in `FuncOp`. + if (id_attr && + static_cast(id_attr.getInt()) == ifrt_call.getProgramId()) { + func_op = f; + break; + } + } + if (!func_op) { + return ifrt_call.emitOpError() << "callee func with program_id " + << ifrt_call.getProgramId() << " not found"; + } + // Point original args to new static shape args. + // + // The `tf._static_shape_arg_idx` attribute on an original argument indicates + // the index of the corresponding static shape argument in the updated + // function signature. For example, if the original arguments are (arg0, arg1, + // arg2) and only arg0 and arg2 have static shapes (ss0, ss2), the new + // signature is (arg0, arg1, arg2, ss0, ss2). Then arg0 will have + // `tf._static_shape_arg_idx = 3` and arg2 will have + // `tf._static_shape_arg_idx = 4`. + mlir::OpBuilder builder(func_op); + for (const auto& [arg_idx, static_shape_idx] : arg_idx_to_static_shape_idx) { + func_op.setArgAttr( + arg_idx, "tf._static_shape_arg_idx", + builder.getI32IntegerAttr(static_cast(static_shape_idx))); + } + // Update function type to include static shape args. + const auto& old_function_type = func_op.getFunctionType(); + llvm::SmallVector new_input_types( + old_function_type.getInputs().begin(), + old_function_type.getInputs().end()); + for (const auto& arg : static_shape_args) { + new_input_types.push_back(arg.getType()); + } + func_op.setType(mlir::FunctionType::get(module.getContext(), new_input_types, + old_function_type.getResults())); + // Append static shape args to the entry block. + for (const auto& arg : static_shape_args) { + func_op.getBody().front().addArgument(arg.getType(), func_op.getLoc()); + } + return UpdateTpuCompileMetadata(func_op, static_shape_args); +} + +template +mlir::LogicalResult ProcessIfrtCall( + OpT ifrt_call, mlir::ModuleOp module, mlir::OpBuilder& op_builder, + llvm::DenseSet& processed_programs) { + llvm::SmallDenseMap + arg_idx_to_static_shape_op = GetArgIdxToStaticShapeOpMap(ifrt_call); + if (arg_idx_to_static_shape_op.empty()) return mlir::success(); + + VLOG(2) << "Found `IfrtCallOp` with " << arg_idx_to_static_shape_op.size() + << " static shaped args. IfrtCallOp: " << ifrt_call; + + mlir::OperandRange old_args = ifrt_call.getArgs(); + llvm::SmallVector updated_args; + updated_args.resize(old_args.size()); + llvm::SmallVector static_shape_args; + llvm::SmallDenseMap arg_idx_to_static_shape_idx; + for (const auto& [i, arg] : llvm::enumerate(old_args)) { + auto iter = arg_idx_to_static_shape_op.find(i); + if (iter != arg_idx_to_static_shape_op.end()) { + SetStaticDimensionBoundsOp static_shape_op = iter->second; + arg_idx_to_static_shape_idx[i] = + old_args.size() + static_shape_args.size(); + static_shape_args.push_back(static_shape_op.getStaticShape()); + updated_args[i] = static_shape_op.getInput(); + } else { + updated_args[i] = arg; + } + } + uint64_t program_id = ifrt_call.getProgramId(); + auto new_ifrt_call = + UpdateIfrtCallOp(ifrt_call, updated_args, static_shape_args, op_builder); + + // Defensive coding behavior: Ensure we only update the callee FuncOp once + // per program_id. In theory, it is highly unlikely that the same program + // is called by both IfrtCallOp and AsyncIfrtCallOp (or multiple times with + // different static shape bounds) in the same module, because the choice of + // op type is usually controlled by a session-wide flag (e.g., + // enable_async_ifrt). However, protecting against it avoids duplicate + // argument promotion in the callee signature. + if (!processed_programs.contains(program_id)) { + if (mlir::failed(UpdateCalleeFuncOp(module, new_ifrt_call, + static_shape_args, + arg_idx_to_static_shape_idx))) { + return mlir::failure(); + } + processed_programs.insert(program_id); + } + + // Erase all `SetStaticDimensionBounds` ops. + for (auto [_, op] : arg_idx_to_static_shape_op) { + op.replaceAllUsesWith(op.getOperand(0)); + op.erase(); + } + return mlir::success(); +} + +// Pass that propagates static shapes from `tf.SetStaticDimensionBoundsOp` to +// `IfrtCallOp` and callee `FuncOp`. +struct PropagateStaticShapesPass : public mlir::OperationPass { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PropagateStaticShapesPass) + + PropagateStaticShapesPass() + : mlir::OperationPass( + mlir::TypeID::get()) {} + PropagateStaticShapesPass(const PropagateStaticShapesPass& other) = default; + + llvm::StringRef getArgument() const override { + return "propagate-static-shapes"; + } + llvm::StringRef getDescription() const override { + return "Propagates static shapes from tf.SetStaticDimensionBoundsOp to " + "IfrtCallOp and callee FuncOp."; + } + llvm::StringRef getName() const override { + return "PropagateStaticShapesPass"; + } + + std::unique_ptr clonePass() const override { + return std::make_unique(*this); + } + + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::OpBuilder op_builder(module.getContext()); + + llvm::DenseSet processed_programs; + + // Collect all calls in the module, before any modifications. + llvm::SmallVector ifrt_calls; + llvm::SmallVector async_ifrt_calls; + module.walk([&ifrt_calls](IfrtCallOp op) { ifrt_calls.push_back(op); }); + module.walk([&async_ifrt_calls](AsyncIfrtCallOp op) { + async_ifrt_calls.push_back(op); + }); + + for (auto ifrt_call : ifrt_calls) { + if (mlir::failed(ProcessIfrtCall(ifrt_call, module, op_builder, + processed_programs))) { + return signalPassFailure(); + } + } + for (auto ifrt_call : async_ifrt_calls) { + if (mlir::failed(ProcessIfrtCall(ifrt_call, module, op_builder, + processed_programs))) { + return signalPassFailure(); + } + } + } +}; + +} // namespace + +std::unique_ptr> +CreatePropagateStaticShapesPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/rewrite_cluster_to_ifrt_call.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/rewrite_cluster_to_ifrt_call.cc new file mode 100644 index 0000000000000000000000000000000000000000..ec413f3ce7b6045a8be88f402ce67643e78befaa --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/rewrite_cluster_to_ifrt_call.cc @@ -0,0 +1,387 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/WalkResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_structs.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/tpu_metadata_utils.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/device_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/tpu_rewrite_device_util.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h" +#include "tensorflow/core/platform/random.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DECL_REWRITECLUSTERTOIFRTCALLPASS +#define GEN_PASS_DEF_REWRITECLUSTERTOIFRTCALLPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +// A pass that inserts tf.ifrt_call and create its callee as a Ifrt +// Program. +class RewriteClusterToIfrtCallPass + : public impl::RewriteClusterToIfrtCallPassBase< + RewriteClusterToIfrtCallPass> { + public: + explicit RewriteClusterToIfrtCallPass(bool arg_enable_async_ifrt = false) { + this->enable_async_ifrt = arg_enable_async_ifrt; + } + + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::SymbolTable symbol_table(module); + mlir::SymbolTableCollection symbol_table_collection; + + mlir::TF::RuntimeDevices devices; + if (failed(tensorflow::GetDevicesFromOp(module.getOperation(), &devices))) + return signalPassFailure(); + + // key: original callee function in tf_device.cluster_func. value: ifrt + // program. + llvm::DenseMap + cluster_to_ifrt_program; + + std::vector cluster_func_ops; + module.walk([&](mlir::tf_device::ClusterFuncOp cluster_func) { + cluster_func_ops.push_back(cluster_func); + }); + for (auto cluster_func : cluster_func_ops) { + Rewrite(symbol_table, symbol_table_collection, cluster_to_ifrt_program, + cluster_func, devices); + } + + // TODO(b/304839793): Move this to a separate pass. The old remove + // compilation result pass rely on TPUPartitionedCall + llvm::SmallVector compilation_result_ops; + module.walk([&](mlir::TF::TPUCompilationResultOp op) { + compilation_result_ops.push_back(op); + }); + for (auto op : compilation_result_ops) { + if (!op.use_empty()) { + module->emitError("TPUCompilationResultOp is under use"); + return signalPassFailure(); + } + op.erase(); + } + } + + private: + // Returns a new unique program id. + static int64_t NewProgramId() { + const uint64_t id = static_cast(tensorflow::random::New64()); + // We use a signed int for program ids since TensorFlow doesn't + // support uint64_t attributes. + return absl::bit_cast(id); + } + + mlir::LogicalResult GetTpuCompileMetadata( + mlir::tf_device::ClusterFuncOp cluster_func, + mlir::TF::RuntimeDevices &devices, + tensorflow::tpu::TPUCompileMetadataProto *metadata) { + // Collect `num_replicas` and `num_cores_per_replica` attributes. + int num_replicas = 1; + mlir::tf_device::ReplicateOp replicate = + cluster_func->getParentOfType(); + if (replicate) num_replicas = replicate.getN(); + + auto num_cores_per_replica_attr = + cluster_func->getAttrOfType( + tensorflow::kNumCoresPerReplicaAttr); + if (!num_cores_per_replica_attr) + return cluster_func.emitOpError() + << "Attribute" << tensorflow::kNumCoresPerReplicaAttr + << " is missing"; + int num_cores_per_replica = num_cores_per_replica_attr.getInt(); + + return mlir::TFTPU::SetMetadataProtoFromClusterFuncOp( + cluster_func, num_replicas, num_cores_per_replica, + /*xla_device_assignment=*/std::nullopt, metadata); + } + + // Returns true if the function or any of its callees recursively contains + // tf.PwStreamResultsOp. + // + // PwStreamResultsOp immediately streams results back to the serving + // controller. We must disable async execution (falling back to synchronous + // IfrtCallOp) if this op is present. Otherwise, the async call might finish + // without properly synchronizing or awaiting the streaming results, causing + // race conditions or broken serving behavior (see b/500390771). + // + // Complexity Note: This method must recursively traverse control flow ops + // (e.g., tf.If, tf.While) and partitioned calls. Since these ops frequently + // reference callee functions via an ArrayAttr of symbols rather than a simple + // SymbolRefAttr, we explicitly iterate through ArrayAttr elements to avoid + // missing streaming ops hidden inside branches. + bool ContainsStreamingOp(mlir::func::FuncOp func, + mlir::SymbolTableCollection& symbol_table_collection, + llvm::DenseSet& visited) { + // Track visited functions to safely handle recursive call graphs and + // prevent infinite loops. + if (!func || !visited.insert(func).second) return false; + + VLOG(1) << "Checking function for streaming ops: " << func.getName().str(); + + // Walk all operations within the current function's body. + auto walk_result = func.walk([&](mlir::Operation* op) { + // Base case - If we find the streaming op directly, interrupt the + // walk and signal discovery. + if (mlir::isa(op)) { + VLOG(1) << "Found tf.PwStreamResults in " << func.getName().str(); + return mlir::WalkResult::interrupt(); + } + + // Recursive case - Many ops (like tf.If, tf.While, tf.PartitionedCall) + // don't hold operations directly but instead reference callee functions + // via attributes. We need to inspect these attributes to traverse the + // call graph. + for (const mlir::NamedAttribute& attr : op->getAttrs()) { + if (auto sym_ref = + mlir::dyn_cast(attr.getValue())) { + // Handle single function references (e.g., tf.PartitionedCall). + // Resolve the symbol to the actual function and recursively check it. + VLOG(1) << "Looking up symbol: " << sym_ref.getLeafReference().str() + << " from op: " << op->getName().getStringRef().str(); + if (auto callee = symbol_table_collection + .lookupNearestSymbolFrom( + op, sym_ref)) { + if (ContainsStreamingOp(callee, symbol_table_collection, visited)) { + return mlir::WalkResult::interrupt(); + } + } + } else if (auto arr_attr = + mlir::dyn_cast(attr.getValue())) { + // Handle arrays of function references (e.g., branches in tf.If). + // We must check every referenced function in the array. + for (auto elt : arr_attr.getValue()) { + if (auto sym_ref = mlir::dyn_cast(elt)) { + VLOG(1) << "Looking up symbol from array: " + << sym_ref.getLeafReference().str() + << " from op: " << op->getName().getStringRef().str(); + if (auto callee = + symbol_table_collection + .lookupNearestSymbolFrom( + op, sym_ref)) { + if (ContainsStreamingOp(callee, symbol_table_collection, + visited)) { + return mlir::WalkResult::interrupt(); + } + } + } + } + } + } + return mlir::WalkResult::advance(); + }); + + // If the walk was interrupted, it means we found the streaming op + // somewhere in this function or its callees. + return walk_result.wasInterrupted(); + } + + void Rewrite(mlir::SymbolTable& symbol_table, + mlir::SymbolTableCollection& symbol_table_collection, + llvm::DenseMap& + cluster_to_ifrt_program, + mlir::tf_device::ClusterFuncOp cluster_func, + mlir::TF::RuntimeDevices& devices) { + mlir::OpBuilder builder(cluster_func); + mlir::FlatSymbolRefAttr callee_symbol = cluster_func.getFuncAttr(); + mlir::func::FuncOp callee_func = + symbol_table.lookup(callee_symbol.getValue()); + + // TODO - b/500390771: Ideally we need to make the await op the preceding + // op of the PwStreamResultsOp, which can be achieved in the tf-to-mlrt + // lowering pass. + bool use_async = enable_async_ifrt; + if (use_async) { + llvm::DenseSet visited; + if (ContainsStreamingOp(callee_func, symbol_table_collection, visited)) { + cluster_func->emitRemark( + "Disabling AsyncIfrtCallOp because tf.PwStreamResults is present " + "in the cluster."); + use_async = false; + } + } + + auto ifrt_program_name = + absl::StrCat("_ifrt_program_", callee_func.getSymName().str()); + if (mlir::func::FuncOp ifrt_program = + cluster_to_ifrt_program[callee_func]) { + // ifrt program already exists + builder.setInsertionPoint(cluster_func); + + mlir::Operation* ifrt_call_op; + if (use_async) { + ifrt_call_op = mlir::TF::AsyncIfrtCallOp::create( + builder, cluster_func->getLoc(), cluster_func.getResultTypes(), + cluster_func->getOperands()); + } else { + ifrt_call_op = mlir::TF::IfrtCallOp::create( + builder, cluster_func->getLoc(), cluster_func.getResultTypes(), + cluster_func->getOperands()); + } + + ifrt_call_op->setAttr( + "operandSegmentSizes", + builder.getDenseI32ArrayAttr( + {static_cast(cluster_func.getNumOperands()), 0})); + + int64_t program_id; + if (auto attr = ifrt_program->getAttrOfType( + "tfrt_ifrt_serving.program_id")) { + program_id = attr.getInt(); + } else { + return signalPassFailure(); + } + + auto metadata_attr = + ifrt_program->getAttrOfType(kMetadataTextAttrName); + auto device_assignment_attr = + ifrt_program->getAttrOfType(kDeviceAssignmentAttr); + if (!metadata_attr || !device_assignment_attr) { + return signalPassFailure(); + } + + // For better debuggability, attach attributes such as + // tpu_compile_metadata and device_assignment to IfrtCallOp. + ifrt_call_op->setAttr(kMetadataTextAttrName, metadata_attr); + ifrt_call_op->setAttr(kDeviceAssignmentAttr, device_assignment_attr); + + // TODO(b/304839793): populate variable names after adding a variable + // hoisting pass. + ifrt_call_op->setAttr("variable_arg_indices", + builder.getI32ArrayAttr({})); + ifrt_call_op->setAttr("program_id", + builder.getI64IntegerAttr(program_id)); + + cluster_func->replaceAllUsesWith(ifrt_call_op->getResults()); + cluster_func->erase(); + + return; + } + mlir::OpBuilder::InsertionGuard insertion_guard(builder); + builder.setInsertionPoint(callee_func); + + mlir::func::FuncOp cloned_ifrt_program = mlir::func::FuncOp::create( + builder, callee_func->getLoc(), ifrt_program_name, + callee_func.getFunctionType()); + mlir::IRMapping mapper; + callee_func.cloneInto(cloned_ifrt_program, mapper); + + tensorflow::tpu::TPUCompileMetadataProto metadata; + if (mlir::failed(GetTpuCompileMetadata(cluster_func, devices, &metadata))) { + return signalPassFailure(); + } + std::string serialized_metadata; + tsl::protobuf::TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.PrintToString(metadata, &serialized_metadata); + + cloned_ifrt_program->setAttr(kMetadataTextAttrName, + builder.getStringAttr(serialized_metadata)); + + auto device_assignment_attr = + cluster_func->getAttrOfType(kDeviceAssignmentAttr); + if (!device_assignment_attr) { + device_assignment_attr = builder.getArrayAttr({}); + } + cloned_ifrt_program->setAttr(kDeviceAssignmentAttr, device_assignment_attr); + + cloned_ifrt_program.setName(ifrt_program_name); + + int64_t program_id = NewProgramId(); + cloned_ifrt_program->setAttr("tfrt_ifrt_serving.program_id", + builder.getI64IntegerAttr(program_id)); + + // Make cloned ifrt program public so that it does not get dropped by + // inliner. + cloned_ifrt_program.setPublic(); + + builder.setInsertionPoint(cluster_func); + + mlir::Operation* ifrt_call_op; + if (use_async) { + ifrt_call_op = mlir::TF::AsyncIfrtCallOp::create( + builder, cluster_func->getLoc(), cluster_func.getResultTypes(), + cluster_func->getOperands()); + } else { + ifrt_call_op = mlir::TF::IfrtCallOp::create( + builder, cluster_func->getLoc(), cluster_func.getResultTypes(), + cluster_func->getOperands()); + } + + ifrt_call_op->setAttr( + "operandSegmentSizes", + builder.getDenseI32ArrayAttr( + {static_cast(cluster_func.getNumOperands()), 0})); + + // TODO(b/304839793): populate variable names after adding a variable + // hoisting pass. + ifrt_call_op->setAttr("variable_arg_indices", builder.getI32ArrayAttr({})); + ifrt_call_op->setAttr("program_id", builder.getI64IntegerAttr(program_id)); + // For better debuggability, attach attributes such as tpu_compile_metadata + // and device_assignment to IfrtCallOp. + ifrt_call_op->setAttr(kMetadataTextAttrName, + builder.getStringAttr(serialized_metadata)); + ifrt_call_op->setAttr(kDeviceAssignmentAttr, device_assignment_attr); + + cluster_func->replaceAllUsesWith(ifrt_call_op->getResults()); + cluster_func->erase(); + + symbol_table.insert(cloned_ifrt_program); + cluster_to_ifrt_program[callee_func] = cloned_ifrt_program; + } +}; +} // namespace + +std::unique_ptr> +CreateRewriteClusterToIfrtCallPass(bool enable_async_ifrt) { + return std::make_unique(enable_async_ifrt); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/sink_variable_as_named_array.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/sink_variable_as_named_array.cc new file mode 100644 index 0000000000000000000000000000000000000000..a8f98a5a428f50ed8f7a67b3fca6d3bea5b7a945 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/sink_variable_as_named_array.cc @@ -0,0 +1,255 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h" +#include "xla/service/computation_placer.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/platform/protobuf.h" // IWYU pragma: keep +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/tfrt/ifrt/ifrt_config.pb.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_SINKVARIABLEASNAMEDARRAYPASS +#define GEN_PASS_DECL_SINKVARIABLEASNAMEDARRAYPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +class SinkVariableAsNamedArrayPass + : public impl::SinkVariableAsNamedArrayPassBase< + SinkVariableAsNamedArrayPass> { + public: + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::OpBuilder builder(&getContext()); + + // Rewrite ReadVariableOp with IfrtLoadVariableOp + llvm::SmallDenseMap + read_to_load; + + mlir::WalkResult walk_result = + module.walk([&](mlir::TF::ReadVariableOp read_variable_op) { + // TODO(b/319045348): consider use resource alias analysis for + // this. + auto var_handle = GetDefiningOp( + read_variable_op.getResource()); + + if (!var_handle) { + read_variable_op->emitError( + "ReadVariableOp has no defining VarHandleOp."); + return mlir::WalkResult::interrupt(); + } + + // Avoid lowering ReadVariableOp to IfrtLoadVariableOp if the + // assignment AssignVariableOp happens at the same module because + // IfrtLoadVariableOp assumes asynchronous assignment of the variable. + for (auto var_handle_user : var_handle->getUsers()) { + if (llvm::isa(var_handle_user)) { + return mlir::WalkResult::advance(); + } + } + std::vector result_types; + result_types.push_back(mlir::RankedTensorType::get( + {}, builder.getType())); + result_types.push_back(read_variable_op.getResult().getType()); + + builder.setInsertionPointAfter(read_variable_op); + auto load_variable_op = mlir::TF::IfrtLoadVariableOp::create( + builder, read_variable_op->getLoc(), result_types, + var_handle.getResult()); + read_to_load[read_variable_op] = load_variable_op; + + return mlir::WalkResult::advance(); + }); + + if (walk_result.wasInterrupted()) { + return signalPassFailure(); + } + + // Rewrite ifrt call: variable tensors are sunk as attribute. + // The runtime guarantees the binding of corresponding loaded ifrt array + // based on attributes. + auto process_call = [&](auto call) -> mlir::WalkResult { + IfrtArgConfigList ifrt_call_argument_configs; + + if (mlir::failed(BuildIfrtCallArgumentConfig( + call.getOperation(), ifrt_call_argument_configs))) { + return mlir::WalkResult::interrupt(); + } + + if (!call.getVariableArgIndicesAttr().empty()) { + call->emitError() << "Expect empty " + << call.getVariableArgIndicesAttrName().str() + << " attributes, but got " + << call.getVariableArgIndicesAttr().size() + << " elements"; + return mlir::WalkResult::interrupt(); + } + if (call->getOpOperands().size() != ifrt_call_argument_configs.size()) { + call->emitError() << "IfrtCallOp got " << call->getOpOperands().size() + << " operands, but expects " + << ifrt_call_argument_configs.size(); + return mlir::WalkResult::interrupt(); + } + + llvm::SmallVector variable_arg_indices; + llvm::SmallVector updated_args; + + for (const auto& [arg_idx, arg] : + llvm::enumerate(ifrt_call_argument_configs)) { + if (arg.is_variable) { + variable_arg_indices.push_back(arg_idx); + // Variable use the key from IfrtLoadVariable. + updated_args.push_back( + read_to_load[arg.read_variable_op].getArrayKey()); + } else { + // non variable + updated_args.push_back(call->getOperand(arg_idx)); + } + } + + builder.setInsertionPointAfter(call); + mlir::Operation* updated_ifrt_call; + if (llvm::isa(call)) { + updated_ifrt_call = mlir::TF::AsyncIfrtCallOp::create( + builder, call->getLoc(), call.getResultTypes(), updated_args); + } else { + updated_ifrt_call = mlir::TF::IfrtCallOp::create( + builder, call->getLoc(), call.getResultTypes(), updated_args); + } + + updated_ifrt_call->setAttrs(call->getAttrs()); + // Update variable_arg_indices attribute. + updated_ifrt_call->setAttr("variable_arg_indices", + builder.getI32ArrayAttr(variable_arg_indices)); + + call.replaceAllUsesWith(updated_ifrt_call); + call.erase(); + return mlir::WalkResult::advance(); + }; + + if (module + .walk([&](mlir::TF::IfrtCallOp call) { return process_call(call); }) + .wasInterrupted()) { + return signalPassFailure(); + } + + if (module + .walk([&](mlir::TF::AsyncIfrtCallOp call) { + return process_call(call); + }) + .wasInterrupted()) { + return signalPassFailure(); + } + + // Remove all ReadVariableOp after replacing the CPU usage of + // ReadVariableOp. + for (auto& [read_variable_op, load_variable_op] : read_to_load) { + if (!read_variable_op->use_empty()) { + // This variable tensor is used by CPU host. + load_variable_op.setUsedByHost(true); + + // Replace CPU use of ReadVariableOp + read_variable_op.replaceAllUsesWith(load_variable_op.getTensorFuture()); + } + read_variable_op.erase(); + } + } + + private: + struct IfrtArgConfig { + bool is_variable; + mlir::TF::ReadVariableOp read_variable_op; + }; + using IfrtArgConfigList = llvm::SmallVector; + + // Build argument configuration map of a IfrtCallOp. + mlir::LogicalResult BuildIfrtCallArgumentConfig(mlir::Operation* call, + IfrtArgConfigList& args) { + for (const auto& [arg_idx, input] : llvm::enumerate(call->getOperands())) { + // Assuming the nested function calls are inlined. + if (auto read_variable_op = + GetDefiningOp(input)) { + args.push_back( + {.is_variable = true, .read_variable_op = read_variable_op}); + } else { + args.push_back({.is_variable = false}); + } + } + + return mlir::success(); + } + + template + OpT GetDefiningOp(const mlir::Value& value) { + mlir::Operation* op = value.getDefiningOp(); + + while (op && !llvm::isa(op)) { + if (llvm::isa(op)) { + op = op->getOperand(0).getDefiningOp(); + } else { + return nullptr; + } + } + + if (op != nullptr) { + return llvm::dyn_cast(op); + } else { + return nullptr; + } + } +}; + +} // namespace + +std::unique_ptr> +CreateSinkVariableAsNamedArrayPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..02254776dd26b7a1c080eaa5671458b0c6e6e3bc --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/BUILD @@ -0,0 +1,15 @@ +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//learning/brain/tfrt/ifrt:__subpackages__", + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + ], + licenses = ["notice"], +) + +filegroup( + name = "testdata", + srcs = glob( + ["*"], + ), +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/ifrt_cluster.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/ifrt_cluster.mlir new file mode 100644 index 0000000000000000000000000000000000000000..97d0ee1a5493f523f0ec2adff65c21ec17c46a4d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/ifrt_cluster.mlir @@ -0,0 +1,9 @@ +module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:worker/replica:0/task:0/device:CPU:0", "/job:worker/replica:0/task:0/device:TPU_SYSTEM:0", "/job:worker/replica:0/task:0/device:TPU:0"]} { + func.func @main() { + "tf_device.cluster_func"() {_replication_info = "cluster0", func = @empty_func, num_cores_per_replica = 1, step_marker_location = "", input_sharding_configuration = [], output_sharding_configuration = [], use_spmd_for_xla_partitioning = false} : () -> () + func.return + } + func.func @empty_func() { + func.return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/restore_with_reference.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/restore_with_reference.mlir new file mode 100644 index 0000000000000000000000000000000000000000..59e6d87be23e41b52442119b9fe75683d856ac64 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/restore_with_reference.mlir @@ -0,0 +1,14 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}, tf_saved_model.semantics} { + "tf_saved_model.session_initializer"() <{initializers = [@"save/restore_all_1"]}> : () -> () + func.func @"save/restore_all_1"() attributes {tf_saved_model.exported_names = ["__tf_saved_model_session_initializer_save/restore_all_1"], tf_saved_model.initializer_type = "restore_op"} { + %cst = "tf.Const"() <{value = dense<"restore_ariables"> : tensor}> : () -> tensor + %cst_0 = "tf.Const"() <{value = dense<["", ""]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + %cst_1 = "tf.Const"() <{value = dense<["y", "z"]> : tensor<2x!tf_type.string>}> : () -> tensor<2x!tf_type.string> + %0:2 = "tf.RestoreV2"(%cst, %cst_1, %cst_0): (tensor, tensor<2x!tf_type.string>, tensor<2x!tf_type.string>) -> (tensor, tensor<1x3xf32>) + %1 = "tf.VariableV2"() <{container = "x", shape = #tf_type.shape<>, shared_name = "y"}> : () -> tensor + %dummy = "tf.Assign"(%1, %0#0) : (tensor, tensor) -> tensor + %2 = "tf.VarHandleOp"() <{container = "x", shared_name = "z"}> : () -> tensor>> + "tf.AssignVariableOp"(%2, %0#1) : (tensor>>, tensor<1x3xf32>) -> () + return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_empty.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_empty.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4c7b67fc9f8bcfe2ba3745feba3473fa70089b62 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_empty.mlir @@ -0,0 +1,5 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main() -> () attributes {__tpu_compile_metadata_text = "num_replicas: 1 num_cores_per_replica: 1"} { + func.return + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_gpu.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_gpu.mlir new file mode 100644 index 0000000000000000000000000000000000000000..cf68e1481b2b3638eac2efcc7e319c3004b83228 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_gpu.mlir @@ -0,0 +1,25 @@ +// MLIR which is legalize only with the right device type. +// The mlir is generated by running +// tensorflow/compiler/mlir/tf-opt -tf-xla-call-module-serialization +// +// module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, +// producer = 268 : i32}} { +// func.func private @_jit_sin(%arg0: tensor) -> tensor { +// %0 = stablehlo.sine %arg0 : tensor +// return %0 : tensor +// } +// func.func @main(%arg0: tensor) -> tensor<*xf32> { +// %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = +// "", dim_args_spec = [], _entry_function = @_jit_sin, module = "", +// platforms = ["CUDA"], version = 6 : i64} : (tensor) -> +// tensor<*xf32> +// func.return %0 : tensor<*xf32> +// } +// } +// +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor) -> tensor<*xf32> attributes {__tpu_compile_metadata_text = "args { dtype: DT_FLOAT kind: PARAMETER } retvals { } num_replicas: 1 num_cores_per_replica: 1"} { + %0 = "tf.XlaCallModule"(%arg0) {Sout = [#tf_type.shape<*>], device = "", dim_args_spec = [], module = "ML\EFR\03MLIRxxx-trunk\00\01\17\05\01\05\01\03\05\03\07\07\t\0B\03K5\07\01\1B\07\0B\13\0B3\0B\0B\0B\0B\0F\0B\13\0B\03\1B\0F\1B\0B\0B\0B\0B\0B\0F\13\0B\0B\0B\0B\03\07\0F\17\07\02\A7\1F\05\0D\03\03\03\07\05\0F\03\0B\0B\1B\0D'\0F)\031\113\05\11\05\13\05\15\05\17\1D\15\17\05\19\17\19\EF\01\05\1B\03\03\1D\0D\05\1F!#%\1D\1D\1D\1F\1D!\1D##\03\03\03+\0D\03-/\1D%\1D'\1D)\1D+)\01\05\11\03\01\03\01\t\04A\05\01\11\01\05\07\03\01\05\03\11\01\t\05\03\05\0B\03\01\01\05\06\13\03\01\03\01\07\04\01\03\03\06\03\01\05\01\00\9A\04-\0F\0B\03!\1B\1D\05\1B\83/\1F\15\1D\15\11\13\15\11\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00sine_v1\00return_v1\00sym_name\00jit_sin\00arg_attrs\00function_type\00res_attrs\00sym_visibility\00jit(sin)/jit(main)/sin\00third_party/py/jax/experimental/jax2tf/tests/back_compat_test.py\00jax.arg_info\00x\00mhlo.sharding\00{replicated}\00jax.result_info\00\00main\00public\00", platforms = ["CUDA"], version = 6 : i64} : (tensor) -> tensor<*xf32> + func.return %0 : tensor<*xf32> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_no_device_assignment.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_no_device_assignment.mlir new file mode 100644 index 0000000000000000000000000000000000000000..4c73e93e4175985d8aac6a2337d3078643c3c65d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_no_device_assignment.mlir @@ -0,0 +1,12 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { +func.func @main(%arg0: tensor<4x64xf32> {mhlo.sharding = "{devices=[2,1]0,1}"}, %arg1: tensor<64x10xf32> {mhlo.sharding = "{devices=[2,1]0,1}"}, %arg2: tensor<*xf32> {mhlo.sharding = ""}) -> (tensor {mhlo.sharding = ""}) attributes {tfrt_ifrt_serving.program_id = -5304490761103885474 : i64, __tpu_compile_metadata_text = "args { dtype: DT_FLOAT shape { unknown_rank: true } kind: PARAMETER sharding { type: OTHER tile_assignment_dimensions: 2 tile_assignment_dimensions: 1 tile_assignment_devices: 0 tile_assignment_devices: 1 } is_bounded_dynamic_dim: false } args { dtype: DT_FLOAT shape { unknown_rank: true } kind: PARAMETER sharding { type: OTHER tile_assignment_dimensions: 2 tile_assignment_dimensions: 1 tile_assignment_devices: 0 tile_assignment_devices: 1 } is_bounded_dynamic_dim: false } args { dtype: DT_FLOAT shape { unknown_rank: true } kind: PARAMETER is_bounded_dynamic_dim: false } retvals { sharding { } } num_replicas: 1 num_cores_per_replica: 2 use_spmd_for_xla_partitioning: true compile_options { }"} { + %cst = "tf.Const"() <{value = dense<[-1, 4]> : tensor<2xi32>}> {device = "/job:localhost/replica:0/task:0/device:CPU:0"} : () -> tensor<2xi32> + %0 = "tf.XlaSharding"(%arg0) <{_XlaSharding = "{devices=[2,1]0,1}", sharding = "{devices=[2,1]0,1}"}> : (tensor<4x64xf32>) -> tensor<4x64xf32> + %1 = "tf.XlaSharding"(%arg1) <{_XlaSharding = "{devices=[2,1]0,1}", sharding = "{devices=[2,1]0,1}"}> : (tensor<64x10xf32>) -> tensor<64x10xf32> + %2 = "tf.Reshape"(%arg2, %cst) : (tensor<*xf32>, tensor<2xi32>) -> tensor + %3 = "tf.MatMul"(%2, %0) <{transpose_a = false, transpose_b = false}> : (tensor, tensor<4x64xf32>) -> tensor + %4 = "tf.Relu"(%3) : (tensor) -> tensor + %5 = "tf.MatMul"(%4, %1) <{transpose_a = false, transpose_b = false}> : (tensor, tensor<64x10xf32>) -> tensor + return %5 : tensor + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_shardy.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_shardy.mlir new file mode 100644 index 0000000000000000000000000000000000000000..641b533efe379cb01831362cb9af47d75b1a8144 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_shardy.mlir @@ -0,0 +1,6 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { +func.func @main(%arg0: tensor<4x64xf32> {mhlo.sharding = "{devices=[2,1]<=[2]}"}) -> (tensor<4x64xf32> {mhlo.sharding = ""}) attributes {tfrt_ifrt_serving.program_id = -5304490761103885474 : i64, __tpu_compile_metadata_text = "args { dtype: DT_FLOAT shape { dim { size: 4 } dim { size: 64 } } kind: PARAMETER sharding { type: OTHER tile_assignment_dimensions: 2 tile_assignment_dimensions: 1 iota_reshape_dims: 2 iota_transpose_perm: 0 } is_bounded_dynamic_dim: false } retvals { sharding { } } num_replicas: 1 num_cores_per_replica: 2 device_assignment { replica_count: 1 computation_count: 2 computation_devices { replica_device_ids: 0 } computation_devices { replica_device_ids: 1 } } use_spmd_for_xla_partitioning: true use_shardy_partitioner: true compile_options { }"} { + %0 = "tf.Relu"(%arg0) : (tensor<4x64xf32>) -> tensor<4x64xf32> + return %0 : tensor<4x64xf32> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_with_device_assignment.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_with_device_assignment.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d2acf79698b797dc3e00407146c032f259265cd0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_spmd_with_device_assignment.mlir @@ -0,0 +1,6 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { +func.func @main(%arg0: tensor<4x64xf32> {mhlo.sharding = "{devices=[2,1]0,1}"}) -> (tensor<4x64xf32> {mhlo.sharding = ""}) attributes {tfrt_ifrt_serving.program_id = -5304490761103885474 : i64, __tpu_compile_metadata_text = "args { dtype: DT_FLOAT shape { dim { size: 4 } dim { size: 64 } } kind: PARAMETER sharding { type: OTHER tile_assignment_dimensions: 2 tile_assignment_dimensions: 1 tile_assignment_devices: 0 tile_assignment_devices: 1 } is_bounded_dynamic_dim: false } retvals { sharding { } } num_replicas: 1 num_cores_per_replica: 2 device_assignment { replica_count: 1 computation_count: 2 computation_devices { replica_device_ids: 0 } computation_devices { replica_device_ids: 1 } } use_spmd_for_xla_partitioning: true compile_options { }"} { + %0 = "tf.Relu"(%arg0) : (tensor<4x64xf32>) -> tensor<4x64xf32> + return %0 : tensor<4x64xf32> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_tuple.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_tuple.mlir new file mode 100644 index 0000000000000000000000000000000000000000..a045254780780d8f248ac62744ef54a7b5318b2a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/tf2hlo_tuple.mlir @@ -0,0 +1,6 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + func.func @main(%arg0: tensor<1x3xf32>, %arg1: tensor<3x1xf32>) -> (tensor<1x1xf32>, tensor<1x3xf32>) attributes {__tpu_compile_metadata_text = "args { dtype: DT_FLOAT kind: PARAMETER } args { dtype: DT_FLOAT kind: PARAMETER } retvals { } retvals { } num_replicas: 1 num_cores_per_replica: 1"} { + %0 = "tf.MatMul"(%arg0, %arg1): (tensor<1x3xf32>, tensor<3x1xf32>) -> tensor<1x1xf32> + func.return %0, %arg0: tensor<1x1xf32>, tensor<1x3xf32> + } +} \ No newline at end of file diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/xla_call_host_callback.mlir b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/xla_call_host_callback.mlir new file mode 100644 index 0000000000000000000000000000000000000000..d01dc5b57894c426d9f8b2f9e6d4666e3a0dccc8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata/xla_call_host_callback.mlir @@ -0,0 +1,23 @@ +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 1758 : i32}} { + + func.func private @callee(%arg0: tensor, %arg1: tensor<*xi32>) { + "tf.XlaHostCompute"(%arg0, %arg1) <{ancestors = [], key = "@test_callee", recv_key = "", send_key = "", shapes = []}> {_xla_original_oc_node_name = "hcb0", _xla_token_input_nodes = ["_xla_token_arg_node"]} : (tensor, tensor<*xi32>) -> () + return + } + + // The mlir module in XlaCallModule is serialized from: + // + // func.func private @_stablehlo_main_0(%arg0: tensor, %arg1: tensor<*xi32>) -> () attributes {_from_xla_call_module} { + // stablehlo.custom_call @tf.call_tf_function(%arg0, %arg1) {api_version = 2 : i32, has_side_effect = true, tf.backend_config = {called_func = @callee}} : (tensor, tensor<*xi32>) -> () + // return + // } + // + // func.func @main(%arg0: tensor, %arg1: tensor<*xi32>) -> () { + // "tf.XlaCallModule"(%arg0, %arg1) {Sout = [#tf_type.shape], dim_args_spec = [], _entry_function = @_stablehlo_main_0, _stablehlo_version = "0.17.6", _stablehlo_module_attrs = { mhlo.num_partitions = 1 }, module = "", platforms = [], version = 5 : i64} : (tensor, tensor<*xi32>) -> () + // func.return + // } + func.func @main(%arg0: tensor, %arg1: tensor<*xi32>) attributes {tfrt_ifrt_serving.program_id = -2372940092539171444 : i64, __tpu_compile_metadata_text = "args { dtype: DT_INT32 kind: PARAMETER sharding { } } args { dtype: DT_INT32 kind: PARAMETER sharding { } } num_replicas: 1 num_cores_per_replica: 1 use_spmd_for_xla_partitioning: true compile_options { }"} { + "tf.XlaCallModule"(%arg0, %arg1) <{Sout = [], dim_args_spec = [], function_list = [@callee], module = "ML\EFR\0DStableHLO_v0.17.6\00\01\19\05\01\05\09\01\03\0B\03\07\0F\13\17\03M-\0D\01\19\0B\13\0B\0F\13\13\13\13\13\0B\13\13\03\15\0B\0B\0B\0B\13\0B\0F\0B\0B\0B\01\03\0F\03\0B3\07\0B\17\07\02\B1\05\0D\03\03\05\07\05\0F\11\01\05\17\01A\0B\17\01!\07\17\01!Q\17\01!}\03\03\13!\05\11\17\01#\0B\17\01%\0B\03\01\1D\13#\09\1D\15\0D\03#%\1D\17\13\0B\01\0B\05\1D\19\05\03\01\02\04)\03\00\FF\FF\FF\FF\FF\FF\FF\FF\05\1B3\05\11\05\03\07\01\1D\04O\05\01Q\09\03\01\07\04=\03\01\05\03P\0B\03\07\04)\03\05\0B\05\07\0D\0F\0F\00\05E\15\11\05\05\01\03\07\00\17\06\03\01\05\01\00j\03\1B)\1B\0B\03%)\95\15\1F\11\0F\0B\11builtin\00vhlo\00module\00func_v1\00custom_call_v1\00return_v1\00experimental/users/deqiangc/mira/testdata/xla_call_module_serialized.mlir\00mhlo.num_partitions\00tf.backend_config\00\00main\00called_index\00tf.call_tf_function\00\08'\07\05\01\01\0B\19\1D\19\1F\1B\11'\1B)\19+\19\19\19", platforms = [], version = 5 : i64}> : (tensor, tensor<*xi32>) -> () + return + } +} diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.cc new file mode 100644 index 0000000000000000000000000000000000000000..753411c974877826be57a8309516263c5f7c9288 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.cc @@ -0,0 +1,323 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.h" + +#include +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v1/compile_mlir_util.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/legalize_tf.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_compilation.pb.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_constants.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h" +#include "tensorflow/compiler/tf2xla/layout_util.h" +#include "tensorflow/compiler/tf2xla/xla_compiler.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/client/client_library.h" +#include "xla/hlo/translate/hlo_to_mhlo/hlo_to_mlir_hlo.h" +#include "xla/pjrt/pjrt_compiler.h" +#include "xla/python/ifrt/client.h" +#include "xla/python/ifrt/layout.h" +#include "xla/service/computation_placer.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform_manager.h" +#include "xla/tsl/lib/strings/proto_serialization.h" +#include "xla/tsl/platform/errors.h" +#include "xla/tsl/platform/statusor.h" +#include "xla/xla_data.pb.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" +#include "tensorflow/core/protobuf/tpu/topology.pb.h" +#include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" +#include "tsl/platform/fingerprint.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { +static constexpr absl::string_view kEntryFuncName = "main"; +uint64_t MlirModuleFingerprint(mlir::ModuleOp module) { + std::string s; + llvm::raw_string_ostream os(s); + mlir::OpPrintingFlags flags; + flags.enableDebugInfo(false); + module.print(os, flags); + return tsl::Fingerprint64(os.str()); +} +} // namespace + +absl::StatusOr Tf2HloArg::Fingerprint() const { + uint64_t fingerprint = tsl::Fingerprint64(platform_name); + if (topology) { + TF_ASSIGN_OR_RETURN(fingerprint, topology->Fingerprint()); + } + if (platform_name != xla::CudaName() && !topology) { + return absl::FailedPreconditionError( + "Topology is required for non-GPU compilation."); + } + fingerprint = + tsl::FingerprintCat64(fingerprint, MlirModuleFingerprint(module)); + for (const auto& dtype_and_shape : input_dtypes_and_shapes) { + fingerprint = tsl::FingerprintCat64( + fingerprint, + tsl::Fingerprint64(tensorflow::DataType_Name(dtype_and_shape.dtype))); + + std::string serialized_shape; + if (!tsl::SerializeToStringDeterministic( + dtype_and_shape.GetShapeForCompilation().AsProto(), + &serialized_shape)) { + return absl::InternalError("Failed to serialize shape"); + } + + fingerprint = tsl::FingerprintCat64(fingerprint, + tsl::Fingerprint64(serialized_shape)); + } + fingerprint = tsl::FingerprintCat64(fingerprint, + tsl::Fingerprint64(entry_function_name)); + std::string serialized_compile_metadata; + if (!tsl::SerializeToStringDeterministic(compile_metadata, + &serialized_compile_metadata)) { + return absl::InternalError("Failed to serialize compile metadata"); + } + fingerprint = tsl::FingerprintCat64( + fingerprint, tsl::Fingerprint64(serialized_compile_metadata)); + fingerprint = tsl::FingerprintCat64( + fingerprint, + tsl::Fingerprint64(populate_layout_in_xla_input_shapes ? "1" : "0")); + return fingerprint; +} + +absl::StatusOr Tf2HloResult::ToProto() const { + Tf2HLOResultProto proto; + *proto.mutable_hlo_module_proto() = hlo_module_proto; + *proto.mutable_compile_metadata() = compile_metadata; + *proto.mutable_host_compute_metadata() = host_compute_metadata; + for (const auto& shape : xla_input_shapes) { + *proto.add_xla_input_shapes() = shape.ToProto(); + } + return proto; +} + +/*static*/ absl::StatusOr Tf2HloResult::FromProto( + const Tf2HLOResultProto& proto) { + Tf2HloResult result; + result.hlo_module_proto = proto.hlo_module_proto(); + result.compile_metadata = proto.compile_metadata(); + result.host_compute_metadata = proto.host_compute_metadata(); + for (const auto& shape : proto.xla_input_shapes()) { + TF_ASSIGN_OR_RETURN(xla::Shape xla_shape, xla::Shape::FromProto(shape)); + result.xla_input_shapes.push_back(std::move(xla_shape)); + } + return result; +} + +absl::Status UpdateCompileMetadata( + tensorflow::tpu::TPUCompileMetadataProto& metadata, + absl::Span inputs) { + VLOG(3) << "TpuCompileMetadata before shape is populated " << metadata; + if (metadata.num_replicas() < 1 || metadata.num_cores_per_replica() < 1) { + return absl::InternalError( + absl::StrCat("Number of replicas ", metadata.num_replicas(), + " and number of cores per replica ", + metadata.num_cores_per_replica(), " must be >= 1")); + } + if (metadata.args_size() != inputs.size()) { + return absl::InternalError( + absl::StrCat("Number of inputs mismatched! Expected ", + metadata.args_size(), " got ", inputs.size())); + } + + for (int i = 0; i < metadata.args_size(); ++i) { + if (metadata.args(i).kind() != + tensorflow::tpu::TPUCompileMetadataProto::Arg::PARAMETER) { + return absl::InternalError(absl::StrCat( + "Only support PARAMETER, but got ", metadata.args(i).kind())); + } + + if (metadata.args(i).dtype() != inputs[i].dtype) { + return absl::InternalError(absl::StrCat("Dtype mismatched! Expected ", + metadata.args(i).dtype(), " got ", + inputs[i].dtype)); + } + + // Update shape. + *metadata.mutable_args(i)->mutable_shape() = + inputs[i].GetShapeForCompilation().AsProto(); + } + return absl::OkStatus(); +} + +absl::StatusOr GetCompileMetadata( + mlir::ModuleOp module, const xla::ifrt::Client& ifrt_client) { + tensorflow::tpu::TPUCompileMetadataProto metadata; + + auto op = module.lookupSymbol(kEntryFuncName); + if (!op) { + return absl::InternalError("Could not find entry function in MLIR Module."); + } + + auto metadata_text_attr = + op->getAttrOfType(kMetadataTextAttrName); + + if (metadata_text_attr && !metadata_text_attr.getValue().empty()) { + // Try __tpu_compile_metadata_text attribute. This only for debugging + // purpose. + VLOG(1) << "Parsing from attribute " << kMetadataTextAttrName + << metadata_text_attr.getValue().str(); + if (!tsl::protobuf::TextFormat::ParseFromString( + metadata_text_attr.getValue().str(), &metadata)) { + return absl::InvalidArgumentError(absl::StrCat( + "Attribute ", kMetadataTextAttrName, ":", + metadata_text_attr.getValue().str(), " cannot be parsed")); + } + } else { + return absl::InvalidArgumentError( + absl::StrCat("Missing ", kMetadataTextAttrName)); + } + + // Create a default device assignment if one is not given by the model. + if (!metadata.has_device_assignment()) { + TF_ASSIGN_OR_RETURN( + auto device_assignment, + ifrt_client.GetDefaultDeviceAssignment( + metadata.num_replicas(), metadata.num_cores_per_replica())); + + xla::DeviceAssignmentProto device_assignment_proto; + device_assignment.Serialize(&device_assignment_proto); + + *metadata.mutable_device_assignment() = device_assignment_proto; + } + + return metadata; +} + +absl::StatusOr CompileTfToHlo(const Tf2HloArg& arg) { + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("ifrt_before_bridge_phase2", arg.module); + } + + // Device_type is a string of + // tensorflow/compiler/mlir/tf2xla/api/v2/device_type.proto:DeviceType + std::string device_type = "XLA_TPU_JIT"; + if (arg.platform_name == xla::CudaName()) { + device_type = "XLA_GPU_JIT"; + } + VLOG(1) << "device_type: " << device_type; + + TF_ASSIGN_OR_RETURN( + auto* platform, + stream_executor::PlatformManager::PlatformWithName("Host")); + TF_ASSIGN_OR_RETURN( + auto* client, xla::ClientLibrary::GetOrCreateCompileOnlyClient(platform)); + + + std::vector arg_shapes; + arg_shapes.reserve(arg.input_dtypes_and_shapes.size()); + for (const auto& input : arg.input_dtypes_and_shapes) { + arg_shapes.push_back(input.GetShapeForCompilation()); + } + + llvm::SmallVector + arg_tensor_or_resource_shapes; + for (const auto& shape : arg_shapes) { + arg_tensor_or_resource_shapes.push_back({shape}); + } + + TF_RETURN_IF_ERROR( + tensorflow::RefineShapes(arg_tensor_or_resource_shapes, arg.module)); + tpu::MlirToHloArgs mlir_to_hlo_args; + std::string module_str = tensorflow::SerializeMlirModule(arg.module); + mlir_to_hlo_args.mlir_module = module_str; + // Use fallback bridge as other modes may get deprecated. + mlir_to_hlo_args.rollout_state = + ConfigProto::Experimental::MLIR_BRIDGE_ROLLOUT_DISABLED; + + bool use_tuple_args = false; + std::vector arg_core_mapping; + std::vector> per_core_arg_shapes; + std::vector> custom_legalization_passes; + + TF_ASSIGN_OR_RETURN( + tensorflow::XlaCompiler::CompilationResult compilation_result, + tensorflow::tf2xla::v2::LegalizeMlirToHlo( + mlir_to_hlo_args, arg.compile_metadata, use_tuple_args, device_type, + custom_legalization_passes, + /*shape_determination_fns=*/ + tensorflow::XlaShapeLayoutHelpers::ShapeDeterminationFns( + tensorflow::UseNoPreferenceLayoutFn(), + arg.shape_representation_fn), + arg_shapes, &arg_core_mapping, &per_core_arg_shapes, client)); + + for (auto arg_shapes_iter = per_core_arg_shapes.begin() + 1; + arg_shapes_iter != per_core_arg_shapes.end(); ++arg_shapes_iter) { + if (per_core_arg_shapes.front() != *arg_shapes_iter) { + return absl::UnimplementedError( + "Only support even sharding SPMD, but get " + "different shapes across cores"); + } + } + + Tf2HloResult result; + if (arg.populate_layout_in_xla_input_shapes) { + result.xla_input_shapes = std::move(compilation_result.xla_input_shapes); + } + + result.hlo_module_proto = std::move(compilation_result.computation->proto()); + result.compile_metadata = arg.compile_metadata; + result.host_compute_metadata = + std::move(compilation_result.host_compute_metadata); + + return result; +} + +absl::StatusOr TfToHloCompiler::CompileTfToHlo(Tf2HloArg& arg) { + return tensorflow::ifrt_serving::CompileTfToHlo(arg); +} + +absl::StatusOr TfToHloCompiler::Key(const Tf2HloArg& arg) { + TF_ASSIGN_OR_RETURN(uint64_t fingerprint, arg.Fingerprint()); + return absl::StrCat(absl::Hex(fingerprint)); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.h new file mode 100644 index 0000000000000000000000000000000000000000..5f5b50cfbc0fce2ba6121c48cb936ff435663f21 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.h @@ -0,0 +1,98 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF2HLO_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF2HLO_H_ + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_compilation.pb.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/python/ifrt/client.h" +#include "xla/python/ifrt/layout.h" +#include "xla/python/ifrt/topology.h" +#include "xla/service/hlo.pb.h" +#include "xla/shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/protobuf/tpu/compile_metadata.pb.h" + +namespace tensorflow { +namespace ifrt_serving { + +struct Tf2HloArg { + mlir::ModuleOp module; + // `input_dtypes_and_shapes` can be mutable during Tf2HLO compilation. + std::vector input_dtypes_and_shapes; + absl::Span variable_arg_indices; + absl::string_view entry_function_name; + // `compile_metadata` can be mutable during Tf2HLO compilation. + tensorflow::tpu::TPUCompileMetadataProto compile_metadata; + tensorflow::XlaHelpers::ShapeRepresentationFn shape_representation_fn; + std::shared_ptr topology; + absl::string_view platform_name; + bool populate_layout_in_xla_input_shapes = false; + + absl::StatusOr Fingerprint() const; +}; + +struct Tf2HloResult { + xla::HloModuleProto hlo_module_proto; + tensorflow::tpu::TPUCompileMetadataProto compile_metadata; + tf2xla::HostComputeMetadata host_compute_metadata; + // `xla_input_shapes[i]` corresponds to the input shape of the i-th argument + // in the original `Tf2HloArg`. It will be empty if + // `populate_layout_in_xla_input_shapes` is false or there is no input in + // `module`. + std::vector xla_input_shapes; + + absl::StatusOr ToProto() const; + + static absl::StatusOr FromProto(const Tf2HLOResultProto& proto); +}; + +absl::Status UpdateCompileMetadata( + tensorflow::tpu::TPUCompileMetadataProto& metadata, + absl::Span inputs); + +absl::StatusOr GetCompileMetadata( + mlir::ModuleOp module, const xla::ifrt::Client& ifrt_client); + +class TfToHloCompiler { + public: + TfToHloCompiler() = default; + virtual ~TfToHloCompiler() = default; + + // Returns a cache key that can be used to identify the result of + // CompileTfToHlo. + virtual absl::StatusOr Key(const Tf2HloArg& arg); + + virtual absl::StatusOr CompileTfToHlo(Tf2HloArg& arg); + + virtual bool IsXlaCompilationDisabled() const { return false; } +}; + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF2HLO_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..54d87227eac96112f8b4bda62fe5feacf91d810d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo_test.cc @@ -0,0 +1,766 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf2hlo.h" + +#include +#include +#include +#include + +#include +#include +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mlir/IR/AsmState.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/dialect_registration.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/ifrt_types.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "xla/backends/cpu/target_machine_options.h" +#include "xla/debug_options_flags.h" +#include "xla/layout.h" +#include "xla/pjrt/pjrt_compiler.h" +#include "xla/pjrt/plugin/xla_cpu/cpu_topology_description.h" +#include "xla/python/ifrt/client.h" +#include "xla/python/ifrt/mock.h" +#include "xla/python/ifrt/test_util.h" +#include "xla/python/pjrt_ifrt/pjrt_topology.h" +#include "xla/service/computation_placer.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/platform/resource_loader.h" +#include "tensorflow/core/platform/test.h" +#include "tsl/platform/protobuf.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Ne; + +// TODO(b/229726259): Make EqualsProto available in OSS +class ProtoStringMatcher { + public: + explicit ProtoStringMatcher(const tsl::protobuf::Message& expected) + : expected_(expected.SerializeAsString()) {} + + template + bool MatchAndExplain(const Message& p, + ::testing::MatchResultListener*) const { + return p.SerializeAsString() == expected_; + } + + void DescribeTo(::std::ostream* os) const { *os << expected_; } + void DescribeNegationTo(::std::ostream* os) const { + *os << "not equal to expected message: " << expected_; + } + + private: + const std::string expected_; +}; + +inline ::testing::PolymorphicMatcher EqualsProto( + const tsl::protobuf::Message& x) { + return ::testing::MakePolymorphicMatcher(ProtoStringMatcher(x)); +} + +class Tf2HloTest : public ::testing::Test { + protected: + void SetUp() override { + mlir::DialectRegistry registry; + mlir::registerAllDialects(registry); + mlir::RegisterAllTensorFlowDialects(registry); + context_ = std::make_unique(registry); + } + TfToHloCompiler tf_to_hlo_compiler_; + std::unique_ptr context_; +}; + +TEST_F(Tf2HloTest, Empty) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/tf2hlo_empty.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, context_.get()); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, {})); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = {}, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .platform_name = xla::CpuName(), + }; + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); + + TF_ASSERT_OK(result.status()); +} + +// Multiple input and multiple out. +TEST_F(Tf2HloTest, Tuple) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/tf2hlo_tuple.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, context_.get()); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {1, 3}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {3, 1}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .platform_name = xla::CpuName(), + }; + + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); + + TF_ASSERT_OK(result.status()); +} + +struct SpmdTestCase { + std::string test_name; + std::string mlir_file; + std::string expected_metadata; +}; + +// Spmd and device assignment is given +class Tf2HloSpmdTest : public Tf2HloTest, + public ::testing::WithParamInterface {}; + +TEST_P(Tf2HloSpmdTest, SpmdTest) { + const SpmdTestCase& test_case = GetParam(); + + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/", test_case.mlir_file)); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, context_.get()); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {4, 64}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .platform_name = xla::CpuName(), + }; + + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); + + LOG(INFO) << result->compile_metadata; + TF_ASSERT_OK(result.status()); + + tensorflow::tpu::TPUCompileMetadataProto expected_compile_metadata; + ASSERT_TRUE(tsl::protobuf::TextFormat::ParseFromString( + test_case.expected_metadata, &expected_compile_metadata)); + + EXPECT_THAT(result->compile_metadata, EqualsProto(expected_compile_metadata)); +} +INSTANTIATE_TEST_SUITE_P( + SpmdTests, Tf2HloSpmdTest, + ::testing::ValuesIn({ + {"SpmdWithDeviceAssignment", "tf2hlo_spmd_with_device_assignment.mlir", + R"pb( + args { + dtype: DT_FLOAT + shape { + dim { size: 4 } + dim { size: 64 } + } + kind: PARAMETER + sharding { + type: OTHER + tile_assignment_dimensions: 2 + tile_assignment_dimensions: 1 + tile_assignment_devices: 0 + tile_assignment_devices: 1 + } + is_bounded_dynamic_dim: false + } + retvals { sharding {} } + num_replicas: 1 + num_cores_per_replica: 2 + device_assignment { + replica_count: 1 + computation_count: 2 + computation_devices { replica_device_ids: 0 } + computation_devices { replica_device_ids: 1 } + } + use_spmd_for_xla_partitioning: true + compile_options {} + )pb"}, + {"SpmdShardy", "tf2hlo_spmd_shardy.mlir", + R"pb( + args { + dtype: DT_FLOAT + shape { + dim { size: 4 } + dim { size: 64 } + } + kind: PARAMETER + sharding { + type: OTHER + tile_assignment_dimensions: 2 + tile_assignment_dimensions: 1 + iota_reshape_dims: 2 + iota_transpose_perm: 0 + } + is_bounded_dynamic_dim: false + } + retvals { sharding {} } + num_replicas: 1 + num_cores_per_replica: 2 + device_assignment { + replica_count: 1 + computation_count: 2 + computation_devices { replica_device_ids: 0 } + computation_devices { replica_device_ids: 1 } + } + use_spmd_for_xla_partitioning: true + use_shardy_partitioner: true + compile_options {} + )pb"}, + }), + [](const ::testing::TestParamInfo& info) { + return info.param.test_name; + }); + +// Spmd and use default device assignment b/c no device assignment is given +TEST_F(Tf2HloTest, UsingDefaultDeviceAssignment) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/tf2hlo_spmd_no_device_assignment.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, context_.get()); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {4, 64}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {64, 10}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {1, 4}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .platform_name = xla::CpuName(), + }; + + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); + + LOG(INFO) << result->compile_metadata; + TF_ASSERT_OK(result.status()); + + tensorflow::tpu::TPUCompileMetadataProto expected_compile_metadata; + ASSERT_TRUE(tsl::protobuf::TextFormat::ParseFromString( + R"pb( + args { + dtype: DT_FLOAT + shape { + dim { size: 4 } + dim { size: 64 } + } + kind: PARAMETER + sharding { + type: OTHER + tile_assignment_dimensions: 2 + tile_assignment_dimensions: 1 + tile_assignment_devices: 0 + tile_assignment_devices: 1 + } + is_bounded_dynamic_dim: false + } + args { + dtype: DT_FLOAT + shape { + dim { size: 64 } + dim { size: 10 } + } + kind: PARAMETER + sharding { + type: OTHER + tile_assignment_dimensions: 2 + tile_assignment_dimensions: 1 + tile_assignment_devices: 0 + tile_assignment_devices: 1 + } + is_bounded_dynamic_dim: false + } + args { + dtype: DT_FLOAT + shape { + dim { size: 1 } + dim { size: 4 } + } + kind: PARAMETER + is_bounded_dynamic_dim: false + } + retvals { sharding {} } + num_replicas: 1 + num_cores_per_replica: 2 + device_assignment { + replica_count: 1 + computation_count: 2 + computation_devices { replica_device_ids: 0 } + computation_devices { replica_device_ids: 1 } + } + use_spmd_for_xla_partitioning: true + compile_options {} + )pb", + &expected_compile_metadata)); + + EXPECT_THAT(result->compile_metadata, EqualsProto(expected_compile_metadata)); +} + +// Multiple input and multiple out. +TEST_F(Tf2HloTest, XlaCallHostCallback) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/xla_call_host_callback.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, + mlir::ParserConfig(context_.get())); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .platform_name = xla::CpuName(), + }; + + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); + + TF_ASSERT_OK(result.status()); + + ASSERT_EQ((*result).host_compute_metadata.device_to_host().size(), 1); + ASSERT_EQ( + (*result).host_compute_metadata.device_to_host().begin()->metadata_size(), + 2); + ASSERT_EQ((*result).host_compute_metadata.host_to_device().size(), 0); +} + +// On GPU enabled build, the compilation should pass. On a GPU disabled build, +// the compilation should fail with a correct error message. +TEST_F(Tf2HloTest, GpuCompile) { + // Create test input module + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/tf2hlo_gpu.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, context_.get()); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + xla::ifrt::MockClient mock_client; + ON_CALL(mock_client, GetDefaultDeviceAssignment) + .WillByDefault([]() -> absl::StatusOr { + return xla::DeviceAssignment(1, 1); + }); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_FLOAT, {}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), mock_client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + std::vector variable_arg_indices; + Tf2HloArg arg{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = nullptr, + .platform_name = xla::CudaName(), + }; + + auto result = tf_to_hlo_compiler_.CompileTfToHlo(arg); +#if defined(GOOGLE_CUDA) + LOG(INFO) << "GPU compile success"; + EXPECT_OK(result); +#else + LOG(INFO) << "Non-GPU compile failure"; + EXPECT_THAT(result, + absl_testing::StatusIs(absl::StatusCode::kUnimplemented, + HasSubstr("CUDA or ROCM build required"))); +#endif +} + +TEST_F(Tf2HloTest, SameArgProduceSameKeyFingerprint) { + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/xla_call_host_callback.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, + mlir::ParserConfig(context_.get())); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg0{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .populate_layout_in_xla_input_shapes = true, + }; + mlir::OwningOpRef mlir_module_clone = + mlir::OwningOpRef(mlir_module->clone()); + Tf2HloArg arg1{ + .module = mlir_module_clone.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .populate_layout_in_xla_input_shapes = true, + }; + + TfToHloCompiler tf_to_hlo_compiler; + TF_ASSERT_OK_AND_ASSIGN(std::string key0, tf_to_hlo_compiler.Key(arg0)); + TF_ASSERT_OK_AND_ASSIGN(std::string key1, tf_to_hlo_compiler.Key(arg1)); + + EXPECT_THAT(key0, Eq(key1)); +} + +TEST_F(Tf2HloTest, DifferentCompileMetadataProduceDifferentKeyFingerprint) { + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/xla_call_host_callback.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, + mlir::ParserConfig(context_.get())); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg0{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + }; + mlir::OwningOpRef mlir_module_clone = + mlir::OwningOpRef(mlir_module->clone()); + compile_metadata.set_num_replicas(11111); + Tf2HloArg arg1{ + .module = mlir_module_clone.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + }; + + TfToHloCompiler tf_to_hlo_compiler; + + TF_ASSERT_OK_AND_ASSIGN(std::string key0, tf_to_hlo_compiler.Key(arg0)); + TF_ASSERT_OK_AND_ASSIGN(std::string key1, tf_to_hlo_compiler.Key(arg1)); + EXPECT_THAT(key0, Ne(key1)); +} + +TEST_F(Tf2HloTest, + DifferentPopulateLayoutInXlaInputShapesProduceDifferentKeyFingerprint) { + constexpr absl::string_view kDataDirectory = + "tensorflow/compiler/mlir/tfrt/transforms/ifrt/testdata"; + std::string mlir_module_path = tensorflow::GetDataDependencyFilepath( + absl::StrCat(kDataDirectory, "/xla_call_host_callback.mlir")); + + mlir::OwningOpRef mlir_module = + mlir::parseSourceFile(mlir_module_path, + mlir::ParserConfig(context_.get())); + + ASSERT_TRUE(mlir_module); + ASSERT_TRUE(mlir_module.get() != nullptr); + + TF_ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + xla::ifrt::test_util::GetClient()); + + std::vector dtype_and_shapes; + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + dtype_and_shapes.push_back(DtypeAndShape{DT_INT32, {1}}); + + TF_ASSERT_OK_AND_ASSIGN( + tensorflow::tpu::TPUCompileMetadataProto compile_metadata, + GetCompileMetadata(mlir_module.get(), *client)); + TF_ASSERT_OK(UpdateCompileMetadata(compile_metadata, dtype_and_shapes)); + + const xla::CpuTopologyDescription cpu_topology( + xla::CpuId(), xla::CpuName(), /*platform_version=*/"", + xla::CpuTopology( + {}, xla::cpu::TargetMachineOptions(xla::GetDebugOptionsFromFlags()))); + std::shared_ptr cpu_topology_ptr = + std::make_shared(cpu_topology); + + std::vector variable_arg_indices; + Tf2HloArg arg0{ + .module = mlir_module.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .populate_layout_in_xla_input_shapes = true, + }; + mlir::OwningOpRef mlir_module_clone = + mlir::OwningOpRef(mlir_module->clone()); + Tf2HloArg arg1{ + .module = mlir_module_clone.get(), + .input_dtypes_and_shapes = dtype_and_shapes, + .variable_arg_indices = variable_arg_indices, + .entry_function_name = "main", + .compile_metadata = compile_metadata, + .shape_representation_fn = tensorflow::IdentityShapeRepresentationFn(), + .topology = std::make_shared(cpu_topology_ptr), + .populate_layout_in_xla_input_shapes = false, + }; + + TfToHloCompiler tf_to_hlo_compiler; + + TF_ASSERT_OK_AND_ASSIGN(std::string key0, tf_to_hlo_compiler.Key(arg0)); + TF_ASSERT_OK_AND_ASSIGN(std::string key1, tf_to_hlo_compiler.Key(arg1)); + EXPECT_THAT(key0, Ne(key1)); +} + +TEST_F(Tf2HloTest, ToProtoAndFromProto) { + Tf2HloResult result; + result.hlo_module_proto.set_name("test_module"); + result.compile_metadata.set_num_replicas(1); + + xla::Shape shape0 = xla::ShapeUtil::MakeShape(xla::F32, {10, 20}); + xla::Shape shape1 = xla::ShapeUtil::MakeShape(xla::F32, {30, 40}); + result.xla_input_shapes = {shape0, shape1}; + + TF_ASSERT_OK_AND_ASSIGN(auto proto, result.ToProto()); + TF_ASSERT_OK_AND_ASSIGN(auto result_from_proto, + Tf2HloResult::FromProto(proto)); + + EXPECT_EQ(result_from_proto.hlo_module_proto.name(), "test_module"); + EXPECT_EQ(result_from_proto.compile_metadata.num_replicas(), 1); + ASSERT_EQ(result_from_proto.xla_input_shapes.size(), 2); + EXPECT_EQ(result_from_proto.xla_input_shapes[0], shape0); + EXPECT_EQ(result_from_proto.xla_input_shapes[1], shape1); +} + +} // namespace +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_device_cleanup.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_device_cleanup.cc new file mode 100644 index 0000000000000000000000000000000000000000..1e26a39f1b1c501b0249c8f8b56d9a1381bbcdfe --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_device_cleanup.cc @@ -0,0 +1,51 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_TFDEVICECLEANUPPASS +#define GEN_PASS_DECL_TFDEVICECLEANUPPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +class TfDeviceCleanupPass + : public impl::TfDeviceCleanupPassBase { + public: + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + func.walk([](mlir::Operation* op) { + if (llvm::isa(op->getDialect())) { + op->removeAttr("device"); + } + }); + } +}; + +} // namespace + +std::unique_ptr> +CreateTfDeviceCleanupPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_identity_propagation.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_identity_propagation.cc new file mode 100644 index 0000000000000000000000000000000000000000..b97e8dd2ef96fe343d2f8b66b9d6ce768f48ffbf --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_identity_propagation.cc @@ -0,0 +1,88 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/strings/string_view.h" +#include "llvm/ADT/STLExtras.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_TFIDENTITYPROPAGATIONPASS +#define GEN_PASS_DECL_TFIDENTITYPROPAGATIONPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +constexpr absl::string_view kXlaShardingAttr = "_XlaSharding"; + +bool IsTerminator(mlir::Operation* op) { + return op->hasTrait(); +} + +class TfIdentityPropagationPass + : public impl::TfIdentityPropagationPassBase { + public: + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + + func.walk([](mlir::TF::IdentityOp identity) { + // Don't propagate inputs of identity ops with sharding annotation since + // identity ops are sometimes used to change output sharding. + if (identity->hasAttr(kXlaShardingAttr)) { + return; + } + // Identity outputs to terminator ops (e.g., `func.return`) cannot be + // replaced unless input/output types are exactly the same. Doing so may + // cause mismatch between the enclosing region's return type and the + // terminator's arg type. + const bool same_type = + identity.getInput().getType() == identity.getOutput().getType(); + identity.getOutput().replaceUsesWithIf( + identity.getInput(), [&](mlir::OpOperand& operand) { + return same_type || !IsTerminator(operand.getOwner()); + }); + }); + + func.walk([](mlir::TF::IdentityNOp identity_n) { + if (identity_n->hasAttr(kXlaShardingAttr)) { + return; + } + for (auto [input, output] : + llvm::zip(identity_n.getInput(), identity_n.getOutput())) { + const bool same_type = input.getType() == output.getType(); + output.replaceUsesWithIf(input, [&](mlir::OpOperand& operand) { + return same_type || !IsTerminator(operand.getOwner()); + }); + } + }); + } +}; + +} // namespace + +std::unique_ptr> +CreateTfIdentityPropagationPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.cc new file mode 100644 index 0000000000000000000000000000000000000000..65b6ae37f8620e08f6b6554fccb67085a8d78bb3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.cc @@ -0,0 +1,167 @@ + +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h" + +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/LogicalResult.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +using mlir::LogicalResult; +using mlir::OpPassManager; +using mlir::PassManager; +using mlir::func::FuncOp; + +void AddClusterToIfrtRuntimeOpsPassPipeline( + OpPassManager& pm, llvm::StringRef module_name, + bool enable_propagate_static_shapes_pass, bool enable_async_ifrt) { + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + + pm.addNestedPass( + mlir::TF::CreateCanonicalizeCompileAndReplicateAttributesPass()); + + pm.addNestedPass(CreateTfIdentityPropagationPass()); + + pm.addNestedPass(CreateTfRestoreSplittingPass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addNestedPass(CreateTfRestorePruningPass()); + pm.addNestedPass(CreateTfRestoreMergingPass()); + // Convert reference variable to resource variable since + // LowerToIfrtRestoreVariablePass does not support reference variable. + pm.addPass(CreateConvertReferenceVariableToResourceVariablePass()); + pm.addPass(CreateLowerToIfrtRestoreVariablePass()); + + pm.addPass(CreateRewriteClusterToIfrtCallPass(enable_async_ifrt)); + + if (enable_propagate_static_shapes_pass) { + pm.addPass(CreatePropagateStaticShapesPass()); + } + + // After device program is extracted, we can clean up device attributes from + // all ops. + pm.addNestedPass(CreateTfDeviceCleanupPass()); + + // Sink VarHandle with ReadVariableOp: subsequent SinkVariableAsNamedArrayPass + // rely on the co-existence of VarHandle and ReadVariable in the same + // function. + // First, we inline all the function calls. This will sink VarHandle + // with ReadVariable in most cases. Then SinkInvariantOpsPass will sink + // VarHandle to a few special Ops that inliner does not handle. + // TODO(b/319045348): the bridge before this pipeline already does some + // inlining. Consider removing this inliner. + pm.addPass(mlir::createInlinerPass()); + // Convert the region control flow to functional control flow so that the + // invariant sinking pass can work correctly. + pm.addPass(mlir::TF::CreateTFRegionControlFlowToFunctional()); + pm.addPass(::tensorflow::CreateSinkInInvariantOpsPass()); + pm.addPass(mlir::TF::CreateTFFunctionalControlFlowToRegions()); + pm.addPass(mlir::createInlinerPass()); + + // Decompose resource ops as resource variables are loaded by ReadVariableOp + // and can be lowered to IfrtLoadVariableOp in the subsequent + // SinkVariableAsNamedArrayPass. + pm.addNestedPass( + mlir::TFDevice::CreateDecomposeResourceOpsPass()); + + // Sink variable tensor as named array in IFRT. + pm.addPass(CreateSinkVariableAsNamedArrayPass()); +} + +} // namespace + +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(PassManager& pm, const std::string& dump_group_name, + llvm::StringRef module_name) { + // Print the whole module after each pass, which requires disabling + // multi-threading as well. + pm.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique<::tensorflow::DataDumperLoggerConfig>( + [module_name, dump_group_name](const std::string& pass_tag_name, + mlir::Operation* op) { + return DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), dump_group_name, pass_tag_name); + }, + /*pass_prefix=*/"", + /*print_module_scope=*/true)); + pm.enableTiming(); +} + +absl::Status RunClusterToIfrtRuntimeOpsPassPipeline( + mlir::ModuleOp module, llvm::StringRef module_name, + bool enable_propagate_static_shapes_pass, bool enable_async_ifrt) { + mlir::StatusScopedDiagnosticHandler diag_handler( + module.getContext(), /*propagate=*/false, + /*filter_stack=*/!VLOG_IS_ON(1)); + + PassManager runtime_lowering(module.getContext()); + ::tensorflow::applyTensorflowAndCLOptions(runtime_lowering); + + AddClusterToIfrtRuntimeOpsPassPipeline(runtime_lowering, module_name, + enable_propagate_static_shapes_pass, + enable_async_ifrt); + + if (VLOG_IS_ON(1)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + "ifrt_runtime_lowering_before"), + module, llvm::StringRef(), &runtime_lowering); + } + + if (VLOG_IS_ON(2)) { + EnablePassIRPrinting(runtime_lowering, kDebugGroupRuntimeLowering, + module_name); + } + + // Ignore the result since diag_handler consumes it + LogicalResult result = runtime_lowering.run(module); + (void)result; + + if (VLOG_IS_ON(1)) { + ::tensorflow::DumpMlirOpToFile( + DEBUG_DATA_DUMPER()->GetDumpFilename(module_name.str(), kDebugGroupMain, + "ifrt_runtime_lowering_after"), + module, llvm::StringRef(), &runtime_lowering); + } + + return diag_handler.ConsumeStatus(); +} + +// Register all IfrtPass +void RegisterTfIfrtPasses() { registerTfrtIfrtServingPasses(); } + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..b254741e2a2e2140ef9521cfb4c6466f5d3d7f91 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_ifrt_passes.h @@ -0,0 +1,94 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF_IFRT_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF_IFRT_PASSES_H_ + +#include +#include + +#include "absl/status/status.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project + +namespace tensorflow { +namespace ifrt_serving { + +// Create a pass to convert tf_device.cluster_func to tf.ifrt_program_call. +std::unique_ptr> +CreateRewriteClusterToIfrtCallPass(bool enable_async_ifrt = false); + +// Creates a pass that propagates static shapes from +// `tf.SetStaticDimensionBoundsOp` to `IfrtCallOp` and callee `FuncOp` +// arguments. +std::unique_ptr> +CreatePropagateStaticShapesPass(); + +// Creates a pass that sinks variable tensor argument to `tf.IfrtCall` as named +// arrays and lowers `tf.ReadVariableOp` to `tf.IfrtLoadVariableOp`. +std::unique_ptr> +CreateSinkVariableAsNamedArrayPass(); + +// Creates a pass that splits `tf.RestoreV2` ops. +std::unique_ptr> +CreateTfRestoreSplittingPass(); + +// Creates a pass that merges `tf.RestoreV2` ops. +std::unique_ptr> +CreateTfRestoreMergingPass(); + +// Creates a pass that propagates inputs of no-op identity ops to their outputs. +std::unique_ptr> +CreateTfIdentityPropagationPass(); + +// Creates a pass that prunes unused `tf.RestoreV2` ops. +std::unique_ptr> +CreateTfRestorePruningPass(); + +// Creates a pass that lower `tf.RestoreVariableOp` to +// `tf.IfrtRestoreVariableOp`. +std::unique_ptr> +CreateLowerToIfrtRestoreVariablePass(); + +// Creates a pass that cleans up device attributes from all ops. +std::unique_ptr> +CreateTfDeviceCleanupPass(); + +#define GEN_PASS_REGISTRATION +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +// Register all passes. +void RegisterTfIfrtPasses(); + +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(mlir::PassManager& pm, + const std::string& dump_group_name, + llvm::StringRef module_name); + +// Convert tf_device.cluster_func to tf.ifrt_program_call. +// The callee function is converted to a ifrt_program. +absl::Status RunClusterToIfrtRuntimeOpsPassPipeline( + mlir::ModuleOp module, llvm::StringRef module_name = llvm::StringRef(), + bool enable_propagate_static_shapes_pass = true, + bool enable_async_ifrt = false); + +} // namespace ifrt_serving +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_IFRT_TF_IFRT_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_merging.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_merging.cc new file mode 100644 index 0000000000000000000000000000000000000000..a4050c28c053bd8247e455544aa3a2397b6e1cdb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_merging.cc @@ -0,0 +1,164 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_TFRESTOREMERGINGPASS +#define GEN_PASS_DECL_TFRESTOREMERGINGPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +class TfRestoreMergingPass + : public impl::TfRestoreMergingPassBase { + public: + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + for (mlir::Block& block : func) { + // Group `tf.RestoreV2` ops by prefixes and merge each group. + llvm::SmallDenseMap> + restore_groups; + for (auto restore : block.getOps()) { + restore_groups[restore.getPrefix()].push_back(restore); + } + for (const auto& restores : llvm::make_second_range(restore_groups)) { + if (mlir::failed(MergeRestores(restores))) { + return signalPassFailure(); + } + } + } + } + + private: + mlir::DenseStringElementsAttr GetStringTensorAttr( + llvm::ArrayRef values) { + const int size = values.size(); + const auto type = mlir::RankedTensorType::get( + {size}, mlir::TF::StringType::get(&getContext())); + return mlir::DenseStringElementsAttr::get(type, values); + } + + // Merges `tf.RestoreV2` ops with the same prefix. Ignores restore ops with + // non-constant `tensor_names` and/or `shape_and_slices`. + mlir::LogicalResult MergeRestores( + llvm::ArrayRef restores) { + if (restores.size() <= 1) { + return mlir::success(); + } + + // All restore ops must have the same prefix. + const mlir::Value prefix = + mlir::TF::RestoreV2Op(restores.front()).getPrefix(); + + std::vector restores_to_merge; + std::vector values_to_replace; + std::vector merged_tensor_names; + std::vector merged_shape_and_slices; + + std::vector restore_locs; + std::vector tensor_names_locs; + std::vector shape_and_slices_locs; + + for (mlir::TF::RestoreV2Op restore : restores) { + mlir::DenseStringElementsAttr tensor_names; + mlir::DenseStringElementsAttr shape_and_slices; + if (!mlir::matchPattern(restore, + mlir::m_Op( + mlir::matchers::m_Val(prefix), + mlir::m_Constant(&tensor_names), + mlir::m_Constant(&shape_and_slices)))) { + continue; + } + if (tensor_names.size() != restore.getNumResults() || + shape_and_slices.size() != restore.getNumResults()) { + return restore.emitOpError() + << "returns an inconsistent number of results"; + } + + restores_to_merge.push_back(restore); + llvm::append_range(values_to_replace, restore.getTensors()); + llvm::append_range(merged_tensor_names, + tensor_names.getValues()); + llvm::append_range(merged_shape_and_slices, + shape_and_slices.getValues()); + + restore_locs.push_back(restore.getLoc()); + tensor_names_locs.push_back(restore.getTensorNames().getLoc()); + shape_and_slices_locs.push_back(restore.getShapeAndSlices().getLoc()); + } + if (restores_to_merge.size() <= 1) { + return mlir::success(); + } + + // Insert the merged restore op right before the first restore op to be + // merged in order to keep the dominance property. + mlir::OpBuilder builder(restores_to_merge.front()); + + auto new_tensor_names = mlir::TF::ConstOp::create( + builder, builder.getFusedLoc(tensor_names_locs), + GetStringTensorAttr(merged_tensor_names)); + auto new_shape_and_slices = mlir::TF::ConstOp::create( + builder, builder.getFusedLoc(shape_and_slices_locs), + GetStringTensorAttr(merged_shape_and_slices)); + + auto new_restore = mlir::TF::RestoreV2Op::create( + builder, builder.getFusedLoc(restore_locs), + mlir::TypeRange(mlir::ValueRange(values_to_replace)), prefix, + new_tensor_names, new_shape_and_slices); + for (auto [old_value, new_value] : + llvm::zip(values_to_replace, new_restore.getTensors())) { + old_value.replaceAllUsesWith(new_value); + } + + for (mlir::TF::RestoreV2Op restore : restores_to_merge) { + restore.erase(); + } + return mlir::success(); + } +}; + +} // namespace + +std::unique_ptr> +CreateTfRestoreMergingPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_pruning.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_pruning.cc new file mode 100644 index 0000000000000000000000000000000000000000..a5e548f356dfc456edb00fcadbc78c19f317e019 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_pruning.cc @@ -0,0 +1,52 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_TFRESTOREPRUNINGPASS +#define GEN_PASS_DECL_TFRESTOREPRUNINGPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +// Prune unused RestoreV2 Op. +class TfRestorePruningPass + : public impl::TfRestorePruningPassBase { + public: + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + func.walk([&](mlir::TF::RestoreV2Op restore) { + if (restore.use_empty()) { + restore.erase(); + } + }); + } +}; + +} // namespace + +std::unique_ptr> +CreateTfRestorePruningPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_splitting.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_splitting.cc new file mode 100644 index 0000000000000000000000000000000000000000..dd5aba8c0ecd39478f34969a6311443452fbb5d5 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/ifrt/tf_restore_splitting.cc @@ -0,0 +1,122 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Diagnostics.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" + +namespace tensorflow { +namespace ifrt_serving { +namespace { + +#define GEN_PASS_DEF_TFRESTORESPLITTINGPASS +#define GEN_PASS_DECL_TFRESTORESPLITTINGPASS +#include "tensorflow/compiler/mlir/tfrt/transforms/ifrt/passes.h.inc" // IWYU pragma: keep + +class TfRestoreSplittingPass + : public impl::TfRestoreSplittingPassBase { + public: + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + const mlir::WalkResult result = + func.walk([&](mlir::TF::RestoreV2Op restore) { + if (mlir::failed(SplitRestore(restore))) { + return mlir::WalkResult::interrupt(); + } + return mlir::WalkResult::advance(); + }); + if (result.wasInterrupted()) { + return signalPassFailure(); + } + } + + private: + mlir::DenseStringElementsAttr GetStringTensorAttr( + llvm::ArrayRef values) { + const int size = values.size(); + const auto type = mlir::RankedTensorType::get( + {size}, mlir::TF::StringType::get(&getContext())); + return mlir::DenseStringElementsAttr::get(type, values); + } + + // Splits the `tf.RestoreV2` op into per-variable restore ops if its + // `tensor_name` and `shape_and_slices` are constant. + mlir::LogicalResult SplitRestore(mlir::TF::RestoreV2Op restore) { + mlir::DenseStringElementsAttr tensor_names; + mlir::DenseStringElementsAttr shape_and_slices; + if (!mlir::matchPattern(restore, + mlir::m_Op( + /*prefix=*/mlir::matchers::m_Any(), + mlir::m_Constant(&tensor_names), + mlir::m_Constant(&shape_and_slices)))) { + return mlir::success(); + } + if (tensor_names.size() != restore.getNumResults() || + shape_and_slices.size() != restore.getNumResults()) { + return restore.emitOpError() + << "returns an inconsistent number of results"; + } + + mlir::OpBuilder builder(restore); + for (auto [tensor_name, shape_and_slice, result] : + llvm::zip(tensor_names.getValues(), + shape_and_slices.getValues(), + restore.getTensors())) { + auto new_tensor_names = + mlir::TF::ConstOp::create(builder, restore.getTensorNames().getLoc(), + GetStringTensorAttr({tensor_name})); + + auto new_shape_and_slices = mlir::TF::ConstOp::create( + builder, restore.getShapeAndSlices().getLoc(), + GetStringTensorAttr({shape_and_slice})); + + auto new_restore = mlir::TF::RestoreV2Op::create( + builder, restore.getLoc(), mlir::TypeRange({result.getType()}), + restore.getPrefix(), new_tensor_names, new_shape_and_slices); + result.replaceAllUsesWith(new_restore.getTensors()[0]); + } + + restore.erase(); + return mlir::success(); + } +}; + +} // namespace + +std::unique_ptr> +CreateTfRestoreSplittingPass() { + return std::make_unique(); +} + +} // namespace ifrt_serving +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/insert_tensor_copy.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/insert_tensor_copy.cc new file mode 100644 index 0000000000000000000000000000000000000000..04bb968d8a7cea66aad7b07a76499bd1676d5348 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/insert_tensor_copy.cc @@ -0,0 +1,187 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tfrt/basic_kernels/opdefs/basic_kernels.h" // from @tf_runtime +#include "tfrt/compiler/stream_analysis.h" // from @tf_runtime + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +// This pass inserts copy kernels for fallback tensors when they are passed to +// multiple threads, to avoid atomic contention on their refcounts. +class InsertFallbackTensorCopy + : public mlir::PassWrapper> { + void getDependentDialects(mlir::DialectRegistry& registry) const override { + registry.insert(); + } + + llvm::StringRef getArgument() const final { + return "tfrt-insert-fallback-tensor-copy"; + } + + llvm::StringRef getDescription() const final { + return "Inserts copy kernels for fallback tensors when they are passed to " + "multiple threads, to avoid atomic contention on refcounts."; + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(InsertFallbackTensorCopy) + + void runOnOperation() override { + mlir::func::FuncOp func_op = getOperation(); + + // Use stream analysis to know whether a value is passed to different + // threads. + tfrt::compiler::StreamAnalysis stream_analysis(func_op); + + auto builder = mlir::OpBuilder::atBlockBegin(&func_op.front()); + + // Process function arguments first. + for (auto arg : func_op.getArguments()) { + if (!mlir::isa(arg.getType())) continue; + InsertFallbackTensorCopyForValue(arg, func_op->getLoc(), builder, + stream_analysis); + } + + // Then process each operations in the block. + for (mlir::Operation& op : llvm::make_early_inc_range(func_op.front())) { + if (llvm::isa(&op)) { + InsertFallbackTensorCopyForFallbackOp(&op, builder, stream_analysis); + } + } + } + + private: + void InsertFallbackTensorCopyForFallbackOp( + mlir::Operation* op, mlir::OpBuilder& builder, + const tfrt::compiler::StreamAnalysis& stream_analysis) { + builder.setInsertionPointAfter(op); + + // Process each result value. + for (auto result : op->getResults()) { + if (!mlir::isa(result.getType())) continue; + InsertFallbackTensorCopyForValue(result, op->getLoc(), builder, + stream_analysis); + } + } + + // Insert copy kernels to copy the result, and allocate new atomic refcount + // if the value is going to be used by different streams/threads, in order to + // avoid contention on the atomic counter. + void InsertFallbackTensorCopyForValue( + mlir::Value value, mlir::Location loc, mlir::OpBuilder& builder, + const tfrt::compiler::StreamAnalysis& stream_analysis) { + llvm::DenseMap> stream_map; + + // Find out streams that use this value and the corresponding uses. + for (mlir::OpOperand& use : value.getUses()) { + // Skip return op as there should not be atomic contention on the return + // op. + if (llvm::isa(use.getOwner())) continue; + + int stream_id = stream_analysis.GetStream(use.getOwner()).id(); + stream_map[stream_id].push_back(&use); + } + + // Organize these uses into groups. If a stream has many uses of this value, + // put these uses into one stream. Otherwise, streams with small number + // of uses are grouped with each other to form groups with enough uses. + constexpr int kCopyGroupThreshold = 16; + llvm::SmallVector, 4> small_copies; + llvm::SmallVector, 4> copies; + for (const auto& iter : stream_map) { + if (iter.second.size() >= kCopyGroupThreshold) { + copies.push_back(iter.second); + } else { + if (small_copies.empty() || + small_copies.back().size() >= kCopyGroupThreshold) { + small_copies.push_back(iter.second); + } else { + small_copies.back().append(iter.second.begin(), iter.second.end()); + } + } + } + + if (!small_copies.empty()) + copies.append(small_copies.begin(), small_copies.end()); + + // If it is only used by one group, then we don't need to copy. + if (copies.size() <= 1) return; + + // Remove one group from the candidates, as we can just use the original + // value for this group. + copies.pop_back(); + + // For each stream, we will create one new value that replaces the uses in + // that stream. + + assert(mlir::isa(value.getType())); + + // The number of results is the number candidate streams. + llvm::SmallVector result_types(copies.size(), + value.getType()); + assert(!result_types.empty()); + + // Create the tfrt_fallback_async.copy_if_small kernel. + auto copy_op = tfrt::fallback_async::CopyIfSmallOp::create( + builder, loc, result_types, value); + + // Finally, replaces all uses with the new value. + for (int i = 0; i < copies.size(); ++i) { + const auto& uses = copies[i]; + auto new_value = copy_op.getResult(i); + for (auto* use : uses) { + use->set(new_value); + } + } + } +}; + +} // namespace + +std::unique_ptr> +CreateInsertFallbackTensorCopyPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateInsertFallbackTensorCopyPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/lower_saved_model.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/lower_saved_model.cc new file mode 100644 index 0000000000000000000000000000000000000000..d00615870c4028f0c3994c5240d725fca12944d4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/lower_saved_model.cc @@ -0,0 +1,772 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_interfaces.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" + +namespace tensorflow { +namespace { + +using ::mlir::tf_saved_model::kTfSavedModelExportedNamesAttr; +using ::mlir::tf_saved_model::kTfSavedModelIndexPathAttr; + +constexpr char kCpuDeviceName[] = + "/job:localhost/replica:0/task:0/device:CPU:0"; + +mlir::TF::ResourceHandle GetResourceHandle(mlir::Operation *op) { + llvm::StringRef device; + if (auto attr = op->getAttrOfType("device")) { + device = attr.getValue(); + } + + llvm::StringRef container; + if (auto attr = op->getAttrOfType("container")) { + container = attr.getValue(); + } + + llvm::StringRef shared_name; + if (auto attr = op->getAttrOfType("shared_name")) { + shared_name = attr.getValue(); + } + + return {container, shared_name, device, /*op=*/nullptr}; +} + +struct HoistInfo { + // All hoisted ops in the topological order. + llvm::SmallVector hoists_in_topological_order; + + // Mapping from the old values produced by hoisted ops before hoisting to the + // new values after hoisting. + mlir::IRMapping value_mapping; + + // `hoisted_values` is to keep all values that are produced by hoisted ops + // but used by non-hoisted ops. These values will be replaced by results of + // tf._TfrtGetResource op. The index of each value in this array will be the + // index used in tf._TfrtGetResource and tf._TfrtSetResource op. This also + // stores the ResourceHandle which has the shared_name and container + // attributes used by later resource alias analysis and side effect analysis + // passes. + llvm::SmallVector, 4> + hoisted_values; +}; + +bool OnlyHasReadOrNoEffect(mlir::Operation *op) { + auto interface = llvm::dyn_cast(op); + if (!interface) return false; + return interface.onlyHasEffect() || + interface.hasNoEffect(); +} + +bool CanHoist(const llvm::DenseSet &read_only_vars, + mlir::Operation *op) { + // return ops should not be hoisted. + if (op->mightHaveTrait()) return false; + + // Fixes a corner case where hoisting the tf.BatchFunction leads to + // a compilation error; such a case may occur in unit tests. + if (llvm::isa(op)) return false; + + // Non-side-effecting ops can be hoisted. + if (mlir::isMemoryEffectFree(op)) return true; + + // ResourceHandle ops can be hoisted. + if (llvm::isa(op)) + return true; + + // If it is ReadVariableOp and the variable is readonly, it can be hoisted. + if (auto read_var_op = llvm::dyn_cast(op)) { + if (auto var_handle_op = llvm::dyn_cast_or_null( + read_var_op.getResource().getDefiningOp())) { + if (read_only_vars.count(GetResourceHandle(var_handle_op)) > 0) + return true; + } + } + + // If it is LookupTableSizeOp, it can be hoisted as the size of the hash table + // cannot be changed after initialization. + if (auto lookup_table_size_op = + llvm::dyn_cast(op)) { + if (auto hash_table_op = llvm::dyn_cast_or_null( + lookup_table_size_op.getTableHandle().getDefiningOp())) { + if (read_only_vars.count(GetResourceHandle(hash_table_op)) > 0) + return true; + } + } + + // TODO(chky): Allow more readonly ops. + + return false; +} + +void HoistInvariantOpsInFunction( + mlir::func::FuncOp func, + const llvm::DenseSet &read_only_vars, + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis, + mlir::OpBuilder &builder, HoistInfo &module_hoist_info) { + // Keep the hoisted ops in this function. + llvm::DenseSet hoists; + + auto all_operands_in_hoists = [&module_hoist_info](mlir::Operation *op) { + for (mlir::Value operand : op->getOperands()) { + if (module_hoist_info.value_mapping.lookupOrNull(operand) == nullptr) + return false; + } + return true; + }; + + auto all_control_predeccessors_in_hoists = [&hoists, &side_effect_analysis]( + mlir::Operation *op) { + auto preds = side_effect_analysis.DirectControlPredecessors(op); + return std::all_of( + preds.begin(), preds.end(), + [&hoists](mlir::Operation *pred) { return hoists.count(pred) > 0; }); + }; + + std::deque work_list; + + // Start with ops with tf.VarHandleOp ops and tf.Const ops. + // + // TODO(chky): Consider allowing other ops including custom ops to be hoisted. + func.walk([&work_list](mlir::Operation *op) { + if (llvm::isa(op)) + work_list.push_back(op); + }); + + while (!work_list.empty()) { + auto *op = work_list.front(); + work_list.pop_front(); + + // Skip if it is already hoisted. + if (hoists.count(op) > 0) continue; + + // If the op can be hoisted, and all of its data dependencies and control + // dependencies are hoisted, then we hoist it. Otherwise, skip. + if (!(CanHoist(read_only_vars, op) && all_operands_in_hoists(op) && + all_control_predeccessors_in_hoists(op))) + continue; + + // Record the hoisted operation. + hoists.insert(op); + module_hoist_info.hoists_in_topological_order.push_back(op); + + // Create a copy in the init function. + builder.clone(*op, module_hoist_info.value_mapping); + + for (mlir::Operation *user : op->getUsers()) { + work_list.push_back(user); + } + } + + // Find out the values that are produced by hoisted ops but used by + // non-hoisted ops. These values need to be replaced. + for (auto *op : hoists) { + for (auto result : op->getResults()) { + if (std::any_of(result.getUsers().begin(), result.getUsers().end(), + [&hoists](mlir::Operation *user) { + return hoists.count(user) == 0; + })) { + module_hoist_info.hoisted_values.push_back( + {result, GetResourceHandle(op)}); + } + } + } +} + +void FindCalleesRecursiveForOp(const mlir::SymbolTable &symbol_table, + mlir::Operation *op, + llvm::StringSet<> &callees) { + for (const auto &named_attr : op->getAttrs()) { + if (auto symbol_attr = + mlir::dyn_cast(named_attr.getValue())) { + auto symbol = symbol_attr.getValue(); + if (!callees.contains(symbol)) { + callees.insert(symbol); + + auto func = symbol_table.lookup(symbol); + if (!func) continue; + + func.walk([&](mlir::Operation *op) { + FindCalleesRecursiveForOp(symbol_table, op, callees); + }); + } + } + } +} + +void FindCalleesRecursive(const mlir::SymbolTable &symbol_table, + mlir::func::FuncOp func, llvm::StringSet<> &callees) { + assert(func); + func.walk([&](mlir::Operation *op) { + FindCalleesRecursiveForOp(symbol_table, op, callees); + }); +} + +// This pass rewrites tf_saved_model dialect's ops according to TFRT's +// requirements: +// +// 1) Remove all tf_saved_model's attributes and ops. +// 2) Create a function for every exported names of the original function. +// 3) Hoist invariant ops (ie. guaranteed to return the same value on every +// invocation) for every non-init function. +// +class LowerTFSavedModelPass + : public mlir::PassWrapper> { + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTFSavedModelPass) + + explicit LowerTFSavedModelPass(bool hoist_invariant_ops, + bool fuse_get_resource_ops) { + hoist_invariant_ops_ = hoist_invariant_ops; + fuse_get_resource_ops_ = fuse_get_resource_ops; + } + LowerTFSavedModelPass() = default; + LowerTFSavedModelPass(const LowerTFSavedModelPass &) {} + + llvm::StringRef getArgument() const final { + return "tfrt-lower-tf-savedmodel"; + } + llvm::StringRef getDescription() const final { + return "Lower tf-saved-model ops according to TFRT's requirements."; + } + + void runOnOperation() override { + auto module = getOperation(); + + // TODO(b/185928201): Create a standalone pass for hoisting invariant ops so + // that it can be reusable and configurable in other contexts than saved + // models. + if (hoist_invariant_ops_) HoistInvariantOps(module); + + // Skip non-savedmodel MLIR module. + if (!mlir::tf_saved_model::HasTfSavedModelSemantics(module)) return; + + mlir::SymbolTable symbol_table(module); + + module->removeAttr("tf_saved_model.semantics"); + + mlir::OpBuilder builder(&getContext()); + auto resource_id = builder.getStringAttr("tf.resource_name"); + auto bound_id = builder.getStringAttr("tf_saved_model.bound_input"); + auto path_id = builder.getStringAttr(kTfSavedModelIndexPathAttr); + + module.walk([resource_id, bound_id, path_id, + &builder](mlir::Operation *op) mutable { + if (auto func_op = llvm::dyn_cast(op)) { + // Remove tf_saved_model specific function arg attributes. + for (unsigned i = 0, e = func_op.getNumArguments(); i != e; ++i) { + if (auto sym = func_op.getArgAttrOfType( + i, bound_id)) { + func_op.removeArgAttr(i, bound_id); + func_op.setArgAttr(i, resource_id, + builder.getStringAttr(sym.getValue())); + } + func_op.removeArgAttr(i, path_id); + } + for (unsigned i = 0, e = func_op.getNumResults(); i != e; ++i) { + func_op.removeResultAttr(i, bound_id); + func_op.removeResultAttr(i, path_id); + } + if (auto exported_names = func_op->getAttrOfType( + kTfSavedModelExportedNamesAttr)) { + bool is_session_initializer = IsSessionInitializer(func_op); + + // Create a function for each exported name. + // + // TODO(b/148477882): TFRT dialect should have similar concepts of + // exported names so that a function can be referenced by multiple + // exported names. + func_op->removeAttr(kTfSavedModelExportedNamesAttr); + for (auto exported_name : exported_names) { + auto exported_func_op = func_op.clone(); + exported_func_op.setName( + mlir::cast(exported_name)); + + // If it is a session initializer, we want to maximize parallelism + // and do not perform any stream merge, to minimize latency. + // + // TODO(b/183219530): This is a workaround as the cost model used + // currently is not very accurate, and leads to performance + // regression on IO ops that are common in initialization functions. + if (is_session_initializer) { + exported_func_op->setAttr("tfrt.cost_threshold", + builder.getI64IntegerAttr(1)); + } + + builder.setInsertionPoint(func_op); + builder.insert(exported_func_op); + } + func_op.erase(); + } + } + }); + + module.walk([](mlir::Operation *op) { + if (llvm::isa( + op->getDialect())) { + // Remove all tf_saved_model ops. + op->erase(); + } + }); + } + + private: + void HoistInvariantOps(mlir::ModuleOp module); + void ReplaceHoistedValues( + llvm::ArrayRef> + hoisted_values, + mlir::OpBuilder &builder); + + Option hoist_invariant_ops_{*this, "hoist-invariant-ops", + llvm::cl::desc("hoist-invariant-ops"), + llvm::cl::init(false)}; + Option fuse_get_resource_ops_{*this, "fuse-get-resource-ops", + llvm::cl::desc("fuse get resource ops"), + llvm::cl::init(true)}; +}; + +void LowerTFSavedModelPass::HoistInvariantOps(mlir::ModuleOp module) { + mlir::SymbolTable symbol_table(module); + + // Find all resources used in non-init functions. + llvm::DenseMap> + resources; + + // Find all callees referenced in the initialization functions. + llvm::StringSet<> init_callees; + + // Recursively find all callees referenced in the tf.XlaLaunch op. + // At and after the point of calling this pass, the MLIR xla function is no + // longer used. So there is no point to do hoisting for xla functions. + llvm::StringSet<> xla_launch_callees; + + module.walk([&](mlir::Operation *op) { + if (llvm::isa(op)) { + auto func = op->getParentOfType(); + if (IsSessionInitializer(func)) return; + resources[GetResourceHandle(op)].push_back(op); + } else if (auto func = llvm::dyn_cast(op)) { + if (!IsSessionInitializer(func)) return; + FindCalleesRecursive(symbol_table, func, init_callees); + } else if (llvm::isa(op)) { + // TODO(b/275095412): Clean up MLIR XLA functions after they are written + // back to function library, so that we don't need to do special handling + // for those functions here. + FindCalleesRecursiveForOp(symbol_table, op, xla_launch_callees); + } + }); + + llvm::DenseSet read_only_vars; + for (const auto &iter : resources) { + const auto &key = iter.first; + const auto &vars = iter.second; + if (std::all_of(vars.begin(), vars.end(), [](mlir::Operation *op) { + for (auto *user : op->getUsers()) { + if (!OnlyHasReadOrNoEffect(user)) return false; + } + return true; + })) { + read_only_vars.insert(key); + } + } + + mlir::TF::SideEffectAnalysis side_effect_analysis(module); + + mlir::OpBuilder builder(&module.getBodyRegion()); + // "_tfrt_resource_init" is the special function that executes all invariant + // ops (eg. read-only variables) used in the model. This function should be + // executed after user-specified initialization. + auto init_func_op = mlir::func::FuncOp::create( + builder, module.getLoc(), "_tfrt_resource_init", + mlir::FunctionType::get(module.getContext(), /*inputs=*/{}, + /*results=*/{})); + auto *block = init_func_op.addEntryBlock(); + builder.setInsertionPointToStart(block); + + HoistInfo module_hoist_info; + + for (auto func : module.getOps()) { + // Skips hoisting if this function is an init function or any callees, + // including recursive ones, of an init functions, because otherwise the + // hoisted values won't be initialized when this function is called. + if (IsSessionInitializer(func) || + init_callees.contains(func.getSymName()) || func == init_func_op || + xla_launch_callees.contains(func.getSymName())) + continue; + + // Skips hoisting if this function runs on TPU. This is will happen when + // fallback to TPUPartitionedCallOp is enabled for SPMD. + // TODO(b/214039254): remove this once tfrt support native SPMD. + bool has_tpu_op = false; + func.walk([&has_tpu_op](mlir::Operation *op) { + if (op->hasAttr("_tpu_replicate")) has_tpu_op = true; + }); + if (has_tpu_op) continue; + + HoistInvariantOpsInFunction(func, read_only_vars, + side_effect_analysis.GetAnalysisForFunc(func), + builder, module_hoist_info); + } + + // Create tf._TfrtSetResource ops in the init function. + for (auto iter : llvm::enumerate(module_hoist_info.hoisted_values)) { + mlir::Value value = iter.value().first; + int64_t index = iter.index(); + + auto new_value = module_hoist_info.value_mapping.lookup(value); + auto *new_op = new_value.getDefiningOp(); + assert(new_op); + builder.setInsertionPointAfter(new_op); + auto set_resource_op = mlir::TF::_TfrtSetResourceOp::create( + builder, new_op->getLoc(), new_value, index); + + // Preserve the device attribute. + llvm::StringRef device = kCpuDeviceName; + if (auto device_attr = new_op->getAttrOfType("device")) { + if (!device_attr.getValue().empty()) device = device_attr.getValue(); + } + set_resource_op->setAttr("device", builder.getStringAttr(device)); + } + + builder.setInsertionPointToEnd(block); + // Finish building the init function by inserting an return op. + mlir::func::ReturnOp::create(builder, init_func_op.getLoc()); + + // Now that we have the index for each value that will be replaced, we can + // create the tf._TfrtGetResource op in each function using these indices. + ReplaceHoistedValues(module_hoist_info.hoisted_values, builder); + + // Lastly, erase the hoisted ops in reverse topological order. + for (auto *op : + llvm::reverse(module_hoist_info.hoists_in_topological_order)) { + assert(op->use_empty()); + op->erase(); + } +} + +void LowerTFSavedModelPass::ReplaceHoistedValues( + llvm::ArrayRef> + hoisted_values, + mlir::OpBuilder &builder) { + struct HoistedValueInfo { + llvm::SmallVector hoisted_values; + llvm::SmallVector indices; + llvm::SmallVector shared_names; + llvm::SmallVector containers; + }; + // Rearrange the hoisted values by each function and each device. + llvm::DenseMap> + hoisted_values_by_block_device; + + // Find a block where to place tf._TfrtGetResource operation. We do not place + // get resource operations inside the `tf_device.cluster` operations, because + // these blocks are intended for later on-device compilation. Insert resource + // reads to the closest block outside of the `tf_device.cluster` operation. + auto hoist_into_block = [](mlir::Value value) -> mlir::Block * { + mlir::Operation *cluster_op = + value.getDefiningOp()->getParentOfType(); + return cluster_op ? cluster_op->getBlock() : value.getParentBlock(); + }; + + for (auto iter : llvm::enumerate(hoisted_values)) { + auto value = iter.value().first; + auto index = iter.index(); + auto &device_map = hoisted_values_by_block_device[hoist_into_block(value)]; + + assert(value.getDefiningOp() && "hoisted values must not be arguments."); + llvm::StringRef device = kCpuDeviceName; + if (auto device_attr = + value.getDefiningOp()->getAttrOfType("device")) { + if (!device_attr.getValue().empty()) device = device_attr.getValue(); + } + + auto &item = device_map[device]; + + item.hoisted_values.push_back(value); + item.indices.push_back(index); + item.shared_names.push_back(iter.value().second.name); + item.containers.push_back(iter.value().second.container); + } + + // Create tf._TfrtGetResource op for each function and device. + for (const auto &block_iter : hoisted_values_by_block_device) { + auto *block = block_iter.first; + const auto &device_map = block_iter.second; + + builder.setInsertionPointToStart(block); + for (const auto &device_iter : device_map) { + llvm::StringRef device = device_iter.getKey(); + mlir::ValueRange old_values = device_iter.getValue().hoisted_values; + const auto &indices = device_iter.getValue().indices; + const auto &shared_name_arr = device_iter.getValue().shared_names; + const auto &container_arr = device_iter.getValue().containers; + + llvm::SmallVector new_values; + + if (fuse_get_resource_ops_) { + auto get_resource_op = mlir::TF::_TfrtGetResourceOp::create( + builder, block->getParentOp()->getLoc(), old_values.getTypes(), + builder.getI64ArrayAttr(indices), + builder.getStrArrayAttr(shared_name_arr), + builder.getStrArrayAttr(container_arr)); + get_resource_op->setAttr("device", builder.getStringAttr(device)); + new_values = get_resource_op.getResults(); + } else { + for (int i = 0; i < old_values.size(); ++i) { + auto get_resource_op = mlir::TF::_TfrtGetResourceOp::create( + builder, block->getParentOp()->getLoc(), + mlir::TypeRange(old_values[i].getType()), + builder.getI64ArrayAttr(indices[i]), + builder.getStrArrayAttr(shared_name_arr[i]), + builder.getStrArrayAttr(container_arr[i])); + get_resource_op->setAttr("device", builder.getStringAttr(device)); + new_values.append(get_resource_op->result_begin(), + get_resource_op->result_end()); + } + } + + for (auto iter : llvm::zip(old_values, new_values)) { + auto old_value = std::get<0>(iter); + auto new_value = std::get<1>(iter); + old_value.replaceAllUsesWith(new_value); + } + } + } +} + +static llvm::SmallVector CompareTypes(mlir::TypeRange x, + mlir::TypeRange y) { + llvm::SmallVector results; + assert(x.size() == y.size()); + for (int i = 0, e = x.size(); i < e; ++i) { + if (x[i] != y[i]) results.push_back(i); + } + return results; +} + +// Converts ref variables to resource variables in a few cases. +// +// If the users of one variable in the entire module satisfies the following +// condition, it will be converted to resource variable: +// +// 1) tf.Identity op +// 2) tf.Assign op +// 3) side-effect-free ops: This is also the TF1 behavior that the TF executor +// will automatically convert ref tensors to non-ref tensors if the user is +// not expecting a ref tensor. Refer to +// http://cs?q=tensorflow/core/common_runtime/executor.cc:932%20at_cl:356873227 +class ConvertReferenceVariableToResourceVariablePass + : public mlir::PassWrapper> { + llvm::StringRef getArgument() const final { + return "tfrt-convert-ref-variables"; + } + llvm::StringRef getDescription() const final { + return "Convert reference variable to resource variables."; + } + void runOnOperation() override; + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + ConvertReferenceVariableToResourceVariablePass) +}; + +mlir::LogicalResult ConvertReferenceVariableToResourceVariable( + mlir::TF::VariableV2Op var_op) { + auto tensor_type = mlir::cast( + mlir::TF::DropRefType(var_op.getRef().getType())); + + llvm::SmallVector identity_ops; + llvm::SmallVector assign_ops; + llvm::SmallVector, 4> + side_effect_free_ops; + + for (mlir::OpOperand &use : var_op.getRef().getUses()) { + mlir::Operation *user = use.getOwner(); + + if (auto identity = llvm::dyn_cast(user)) { + identity_ops.push_back(identity); + continue; + } else if (auto assign = llvm::dyn_cast(user)) { + // Conservatively we only allow the case that the output of this tf.Assign + // is not consumed by any other ops. + if (assign.getOutputRef().use_empty()) { + assign_ops.push_back(assign); + continue; + } + } else if (mlir::isMemoryEffectFree(user)) { + side_effect_free_ops.push_back({user, use.getOperandNumber()}); + continue; + } + + return var_op.emitOpError() + << "failed to convert reference variables with unexpected users. " + << *user; + } + + mlir::OpBuilder builder(var_op); + + auto var_handle_op = mlir::TF::VarHandleOp::create( + builder, var_op.getLoc(), + mlir::RankedTensorType::get( + {}, mlir::TF::ResourceType::get( + llvm::ArrayRef{tensor_type}, + builder.getContext())), + var_op.getContainer(), var_op.getSharedName()); + + for (auto op : identity_ops) { + // Set insertion point to this identity_op so that the side-effect + // visibility is preserved. + builder.setInsertionPoint(op); + auto read_var_op = mlir::TF::ReadVariableOp::create( + builder, op.getLoc(), op.getType(), var_handle_op); + op.replaceAllUsesWith(read_var_op.getValue()); + op.erase(); + } + + for (auto op : assign_ops) { + // Set the insertion point after the assign op so that all operands are + // dominating the newly created op. + builder.setInsertionPoint(op); + mlir::TF::AssignVariableOp::create(builder, op.getLoc(), var_handle_op, + op.getValue()); + op.erase(); + } + + for (auto pair : side_effect_free_ops) { + mlir::Operation *op = pair.first; + unsigned idx = pair.second; + // Set the insertion point after the op so that all operands are dominating + // the newly created op. + builder.setInsertionPoint(op); + // Create a new read variable op, so that the side-effects are preserved. + auto read_var_op = mlir::TF::ReadVariableOp::create( + builder, op->getLoc(), tensor_type, var_handle_op); + op->setOperand(idx, read_var_op.getValue()); + } + + return mlir::success(); +} + +void ConvertReferenceVariableToResourceVariablePass::runOnOperation() { + auto module = getOperation(); + + // The key here is a tuple of device, container and shared_name to uniquely + // identify a variable. + llvm::DenseMap, + llvm::SmallVector> + ref_vars; + + // First, we collect all variables' corresponding tf.VariableV2 ops. + module.walk([&ref_vars](mlir::TF::VariableV2Op op) { + if (op.getSharedName().empty()) { + op.emitOpError() + << "unable to convert reference variables with empty shared_names."; + return mlir::WalkResult::interrupt(); + } + + llvm::StringRef device; + if (auto device_attr = op->getAttrOfType("device")) { + device = device_attr.getValue(); + } + + ref_vars[{device, op.getContainer(), op.getSharedName()}].push_back(op); + + return mlir::WalkResult::advance(); + }); + + // Then we perform rewrite for each variable if possible. + for (const auto &iter : ref_vars) { + const auto &var_ops = iter.second; + + for (auto var_op : var_ops) { + if (mlir::succeeded(ConvertReferenceVariableToResourceVariable(var_op))) + var_op.erase(); + } + } +} + +} // namespace + +std::unique_ptr> +CreateLowerTFSavedModelPass(bool hoist_invariant_ops, + bool fuse_get_resource_ops) { + return std::make_unique(hoist_invariant_ops, + fuse_get_resource_ops); +} + +std::unique_ptr> +CreateConvertReferenceVariableToResourceVariablePass() { + return std::make_unique(); +} + +static mlir::PassRegistration saved_model_pass; + +static mlir::PassRegistration + ref_var_pass; + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/merge_tf_if_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/merge_tf_if_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..7a2055b93a486276574fe2673f9f40a42a7fc72c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/merge_tf_if_ops.cc @@ -0,0 +1,296 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +// A special DenseMapInfo that hashes only operands of a operation, and treats +// two operations equivalent if their operands are the same. +struct OpWithSameArgsInfo : llvm::DenseMapInfo { + static unsigned getHashValue(const mlir::Operation *const_op) { + auto *op = const_cast(const_op); + return llvm::hash_combine( + llvm::hash_combine_range(op->operand_begin(), op->operand_end())); + } + + static bool isEqual(const mlir::Operation *const_lhs, + const mlir::Operation *const_rhs) { + auto *lhs = const_cast(const_lhs); + auto *rhs = const_cast(const_rhs); + if (lhs == rhs) return true; + + return std::equal(lhs->operand_begin(), lhs->operand_end(), + rhs->operand_begin(), rhs->operand_end()); + } +}; + +// This pass merges non-side-effecting tf.If ops if their operands are the same. +// For example, +// %r0 = tf.If(%cond, %x) {else = @else_0, then = @then_0} +// %r1, %r2 = tf.If(%cond, %x) {else = @else_1, then = @then_1} +// +// will be converted to: +// func private @merge_else(%arg) { +// %r0 = tf.PartitionedCall(%arg) {f = @else_0} +// %r1, %r2 = tf.PartitionedCall(%arg) {f = @else_1} +// return %r0, %r1, %r2 +// } +// func private @merge_then(%arg) { +// %r0 = tf.PartitionedCall(%arg) {f = @then_0} +// %r1, %r2 = tf.PartitionedCall(%arg) {f = @then_1} +// return %r0, %r1, %r2 +// } +// +// %r0, %r1, %r2 = tf.If(%cond, %arg) {else = @merge_else, then = @merge_then} +// +// Then the inliner pass is run on the module, so the bodies of else_0 and +// else_1 are inlined into the body of merge_else, and the bodies of then_0 and +// then_1 are inlined into the body of merge_then. +// +// Note that the results will be concatenated. +class MergeTfIfOpsPass + : public mlir::PassWrapper> { + llvm::StringRef getArgument() const final { return "tfrt-merge-tf-if-ops"; } + llvm::StringRef getDescription() const final { + return "Merge tf.If ops with the same arguments."; + } + + void runOnOperation() override { + constexpr int kMaxIter = 20; + auto module = getOperation(); + + bool changed = true; + + for (int i = 0; i < kMaxIter && changed; ++i) { + changed = false; + for (auto func_op : + llvm::make_early_inc_range(module.getOps())) { + changed |= ProcessFunction(func_op, i); + } + + mlir::OperationFingerPrint fingerprint(module); + + // Run optimization passes to expose more merge opportunities among the + // then-branch functions and the else-branch functions that are now + // respectively merged, for the next iteration. + mlir::OpPassManager pm(module.getOperationName()); + pm.addPass(mlir::createInlinerPass()); + pm.addPass(mlir::createCSEPass()); + pm.addPass(CreateDeduplicateIfResultPass()); + pm.addPass(mlir::createInlinerPass()); + pm.addPass(mlir::createCSEPass()); + pm.addNestedPass( + tfrt_compiler::CreateOptimizeTfForTfrtPass()); + if (mlir::failed(runPipeline(pm, module))) { + module.emitWarning( + absl::StrCat("could not run inliner pass within the " + "tfrt-merge-tf-if-ops pass iteration ", + i)); + break; + } + + changed |= fingerprint != mlir::OperationFingerPrint(module); + } + } + + bool ProcessFunction(mlir::func::FuncOp op, int iteration) { + // Use a hash map to group tf.If ops with the same operands. + llvm::SmallDenseMap, + 2, OpWithSameArgsInfo> + if_ops_to_merge; + + for (mlir::Operation &op : op.front()) { + auto if_op = llvm::dyn_cast(&op); + + // Skip non tf.If ops. + if (!if_op) continue; + + if_ops_to_merge[if_op].push_back(if_op); + } + + int id = 0; + + // Set the insertion point to the current function, as we will insert new + // functions here. + mlir::OpBuilder builder(op); + + // Track the tf.If ops that should be removed as they are merged. + llvm::SmallVector if_ops_to_remove; + + bool changed = false; + for (auto &iter : if_ops_to_merge) { + if (iter.second.size() <= 1) continue; + + // Merge tf.If ops that have the same operands. The merged branches will + // be given unique names. + MergeIfOpsWithSameArgs(builder, iter.first->getLoc(), + /*branch_prefix=*/ + absl::StrCat(op.getSymName().str(), "_merged_if_", + iteration, "_", id++), + iter.second); + + if_ops_to_remove.append(iter.second.begin(), iter.second.end()); + changed = true; + } + + // Now that we are no longer using `if_ops_to_merge` or any other data + // structures that uses the operations that will be removed, we can now + // erase these if ops. + for (auto op : if_ops_to_remove) op->erase(); + + return changed; + } + + void MergeIfOpsWithSameArgs(mlir::OpBuilder &builder, mlir::Location loc, + absl::string_view branch_prefix, + llvm::MutableArrayRef if_ops) { + assert(if_ops.size() > 1); + + // The results of the merged tf.If op are the concatenation of results of + // the original tf.If ops. + llvm::SmallVector new_result_types; + for (auto if_op : if_ops) { + new_result_types.append(if_op->result_type_begin(), + if_op->result_type_end()); + } + + auto branch_function_type = builder.getFunctionType( + if_ops.front().getInput().getTypes(), new_result_types); + + // Create new branches for the merged tf.If op. + auto then_branch_name = CreateBranchFunction( + builder, loc, branch_prefix, + /*branch_suffix=*/"_then", branch_function_type, if_ops, + [](mlir::TF::IfOp op) { return op.getThenBranchAttr(); }); + + auto else_branch_name = CreateBranchFunction( + builder, loc, branch_prefix, + /*branch_suffix=*/"_else", branch_function_type, if_ops, + [](mlir::TF::IfOp op) { return op.getElseBranchAttr(); }); + + mlir::OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPoint(if_ops.front()); + + bool is_stateless = + std::all_of(if_ops.begin(), if_ops.end(), + [](mlir::TF::IfOp op) { return op.getIsStateless(); }); + + // Create the merged tf.If op using the new branches. + auto new_if_op = mlir::TF::IfOp::create( + builder, loc, new_result_types, if_ops.front().getCond(), + if_ops.front().getInput(), then_branch_name, else_branch_name, + is_stateless); + + // Replace the uses of results of the original tf.If ops with the results of + // the merged tf.If op. + auto new_result_iter = new_if_op.getOutput().begin(); + for (auto if_op : if_ops) { + for (auto result : if_op.getOutput()) { + assert(new_result_iter != new_if_op.getOutput().end()); + result.replaceAllUsesWith(*new_result_iter); + ++new_result_iter; + } + } + } + + llvm::StringRef CreateBranchFunction( + mlir::OpBuilder &builder, mlir::Location loc, + absl::string_view branch_prefix, absl::string_view branch_suffix, + mlir::FunctionType branch_function_type, + llvm::ArrayRef if_ops, + llvm::function_ref get_branch) { + std::string branch_name = absl::StrCat(branch_prefix, branch_suffix); + auto branch = mlir::func::FuncOp::create(builder, loc, branch_name, + branch_function_type); + branch.setVisibility(mlir::func::FuncOp::Visibility::Private); + + mlir::OpBuilder::InsertionGuard guard(builder); + + // In the body of newly created branch function, we insert + // tf.PartitionedCall ops to call the original branches. + auto *block = branch.addEntryBlock(); + builder.setInsertionPointToStart(block); + auto empty_string_attr = builder.getStringAttr(""); + + llvm::SmallVector results; + results.reserve(branch_function_type.getNumResults()); + + for (auto if_op : if_ops) { + // Create the call op to the original branch. The arguments are simply + // the arguments from the wrapper function. + auto call_op = mlir::TF::PartitionedCallOp::create( + builder, if_op.getLoc(), if_op.getResultTypes(), + block->getArguments(), + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, get_branch(if_op), + empty_string_attr, empty_string_attr, empty_string_attr); + + // The results are the concatenation of the original branches. + results.append(call_op.getOutput().begin(), call_op.getOutput().end()); + } + + mlir::func::ReturnOp::create(builder, loc, results); + + return branch.getSymName(); + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(MergeTfIfOpsPass) +}; + +} // namespace + +std::unique_ptr> CreateMergeTfIfOpsPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateMergeTfIfOpsPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..16ff39ae4dfb3bc9cb6f959e8b2f83180951a458 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/BUILD @@ -0,0 +1,289 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + "//tensorflow/compiler/mlir/tfrt:__subpackages__", + "//tensorflow/core/tfrt:__subpackages__", + ], +) + +cc_library( + name = "parallelization", + srcs = ["parallelization.cc"], + hdrs = ["parallelization.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tfrt:constants", + "//tensorflow/compiler/mlir/tfrt:cost_analysis", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@tf_runtime//:stream_analysis", + ], +) + +cc_library( + name = "assign_op_key", + srcs = ["assign_op_key.cc"], + hdrs = ["assign_op_key.h"], + deps = [ + ":util", + "//tensorflow/compiler/mlir/tfrt:constants", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "mlrt_device_constants", + hdrs = ["mlrt_device_constants.h"], +) + +cc_library( + name = "util", + srcs = ["util.cc"], + hdrs = ["util.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_a_m_inc_gen", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_ops_n_z_inc_gen", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) + +cc_library( + name = "tf_to_mlrt", + srcs = ["tf_to_mlrt.cc"], + hdrs = ["tf_to_mlrt.h"], + deps = [ + ":execute_op_registry", + ":mlrt_device_constants", + ":tpu_conversion_patterns", + ":util", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:convert_tensor", + "//tensorflow/compiler/mlir/tensorflow:translate_utils", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops_inc_gen", + "//tensorflow/compiler/mlir/tfrt:constants", + "//tensorflow/compiler/mlir/tfrt:tfrt_pipeline_options", + "//tensorflow/compiler/mlir/tfrt:transform_utils", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_tpu_ops", + "//tensorflow/core:framework", + "//tensorflow/core/platform:status", + "//tensorflow/core/tfrt/fallback:fallback_state", + "//tensorflow/core/tfrt/fallback:op_kernel_runner_cache", + "@com_google_absl//absl/log", + "@com_google_protobuf//:protobuf_headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + "@xla//xla/tsl/platform:status", + ], +) + +cc_library( + name = "passes", + srcs = ["passes.cc"], + hdrs = ["passes.h"], + deps = [ + ":assign_op_key", + ":async_while", + ":fuse_mlrt_ops", + ":ifrt_set_tpu_host_allocator", + ":parallelization", + ":rewrite_ifrt_load_variable", + ":tf_to_mlrt", + ":while_to_map_fn", + "//tensorflow/compiler/mlir/tfrt:tfrt_pipeline_options", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "//tensorflow/core/tfrt/fallback:fallback_state", + "@com_google_absl//absl/log:check", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Transforms", + ], +) + +cc_library( + name = "execute_op_registry", + hdrs = ["execute_op_registry.h"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +cc_library( + name = "tpu_conversion_patterns", + srcs = ["tpu_conversion_patterns.cc"], + hdrs = ["tpu_conversion_patterns.h"], + deps = [ + ":execute_op_registry", + ":mlrt_device_constants", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tfrt:tfrt_pipeline_options", + "//tensorflow/compiler/mlir/tfrt:transform_utils", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_tpu_ops", + "@com_google_absl//absl/log:check", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "fuse_mlrt_ops", + srcs = ["fuse_mlrt_ops.cc"], + hdrs = ["fuse_mlrt_ops.h"], + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:Pass", + ], +) + +cc_library( + name = "import_model", + srcs = ["import_model.cc"], + hdrs = ["import_model.h"], + deps = [ + ":assign_op_key", + ":passes", + ":while_to_map_fn", + "//tensorflow/compiler/mlir/tensorflow:bridge_logger", + "//tensorflow/compiler/mlir/tensorflow:dump_mlir_util", + "//tensorflow/compiler/mlir/tensorflow:error_util", + "//tensorflow/compiler/mlir/tfrt:export", + "//tensorflow/compiler/mlir/tfrt:import_model", + "//tensorflow/compiler/mlir/tfrt:tf_to_tfrt", + "//tensorflow/compiler/mlir/tfrt:tfrt_compile_options", + "//tensorflow/compiler/mlir/tfrt:tfrt_pipeline_options", + "//tensorflow/compiler/mlir/tfrt/translate/mlrt:mlir_to_bytecode", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:status", + "//tensorflow/core/platform:statusor", + "//tensorflow/core/tfrt/fallback:cost_recorder", + "//tensorflow/core/tfrt/fallback:fallback_state", + "//tensorflow/core/tfrt/mlrt/attribute", + "//tensorflow/core/tfrt/mlrt/bytecode", + "//tensorflow/core/tfrt/runtime", + "@com_google_absl//absl/log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + "@xla//xla/tsl/platform:errors", + ], +) + +cc_library( + name = "while_to_map_fn", + srcs = ["while_to_map_fn.cc"], + hdrs = ["while_to_map_fn.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "async_while", + srcs = ["async_while.cc"], + hdrs = ["async_while.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:tensorflow_analysis", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "rewrite_ifrt_load_variable", + srcs = ["rewrite_ifrt_load_variable.cc"], + hdrs = ["rewrite_ifrt_load_variable.h"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) + +cc_library( + name = "ifrt_set_tpu_host_allocator", + srcs = ["ifrt_set_tpu_host_allocator.cc"], + hdrs = ["ifrt_set_tpu_host_allocator.h"], + deps = [ + ":mlrt_device_constants", + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow/ir/host_runtime:tensorflow_tfrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:tf_mlrt_ops", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TransformUtils", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.cc new file mode 100644 index 0000000000000000000000000000000000000000..5ed413f4b928060ed3911a0637c3bb68df8da07a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.cc @@ -0,0 +1,71 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h" + +#include + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/constants.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +class AssignOpKeyPass + : public mlir::PassWrapper> { + public: + AssignOpKeyPass() = default; + AssignOpKeyPass& operator=(const AssignOpKeyPass&) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AssignOpKeyPass) + + private: + llvm::StringRef getArgument() const final { return "tf-mlrt-assign-op-key"; } + llvm::StringRef getDescription() const final { + return "tf-mlrt-assign-op-key"; + } + + void runOnOperation() override; +}; + +void AssignOpKeyPass::runOnOperation() { + auto module = getOperation(); + mlir::OpBuilder builder(module); + + int32_t op_key = 0; + module.walk([&builder, &op_key](mlir::Operation* op) mutable { + if (UseFallback(op)) { + op->setAttr(tensorflow::tfrt_compiler::kOpKeyAttrName, + builder.getI32IntegerAttr(op_key)); + op_key++; + } + }); +} + +} // namespace + +std::unique_ptr> CreateAssignOpKeyPass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h new file mode 100644 index 0000000000000000000000000000000000000000..7dcea29ff89e05d671dc8406e140a02a9af7d233 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h @@ -0,0 +1,32 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASSIGN_OP_KEY_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASSIGN_OP_KEY_H_ +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +// Create a pass that assigns an op_key to every fallback OP. The op_key +// provides a uniform key to look up online cost for a specific op. +// This pass is expected to run before parallerization. +std::unique_ptr> CreateAssignOpKeyPass(); + +} // namespace mlrt_compiler +} // namespace tensorflow +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASSIGN_OP_KEY_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.cc new file mode 100644 index 0000000000000000000000000000000000000000..b4cbcd50dea9ec77b65f50af5c1d83a2bb80e1ff --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.cc @@ -0,0 +1,745 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +// Move await right before its tensor is used. +void MoveTFAwaitOpToFirstUse(mlir::Block &block) { + llvm::SmallVector await_ops; + for (auto &op : block) { + if (auto await_op = llvm::dyn_cast(&op)) { + await_ops.push_back(await_op); + } + } + + for (auto op : await_ops) { + auto result = op.getResult(); + if (result.use_empty()) continue; + + mlir::Operation *first_user = *result.user_begin(); + for (auto *user : result.getUsers()) { + if (user->isBeforeInBlock(first_user)) { + first_user = user; + } + } + + op->moveBefore(first_user); + } +} + +// Move promise right after the tensor becomes available. +void MoveTFPromiseOpRightAfterAvailable(mlir::Block &block) { + llvm::SmallVector promise_ops; + for (auto &op : block) { + if (auto promise_op = llvm::dyn_cast(&op)) { + promise_ops.push_back(promise_op); + } + } + + for (auto promise_op : promise_ops) { + auto tensor = promise_op.getOperand(1); + // Promise is on variant tensor, which must have a defining op inside the + // block. + DCHECK(tensor.getDefiningOp() != nullptr); + promise_op->moveAfter(tensor.getDefiningOp()); + } +} + +// Converts applicable tf.While to tf_mlrt.AsyncWhile. +// +// In pseudo code, the following +// +// %res0, %res1 = tf.While(%arg0, %arg1)(body = @while_body, cond = +// @while_predicate) +// +// is converted to +// +// %0 = tf.PartitionedCall(%res0) (f = +// @while_predicate/TfMlrtAsyncWhilePredicate) %1, %2, %3 = +// tf_mlrt.async_while(%0, %arg0, %arg1) (body = +// @while_body/TfMlrtAsyncWhileBody, immutable_size = 1) %res0 = +// tf_mlrt.await(%2) %res1 = tf_mlrt.await(%3) +// +class AsyncWhilePass + : public mlir::PassWrapper> { + public: + AsyncWhilePass() = default; + AsyncWhilePass &operator=(const AsyncWhilePass &) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AsyncWhilePass) + + private: + // The input to while op will be re-order and supplied as input to async_while + // op. + struct ArgumentRemap { + std::vector new_to_old_remap; + std::vector old_to_new_remap; + }; + + // The input to while op will grouped into immutables(invariants) between + // iterations and mutables(variants). + struct ArgumentGroups { + std::vector immutables; + std::vector mutables; + }; + + // First argument of the new async_while is a predicate. + static constexpr int kNonPredicateStartIndex = 1; + + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + llvm::StringRef getArgument() const final { return "tf-mlrt-async-while"; } + + llvm::StringRef getDescription() const final { + return "Convert tf.while to tf_mlrt.async_while when possible for parallel " + "execution."; + } + + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::SymbolTable symbol_table(module); + mlir::TF::SideEffectAnalysis side_effect_analysis(module); + + // Keep a record of predicate and body functions that we already used to + // create new predicate and body functions. + // key is the original function name and value is the newly created + // function. + // We keep our own record instead of relying on SymbolTable to know whether + // a function is already converted is because SymbolTable might create new + // symbol name for inserted function to avoid conflict. + llvm::SmallDenseMap + processed_functions; + // Use make_early_inc_range because the processing might insert new node + // into the list + for (auto func_op : + llvm::make_early_inc_range(module.getOps())) { + MayConvertWhileToAsyncWhile( + func_op, symbol_table, processed_functions, + side_effect_analysis.GetAnalysisForFunc(func_op)); + } + } + + void MayConvertWhileToAsyncWhile( + mlir::func::FuncOp op, mlir::SymbolTable &symbol_table, + llvm::SmallDenseMap + &processed_functions, + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis) { + mlir::OpBuilder builder(op); + for (mlir::Operation &op : llvm::make_early_inc_range(op.front())) { + auto while_op = llvm::dyn_cast(&op); + if (!while_op) continue; + + // Fill in bodies + mlir::func::FuncOp body_fn = + symbol_table.lookup(while_op.getBody()); + + mlir::func::FuncOp predicate_fn = + symbol_table.lookup(while_op.getCond()); + + if (ShouldConvertWhileToAsyncWhile(body_fn, predicate_fn)) { + std::vector predicate_determinat = + GetPredicateDeterminat(predicate_fn); + + ArgumentGroups argument_groups = GetArgumentGroups(body_fn); + ArgumentRemap argument_remap = + CreateArgumentRemap(body_fn, argument_groups.immutables); + + mlir::func::FuncOp new_predicate_fn = + CreatePredicate(builder, symbol_table, processed_functions, + predicate_fn, predicate_determinat); + + mlir::func::FuncOp new_body = + CreateBody(builder, symbol_table, processed_functions, body_fn, + new_predicate_fn, predicate_determinat, argument_groups, + argument_remap, side_effect_analysis); + + mlir::OpBuilder::InsertionGuard insertion_guard(builder); + builder.setInsertionPoint(while_op); + + // Call the predicate function first. async_while requires the first + // predicate to be given. + llvm::SmallVector predicate_input; + for (int i : predicate_determinat) { + predicate_input.push_back(while_op->getOperand(i)); + } + + auto empty_string_attr = builder.getStringAttr(""); + auto init_predicate = mlir::TF::PartitionedCallOp::create( + builder, while_op->getLoc(), predicate_fn.getResultTypes()[0], + predicate_input, /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, + mlir::FlatSymbolRefAttr::get(new_predicate_fn.getSymNameAttr()), + empty_string_attr, empty_string_attr, empty_string_attr); + + // These arguments exclude predicate. + llvm::SmallVector async_while_input; + async_while_input.reserve(while_op->getNumOperands()); + for (auto old_arg_idx_iter = + argument_remap.new_to_old_remap.begin() + 1; + old_arg_idx_iter != argument_remap.new_to_old_remap.end(); + ++old_arg_idx_iter) { + async_while_input.push_back(while_op.getOperand(*old_arg_idx_iter)); + } + + llvm::SmallVector async_while_output_types; + async_while_output_types.resize(1 + while_op->getNumResults()); + std::fill(async_while_output_types.begin(), + async_while_output_types.end(), + builder.getType()); + + auto async_while_op = tf_mlrt::TFAsyncWhileOp::create( + builder, while_op.getLoc(), async_while_output_types, + init_predicate->getResult(0), async_while_input, + new_body.getSymName(), argument_groups.immutables.size()); + + for (int i = 0; i < while_op.getResults().size(); ++i) { + if (!while_op.getResult(i).use_empty()) { + auto async_while_result = tf_mlrt::TFAwaitOp::create( + builder, while_op.getLoc(), while_op->getResultTypes()[i], + async_while_op.getResult(argument_remap.old_to_new_remap[i])); + while_op.getResult(i).replaceAllUsesWith( + async_while_result.getResult()); + } + } + + while_op.erase(); + } + } + + // TFAwaitOp may have been inserted, move them to first use. + MoveTFAwaitOpToFirstUse(op.getBody().front()); + } + + // Creates a new predicate function that returns a boolean tensor. This new + // predicate function is converted from the original predicate function after + // removing the unused input arguments. We want the inputs to the new + // predicate function to be the minimal set because the new body function + // wants to call the predicate as early as possible to maximize the chances of + // parallerization. Also since the new body function reorders the input + // arguments, the new predicate function follows that new order to be + // consistent (just like the original predicate function and body function + // have the same order of arguments). + mlir::func::FuncOp CreatePredicate( + mlir::OpBuilder &builder, mlir::SymbolTable &symbol_table, + llvm::SmallDenseMap + &processed_functions, + mlir::func::FuncOp original_predicate_fn, + const std::vector &predicate_determinat) { + auto &processed_function = + processed_functions[original_predicate_fn.getName()]; + if (processed_function) { + return processed_function; + } + mlir::OpBuilder::InsertionGuard insertion_guard(builder); + builder.setInsertionPointAfter(original_predicate_fn); + + // Input to the new predicate only contains arguments listed in + // predicate_determinat. + llvm::SmallVector remapped_input_types; + for (int i : predicate_determinat) { + remapped_input_types.push_back( + original_predicate_fn.getArgumentTypes()[i]); + } + + std::string new_predicate_fn_name = absl::StrCat( + original_predicate_fn.getName().str(), "/TfMlrtAsyncWhilePredicate"); + mlir::func::FuncOp new_predicate_fn = mlir::func::FuncOp::create( + builder, original_predicate_fn->getLoc(), new_predicate_fn_name, + mlir::FunctionType::get(original_predicate_fn.getContext(), + remapped_input_types, + original_predicate_fn.getResultTypes())); + + processed_function = new_predicate_fn; + + new_predicate_fn.setPrivate(); + + builder.setInsertionPointToEnd(new_predicate_fn.addEntryBlock()); + + // Inputs will be reordered according to the body functions' reorder. + mlir::IRMapping remap; + int j = 0; + for (int i : predicate_determinat) { + remap.map(original_predicate_fn.getArgument(i), + new_predicate_fn.getArgument(j)); + j++; + } + + for (auto &op_it : original_predicate_fn.getBody().front()) { + builder.clone(op_it, remap); + } + + symbol_table.insert(new_predicate_fn); + return new_predicate_fn; + } + + // Creates the new body function for the async while. + // The new body function has the input signature of + // (predicate_promise, mutable0_future, mutable0_promise, mutable1_future, + // mutable1_promise, ..., immutable0, immutable1,...) The new body is mostly a + // clone from the old body function, but the input arguments are reordered + // such that the immutables(invariants) between iterations are always at the + // bottom of the argument list. Additionaly, the new body function calls the + // new predicate function and puts the boolean predicate for the next + // iteration into the predicate_promise. + mlir::func::FuncOp CreateBody( + mlir::OpBuilder &builder, mlir::SymbolTable &symbol_table, + llvm::SmallDenseMap + &processed_functions, + mlir::func::FuncOp original_body_fn, mlir::func::FuncOp new_predicate_fn, + const std::vector &predicate_determinat, + const ArgumentGroups &argument_groups, + const ArgumentRemap &argument_remap, + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis) { + auto &processed_function = processed_functions[original_body_fn.getName()]; + if (processed_function) { + return processed_function; + } + + // Safe to do topology sort here b/c there are no promise/future involved + // here. + SortBodyFnByFirstReadyFirstRun(original_body_fn, predicate_determinat, + side_effect_analysis); + + mlir::OpBuilder::InsertionGuard insertion_guard(builder); + builder.setInsertionPointAfter(original_body_fn); + + // Create the new body function signature. + std::vector remapped_input_types; + std::vector remapped_output_types; + + // predicate promise is the first argument. + remapped_input_types.push_back( + builder.getType()); + + // future and promise pairs for the mutables follows. + for (int i = 0; i < argument_groups.mutables.size(); ++i) { + remapped_input_types.push_back( + builder.getType()); + remapped_input_types.push_back( + builder.getType()); + } + + // immutables are always at the bottom of the argument list. + for (int i : argument_groups.immutables) { + remapped_input_types.push_back(original_body_fn.getArgumentTypes()[i]); + } + + std::string new_body_fn_name = + absl::StrCat(original_body_fn.getName().str(), "/TfMlrtAsyncWhileBody"); + auto body_fn = mlir::func::FuncOp::create( + builder, original_body_fn->getLoc(), new_body_fn_name, + mlir::FunctionType::get(original_body_fn.getContext(), + remapped_input_types, remapped_output_types)); + processed_function = body_fn; + + body_fn.setPrivate(); + + // Inserts await for futures so that all mutable tensors are ready. + // We will move these awaits to first use location later on. + builder.setInsertionPointToEnd(body_fn.addEntryBlock()); + // First future starts at index 1 after predicate promise. + int future_index = 1; + mlir::IRMapping mapping; + int body_arg_idx = kNonPredicateStartIndex; + for (int i : argument_groups.mutables) { + auto future_value = tf_mlrt::TFAwaitOp::create( + builder, body_fn->getLoc(), original_body_fn.getArgumentTypes()[i], + body_fn.getArgument(future_index)); + mapping.map(original_body_fn.getArgument(i), future_value); + // Move by 2 b/c each variant correspond to one future and one promise + body_arg_idx += 2; + future_index += 2; + } + + for (int i : argument_groups.immutables) { + mapping.map(original_body_fn.getArgument(i), + body_fn.getArgument(body_arg_idx)); + body_arg_idx++; + } + + // All future values are ready; so we can clone from the original body + // function. + for (auto &op : original_body_fn.getBody().front()) { + builder.clone(op, mapping); + } + + auto return_op = body_fn.getBody().front().getTerminator(); + + // Find the earliest location that the new predicate function can execute. + auto *body_block = &body_fn.getBody().front(); + auto *earliest_op = &body_block->front(); + for (int i = 0; i < return_op->getOperands().size(); ++i) { + if (std::find(predicate_determinat.begin(), predicate_determinat.end(), + i) != predicate_determinat.end()) { + auto latest_op = return_op->getOperands()[i].getDefiningOp(); + if (latest_op != nullptr && earliest_op->isBeforeInBlock(latest_op)) { + earliest_op = latest_op; + } + } + } + + llvm::SmallVector predicate_inputs; + for (int i : predicate_determinat) { + predicate_inputs.push_back(return_op->getOperands()[i]); + } + // Call new predicate when all dependencies are ready. + builder.setInsertionPointAfter(earliest_op); + llvm::SmallVector predicate_returned_types; + auto empty_string_attr = builder.getStringAttr(""); + auto predicate_op = mlir::TF::PartitionedCallOp::create( + builder, earliest_op->getLoc(), new_predicate_fn.getResultTypes(), + predicate_inputs, /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, + mlir::FlatSymbolRefAttr::get(new_predicate_fn.getSymNameAttr()), + empty_string_attr, empty_string_attr, empty_string_attr); + + // Insert promise right before return + builder.setInsertionPoint(return_op); + // builder.create(return_op->getLoc()); + auto predicate_tensor = predicate_op->getResult(0); + + const int kPredicatePromiseIndex = 0; + tf_mlrt::TFPromiseOp::create(builder, return_op->getLoc(), + body_fn.getArgument(kPredicatePromiseIndex), + predicate_tensor); + // First variant promise start at 2 as [predicate_promise, arg0_future, + // arg0_promise, ...] + int promise_index = 2; + for (int i : argument_groups.mutables) { + tf_mlrt::TFPromiseOp::create(builder, return_op->getLoc(), + body_fn.getArgument(promise_index), + return_op->getOperand(i)); + promise_index += 2; + } + mlir::func::ReturnOp::create(builder, return_op->getLoc()); + return_op->erase(); + + // Reorder await and promise to reduce blocking waits. + MoveTFAwaitOpToFirstUse(body_fn.getBody().front()); + MoveTFPromiseOpRightAfterAvailable(body_fn.getBody().front()); + + symbol_table.insert(body_fn); + return body_fn; + } + + // Decides whether we should convert a while to async_while. + // This determination is largely by heuristic. + bool ShouldConvertWhileToAsyncWhile(mlir::func::FuncOp body_fn, + mlir::func::FuncOp predicate_fn) { + // If the predicate are NOT determined by simple ops such as (less) + // comparison and logic and, do not apply conversion. + for (auto &op : predicate_fn.getBody().front()) { + if (!llvm::isa(&op)) + return false; + } + + // We then apply some heuristics: + // 1. If all variants are needed to determine the predicate, do not convert + // to async while because the next iteration has a low chance of overlapping + // with the current iteration. + // 2. The predicate shall be updated before any other variants is updated. + std::vector predicate_determinat = + GetPredicateDeterminat(predicate_fn); + ArgumentGroups argument_groups = GetArgumentGroups(body_fn); + + // Find out defining ops for predicate and non-predicate inside the body. + llvm::SmallSetVector predicate_determinant_ops; + llvm::SmallSetVector worker_ops; + for (int i : argument_groups.mutables) { + auto *defining_op = body_fn.getBody() + .front() + .getTerminator() + ->getOperand(i) + .getDefiningOp(); + if (std::find(predicate_determinat.begin(), predicate_determinat.end(), + i) != predicate_determinat.end()) { + predicate_determinant_ops.insert(defining_op); + } else { + worker_ops.insert(defining_op); + } + } + + // simple heuristic to reject pipelining: predicate are determined by all + // variants. + if (worker_ops.empty()) return false; + + // Validate that the determinant of predicate must be updated before any + // other variants and variants are determined in a cluster. + int predicate_determinant_cnt = 0; + for (auto &body_op : body_fn.getBlocks().front()) { + if (predicate_determinant_cnt < predicate_determinant_ops.size()) { + if (worker_ops.contains(&body_op)) { + return false; + } else if (predicate_determinant_ops.contains(&body_op)) { + predicate_determinant_cnt++; + if (predicate_determinant_cnt >= predicate_determinant_ops.size()) { + break; + } + } + } + } + + return true; + } + + // Returns the list of minimal argument indices that determines the predicate. + std::vector GetPredicateDeterminat(mlir::func::FuncOp predicate_fn) { + std::vector determinat; + for (auto &op : predicate_fn.getArguments()) { + if (!op.use_empty()) { + determinat.push_back(op.getArgNumber()); + } + } + return determinat; + } + + // Groups the arguments of the original body function into two groups: + // mutables between iterations and immutables across iterations. + ArgumentGroups GetArgumentGroups(mlir::func::FuncOp body_fn) { + ArgumentGroups groups; + auto *return_op = body_fn.getBlocks().front().getTerminator(); + + int i = 0; + for (auto return_op_operand : return_op->getOperands()) { + if (std::find(body_fn.getArguments().begin(), + body_fn.getArguments().end(), + return_op_operand) != body_fn.getArguments().end()) { + groups.immutables.push_back(i); + } else { + groups.mutables.push_back(i); + } + i++; + } + return groups; + } + + // Re-orders the argument such that all the immutables are at the bottom of + // the argument list. This function returns a mapping for such re-order. + ArgumentRemap CreateArgumentRemap(mlir::func::FuncOp original_body_fn, + std::vector invariant) { + absl::flat_hash_set invariant_set(invariant.begin(), invariant.end()); + ArgumentRemap argument_remap; // new to old + argument_remap.old_to_new_remap.resize( + original_body_fn.getArguments().size()); + argument_remap.new_to_old_remap.resize( + original_body_fn.getArguments().size() + 1); + + const int invariant_start_index = + original_body_fn.getArguments().size() - invariant_set.size(); + int variant_index = 0 + kNonPredicateStartIndex; + int invariant_index = invariant_start_index + kNonPredicateStartIndex; + for (int i = 0; i < original_body_fn.getArguments().size(); ++i) { + if (invariant_set.contains(i)) { + argument_remap.old_to_new_remap[i] = invariant_index; + argument_remap.new_to_old_remap[invariant_index] = i; + invariant_index++; + } else { + argument_remap.old_to_new_remap[i] = variant_index; + argument_remap.new_to_old_remap[variant_index] = i; + variant_index++; + } + } + DCHECK_EQ(variant_index, invariant_start_index + kNonPredicateStartIndex); + DCHECK_EQ(invariant_index, + original_body_fn.getArguments().size() + kNonPredicateStartIndex); + return argument_remap; + } + + // Sorts the operations in the block to an order of first ready first run to + // increase the parallelism between iterations. The order of ops before the + // predicate can be computed remains unchanged because we prefer to have the + // next iteration starts as soon as possible. For ops that are + // ready at the same time, ops producing the final results will be executed + // first. + void SortBodyFnByFirstReadyFirstRun( + mlir::func::FuncOp body_fn, const std::vector &predicate_determinat, + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis) { + mlir::Block &body_block = body_fn.getBody().front(); + mlir::Operation *return_op = body_block.getTerminator(); + if (!return_op) return; + + // Ops in final execution order. + std::vector sorted_ops; + // Number of outstanding (not executed yet) dependency for an op. When that + // number hits zero, this op is ready to be executed. + llvm::DenseMap dependency_cnt; + + // Build the graph and mark ops that depends on input arguments in the + // `new_ready_ops`. We will prioritize final result producing ops later on + // before finalizing their order. + std::list new_ready_ops; + for (mlir::Operation &op : body_block.without_terminator()) { + // Existing side effecting ops are dependencies on all subsequent ops. + dependency_cnt[&op] = + op.getNumOperands() + + side_effect_analysis.DirectControlPredecessors(&op).size(); + + // Block argument is viewed as ready right away + for (const mlir::Value &operand : op.getOperands()) { + if (!operand.getDefiningOp()) { + dependency_cnt[&op]--; + } + } + if (!dependency_cnt[&op]) new_ready_ops.push_back(&op); + } + + // Ops that produces final returned results have priority among ops that are + // ready at the same time. + llvm::DenseSet final_result_producing_ops; + for (const auto &operand : return_op->getOperands()) { + if (auto *op = operand.getDefiningOp()) { + final_result_producing_ops.insert(op); + } + } + + // The order of all ops before the predicate for the next iteration + // can be computed remains unchanged because we want to kick off next + // iteration preparation asap by making the predicate ready asap. + llvm::SmallDenseSet predicate_determined_ops; + for (int i : predicate_determinat) { + if (auto *defining_op = return_op->getOperand(i).getDefiningOp()) { + predicate_determined_ops.insert(defining_op); + } + } + auto iter = body_block.without_terminator().begin(); + // Make those ops before predicate ready and remember them so that they + // don't get resorted. + llvm::SmallDenseSet pre_scheduled_ops; + while (!predicate_determined_ops.empty()) { + if (predicate_determined_ops.contains(&*iter)) { + predicate_determined_ops.erase(&*iter); + } + sorted_ops.push_back(&*iter); + pre_scheduled_ops.insert(&*iter); + iter++; + } + + // Topologically sorts the ops in a breadth first order. + std::deque ready_ops; + MoveNewReadyOps(ready_ops, std::move(new_ready_ops), + final_result_producing_ops); + while (!ready_ops.empty()) { + mlir::Operation *active_op = ready_ops.front(); + if (!pre_scheduled_ops.contains(active_op)) + sorted_ops.push_back(active_op); + ready_ops.pop_front(); + // Do not directly push new ready ops into ready_ops yet. + // We will re-arrange the new ready ops by moving those producing the + // final results to the top so that they get executed first. + std::list new_ready_ops; + for (const auto &result : active_op->getOpResults()) { + // Uses instead of users in case users can dedup. + for (const auto &use : result.getUses()) { + int cnt = --dependency_cnt[use.getOwner()]; + if (!cnt) new_ready_ops.push_back(use.getOwner()); + } + } + for (auto *control_dependent : + side_effect_analysis.DirectControlSuccessors(active_op)) { + int cnt = --dependency_cnt[control_dependent]; + if (!cnt) new_ready_ops.push_back(control_dependent); + } + + MoveNewReadyOps(ready_ops, std::move(new_ready_ops), + final_result_producing_ops); + } + + // Update the block with the new order. + sorted_ops.push_back(return_op); + for (mlir::Operation *op : sorted_ops) { + op->remove(); + body_block.push_back(op); + } + } + + // Add ops in the `new_ready_ops` to `ready_ops` in the following order: + // ops that produces final returned result are added first and then the rest + // ops. + void MoveNewReadyOps( + std::deque &ready_ops, + std::list new_ready_ops, + const llvm::DenseSet &final_result_producing_ops) { + auto iter = new_ready_ops.begin(); + while (iter != new_ready_ops.end()) { + if (final_result_producing_ops.contains(*iter)) { + ready_ops.push_back(*iter); + iter = new_ready_ops.erase(iter); + } else { + iter++; + } + } + for (mlir::Operation *op : new_ready_ops) { + ready_ops.push_back(op); + } + } +}; + +} // namespace + +std::unique_ptr> CreateAsyncWhilePass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.h new file mode 100644 index 0000000000000000000000000000000000000000..dd12418c54af0c60874697f9a1aafde46a4d2f6e --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.h @@ -0,0 +1,36 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASYNC_WHILE_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASYNC_WHILE_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +// Creates a pass that converts applicable tf.While to tf_mlrt.AsyncWhile. +// tf_mlrt.AsyncWhile dispatch iterations asynchronously, thus allowing +// pipelining between iterations to reduce latency. This is intended for +// tf.While that is not converted from tf.MapFn, but still can benefit from +// asynchronous execution of iterations to reduce latency. +std::unique_ptr> CreateAsyncWhilePass(); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_ASYNC_WHILE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h new file mode 100644 index 0000000000000000000000000000000000000000..1f94c2b6513e3297bca2af535af37866b7d8913d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h @@ -0,0 +1,60 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_EXECUTE_OP_REGISTRY_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_EXECUTE_OP_REGISTRY_H_ + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +class ExecuteOpRegistry { + public: + mlir::LogicalResult RegisterExecuteOp(mlir::Operation* op, uint32_t op_key) { + if (op_key >= execute_ops_.size()) { + execute_ops_.resize(op_key + 1); + } + if (auto* register_op = execute_ops_[op_key]) { + if (register_op->getName() != op->getName() || + register_op->getAttrs() != op->getAttrs()) { + return op->emitError() << "Key " << op_key << " already registered."; + } + return mlir::success(); + } + execute_ops_[op_key] = op; + return mlir::success(); + } + + void ReplaceExecuteOp(int64_t key, mlir::Operation* op) { + execute_ops_[key] = op; + } + + llvm::ArrayRef GetExecuteOps() const { + return execute_ops_; + } + + private: + // Using a vector to keep fallback ops in order, and the key for a fallback op + // is its corresponding index here. + llvm::SmallVector execute_ops_; +}; + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_EXECUTE_OP_REGISTRY_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..fbe7f73e32d089b7fb91dac2b5c604e11612dfe9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.cc @@ -0,0 +1,175 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.h" + +#include + +#include "llvm/ADT/SmallVector.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +class FuseMlrtOpPass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FuseMlrtOpPass) + + private: + llvm::StringRef getArgument() const final { return "tf-mlrt-fuse"; } + + llvm::StringRef getDescription() const final { + return "Fuse consecutive mlrt ops of the same kind into one."; + } + + void runOnOperation() override; +}; + +void FuseGetResourceOps(mlir::OpBuilder& builder, mlir::Block& block) { + llvm::SmallVector get_resource_ops; + for (auto& op : llvm::make_early_inc_range(block)) { + if (auto get_resource_op = llvm::dyn_cast(&op)) { + get_resource_ops.push_back(get_resource_op); + } + } + + if (get_resource_ops.empty()) return; + + // The last op is always a return op, so it is guaranteed to process all + // groups of the candidate ops. + auto first_get = get_resource_ops.front(); + + builder.setInsertionPointAfter(first_get); + + llvm::SmallVector indices; + llvm::SmallVector result_types; + llvm::SmallVector old_values; + + indices.reserve(get_resource_ops.size()); + result_types.reserve(get_resource_ops.size()); + old_values.reserve(get_resource_ops.size()); + + for (auto op : get_resource_ops) { + auto indices_attr = op.getIndices(); + indices.append(indices_attr.begin(), indices_attr.end()); + result_types.append(op.result_type_begin(), op.result_type_end()); + old_values.append(op.result_begin(), op.result_end()); + } + + auto new_op = tf_mlrt::GetResourceOp::create( + builder, first_get.getLoc(), result_types, builder.getArrayAttr(indices)); + + for (auto [old_value, new_value] : + llvm::zip(old_values, new_op.getResults())) { + old_value.replaceAllUsesWith(new_value); + } + + for (auto get_resource_op : get_resource_ops) { + get_resource_op->erase(); + } +} + +template +void FuseAwaitOps(mlir::OpBuilder& builder, mlir::Block& block) { + llvm::SmallVector await_ops; + for (auto& op : llvm::make_early_inc_range(block)) { + if (auto await_op = llvm::dyn_cast(&op)) { + await_ops.push_back(await_op); + continue; + } + + // The last op is always a return op, so it is guaranteed to process all + // groups of the candidate ops. + if (await_ops.size() > 1) { + auto last_await = await_ops.back(); + + builder.setInsertionPointAfter(last_await); + + llvm::SmallVector futures; + futures.reserve(await_ops.size()); + for (auto op : await_ops) { + futures.push_back(op.getOperand()); + } + + llvm::SmallVector result_types; + if constexpr (!std::is_same_v) { + result_types.assign(futures.size(), builder.getType()); + } + + auto await_all = + AwaitAllOpType::create(builder, op.getLoc(), result_types, futures); + + if constexpr (!std::is_same_v) { + for (auto [await_op, new_value] : + llvm::zip(await_ops, await_all.getResults())) { + await_op.getResult().replaceAllUsesWith(new_value); + } + } + + for (auto await_op : await_ops) { + await_op->erase(); + } + } + + await_ops.clear(); + } +} + +void FusePromiseReturn(mlir::OpBuilder& builder, mlir::Block& block) { + auto* terminator = block.getTerminator(); + auto return_op = llvm::dyn_cast(terminator); + if (!return_op || return_op->getNumOperands() > 0) return; + + auto promise_op = + llvm::dyn_cast_or_null(return_op->getPrevNode()); + if (!promise_op) return; + + builder.setInsertionPointAfter(return_op); + tf_mlrt::PromiseReturnOp::create(builder, return_op->getLoc(), + promise_op->getResultTypes(), + promise_op->getOperands()); + return_op->erase(); + promise_op->erase(); +} + +void FuseMlrtOpPass::runOnOperation() { + auto func = getOperation(); + + mlir::OpBuilder builder(func); + + FuseAwaitOps( + builder, func.front()); + FuseAwaitOps( + builder, func.front()); + FuseAwaitOps(builder, func.front()); + FuseGetResourceOps(builder, func.front()); + FusePromiseReturn(builder, func.front()); +} + +} // namespace + +std::unique_ptr> +CreateFuseMlrtOpPass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..ac69d1f5ff670f10257630fd0a04393c03f48425 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.h @@ -0,0 +1,31 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_FUSE_MLRT_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_FUSE_MLRT_OPS_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +std::unique_ptr> CreateFuseMlrtOpPass(); + +} +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_FUSE_MLRT_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.cc new file mode 100644 index 0000000000000000000000000000000000000000..cb81ae644be6c7264a9dd9287116ca70a0db43ba --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.cc @@ -0,0 +1,129 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.h" + +#include +#include + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +class IfrtSetTpuHostAllocatorPass + : public mlir::PassWrapper> { + public: + IfrtSetTpuHostAllocatorPass() = default; + IfrtSetTpuHostAllocatorPass &operator=(const IfrtSetTpuHostAllocatorPass &) = + delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(IfrtSetTpuHostAllocatorPass) + + private: + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + llvm::StringRef getArgument() const final { + return "tf-mlrt-ifrt-set-tpu-host-allocator"; + } + + llvm::StringRef getDescription() const final { + return "Set input producer to IfrtCall to use Tpu Host Allocator"; + } + void runOnOperation() override { + mlir::func::FuncOp func = getOperation(); + mlir::OpBuilder builder(&getContext()); + + llvm::SmallDenseSet producers; + + auto process_call = [&](auto call) { + std::vector variable_arg_indices; + variable_arg_indices.reserve(call.getVariableArgIndices().size()); + for (auto variable_index_attr : call.getVariableArgIndices()) { + auto variable_index = + llvm::dyn_cast_or_null(variable_index_attr); + if (!variable_index) { + call->emitError() + << "Expect variable_arg_indices to be integer, but get " + << call.getVariableArgIndices(); + return mlir::WalkResult::interrupt(); + } + variable_arg_indices.push_back(variable_index.getInt()); + } + + int variable_index = 0; + for (int i = 0; i < call.getOperands().size(); ++i) { + if (variable_index < variable_arg_indices.size() && + i == variable_arg_indices[variable_index]) { + variable_index++; + continue; + } + producers.insert(call.getOperands()[i].getDefiningOp()); + } + return mlir::WalkResult::advance(); + }; + + mlir::WalkResult walk_result = func.walk( + [&](mlir::TF::IfrtCallOp call) { return process_call(call); }); + if (walk_result.wasInterrupted()) { + return signalPassFailure(); + } + + walk_result = func.walk( + [&](mlir::TF::AsyncIfrtCallOp call) { return process_call(call); }); + if (walk_result.wasInterrupted()) { + return signalPassFailure(); + } + + for (auto *def : producers) { + if (def && llvm::isa(def->getDialect())) { + def->setAttr(kTfMlrtCustomDevice, + builder.getStringAttr(kTpuHostDevice)); + } + } + } +}; + +} // namespace + +std::unique_ptr> +CreateIfrtSetTpuHostAllocatorPass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.h new file mode 100644 index 0000000000000000000000000000000000000000..f6e5910f27b48e30339182cdd450199b379655ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.h @@ -0,0 +1,34 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IFRT_SET_TPU_HOST_ALLOCATOR_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IFRT_SET_TPU_HOST_ALLOCATOR_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +// Creates a pass that set tpu input producers to use tpu host allocators. +std::unique_ptr> +CreateIfrtSetTpuHostAllocatorPass(); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IFRT_SET_TPU_HOST_ALLOCATOR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.cc new file mode 100644 index 0000000000000000000000000000000000000000..4ccda35345707f62531b0188721e50db409479d0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.cc @@ -0,0 +1,206 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.h" + +#include +#include +#include +#include + +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/utils/data_dumper_logger_config.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/compiler/mlir/tfrt/translate/import_model.h" +#include "tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/compiler/mlir/tfrt/utils/export.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/statusor.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/mlrt/attribute/attribute.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/bytecode.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tensorflow/core/util/debug_data_dumper.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { +// Setup the input pass manager to enable IR dumping after each pass. +// Note a side effect of this method is that multi threading will be disabled. +void EnablePassIRPrinting(mlir::PassManager& pm, + const std::string& dump_group_name, + llvm::StringRef module_name) { + // Print the whole module after each pass, which requires disabling + // multi-threading as well. + pm.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique<::tensorflow::DataDumperLoggerConfig>( + [module_name, dump_group_name](const std::string& pass_tag_name, + mlir::Operation* op) { + return DEBUG_DATA_DUMPER()->GetDumpFilename( + module_name.str(), dump_group_name, pass_tag_name); + }, + /*pass_prefix=*/"", + /*print_module_scope=*/true)); + pm.enableTiming(); +} +} // namespace + +absl::StatusOr ConvertTfMlirToBytecode( + const TfrtCompileOptions& options, tfrt_stub::FallbackState& fallback_state, + mlir::ModuleOp module, tfrt_stub::ModelRuntimeContext& model_context, + mlir::OwningOpRef* module_with_op_keys, + std::vector* added_xla_function_names) { + mlrt::bc::Buffer bytecode_buffer; + TF_RETURN_IF_ERROR(ConvertTfMlirToRuntimeExecutable( + options, module, + [&bytecode_buffer, &fallback_state, &model_context, + backend_compiler = options.backend_compiler, + module_with_op_keys](mlir::PassManager& pm, mlir::ModuleOp module, + const TfrtPipelineOptions& options) { + if (backend_compiler) { + if (auto* flib_def = model_context.function_library_definition()) { + // Copy the module before exporting as exporting to graph will + // transform the MLIR to TFG dialect. + mlir::OwningOpRef copy(module.clone()); + TF_RETURN_IF_ERROR( + ExportFunctionDefs(*copy, [flib_def](FunctionDef function_def) { + VLOG(1) << absl::StrCat( + "Exporting MLIR function as function_def: ", + // NOLINTNEXTLINE + function_def.DebugString()); + + // The TF MLIR compiler may change the function name. Then we + // need to retrieve the original name from the + // _original_func_name attribute. + auto iter = function_def.attr().find("_original_func_name"); + if (iter != function_def.attr().end()) { + function_def.mutable_signature()->set_name( + iter->second.s()); + } + + const auto& name = function_def.signature().name(); + if (flib_def->Contains(name)) { + TF_RETURN_IF_ERROR(flib_def->RemoveFunction(name)); + } + + return flib_def->AddFunctionDef(function_def); + })); + } + } + + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + + tensorflow::CreateTFInvariantOptimizationPipelineHelper(pm, options); + + pm.addPass(mlrt_compiler::CreateWhileToMapFnPass()); + // Remove unreachable private functions after map_fn conversion. + pm.addPass(mlir::createSymbolDCEPass()); + + // TODO(b/283481729): Add test to cover unused constants that do not + // cause op_key discontinuity + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlrt_compiler::CreateAssignOpKeyPass()); + + // Run passes until (including) AssignOpKeyPass. + if (VLOG_IS_ON(4)) { + EnablePassIRPrinting(pm, "mlrt_runtime_lowering_tf", + "mlrt_runtime_lowering_tf"); + } + + if (mlir::failed(pm.run(module))) { + return diag_handler.Combine(absl::InternalError( + "failed to finish passes before (including) assign op keys.")); + } + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tf_dialect_after_assign_op_key", + module); + } + // Save the module. + if (module_with_op_keys != nullptr) { + *module_with_op_keys = module.clone(); + } + // Clear passes already run. + pm.clear(); + // Create the remaining pipeline and run. + CreateTfToMlrtPipeline(pm, options, &fallback_state); + if (mlir::failed(pm.run(module))) { + return diag_handler.Combine(absl::InternalError( + "failed to lower TF Dialect to MLRT dialect.")); + } + // Generate bytecode. + mlrt::AttributeEncoderRegistry registry; + registry.Register("tf_mlrt", + &tensorflow::tf_mlrt::EncodeTensorflowAttribute); + auto statusor = mlrt::EmitExecutable(registry, module); + if (!statusor.ok()) return statusor.status(); + bytecode_buffer = std::move(*statusor); + return absl::OkStatus(); + }, + model_context, &fallback_state, added_xla_function_names)); + return bytecode_buffer; +} + +absl::StatusOr ConvertTfMlirWithOpKeysToBytecode( + const TfrtCompileOptions& options, + const tfrt_stub::FallbackState& fallback_state, + mlir::ModuleOp module_with_op_keys, + const tfrt_stub::CostRecorder& cost_recorder) { + mlir::StatusScopedDiagnosticHandler diag_handler( + module_with_op_keys.getContext()); + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tf_dialect_with_op_keys", + module_with_op_keys); + } + // Create the reconversion pipeline and run. + mlir::PassManager pm(module_with_op_keys.getContext()); + const auto pipeline_options = GetTfrtPipelineOptions(options); + CreateTfToMlrtPipeline(pm, *pipeline_options, &fallback_state, + &cost_recorder); + if (mlir::failed(pm.run(module_with_op_keys))) { + return diag_handler.Combine( + absl::InternalError("failed to lower TF Dialect to MLRT dialect.")); + } + // Generate bytecode. + mlrt::AttributeEncoderRegistry registry; + registry.Register("tf_mlrt", &tensorflow::tf_mlrt::EncodeTensorflowAttribute); + auto statusor = mlrt::EmitExecutable(registry, module_with_op_keys); + if (!statusor.ok()) return statusor.status(); + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tfrt_dialect_from_tf_dialect_with_op_keys", + module_with_op_keys); + } + return std::move(*statusor); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.h new file mode 100644 index 0000000000000000000000000000000000000000..249fd0ff1229b872fcedaed69d46198ebfa46b19 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/import_model.h @@ -0,0 +1,54 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IMPORT_MODEL_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IMPORT_MODEL_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/OwningOpRef.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/core/platform/statusor.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/bytecode.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" + +namespace tensorflow { +namespace mlrt_compiler { + +// Converts an MLIR `module` in TF dialect to MLRT's bytecode format. If +// `module_with_op_keys` is non-null, the intermediate module on which passes +// until (including) AssignOpKeyPass have run will be cloned to it. +// +// This is for initial conversion. +absl::StatusOr ConvertTfMlirToBytecode( + const TfrtCompileOptions& options, tfrt_stub::FallbackState& fallback_state, + mlir::ModuleOp module, tfrt_stub::ModelRuntimeContext& model_context, + mlir::OwningOpRef* module_with_op_keys = nullptr, + std::vector* added_xla_function_names = nullptr); + +// Converts an MLIR `module_with_op_keys` in TF dialect to MLRT's bytecode +// format, with op costs from `cost_recorder`. +// +// This is for re-conversion. +absl::StatusOr ConvertTfMlirWithOpKeysToBytecode( + const TfrtCompileOptions& options, + const tfrt_stub::FallbackState& fallback_state, + mlir::ModuleOp module_with_op_keys, + const tfrt_stub::CostRecorder& cost_recorder); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_IMPORT_MODEL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..1fd763b9312225a84c3e1bd23e670612002fca38 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h @@ -0,0 +1,27 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_MLRT_DEVICE_CONSTANTS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_MLRT_DEVICE_CONSTANTS_H_ + +namespace tensorflow { +namespace mlrt_compiler { + +inline constexpr char kTfMlrtCustomDevice[] = "tf_mlrt.custom_device"; +inline constexpr char kTpuHostDevice[] = "tpu_host_device"; + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_MLRT_DEVICE_CONSTANTS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.cc new file mode 100644 index 0000000000000000000000000000000000000000..34544d35da312d71b35b5526d144c2d78b9d3c84 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.cc @@ -0,0 +1,835 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.h" + +#include +#include +#include +#include + +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SetVector.h" +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h" +#include "tensorflow/compiler/mlir/tfrt/constants.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tfrt/compiler/stream_analysis.h" // from @tf_runtime + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +using tensorflow::tfrt_compiler::CostAnalysis; +using tfrt::compiler::Stream; +using tfrt::compiler::StreamAnalysis; + +std::string GetStreamFunctionName(absl::string_view func_name, + const Stream& stream) { + return absl::StrCat(func_name, "_stream_", stream.id()); +} + +bool IsConstant(mlir::Operation* op) { + return op && llvm::isa(op); +} + +// StreamInfo is a bookkeeping for inputs, futures, and promises for a stream. +struct StreamInfo { + const Stream* parent = nullptr; + + // The values that are produced by constant ops. Instead of using + // promise/await to pass these values between streams, we can just copying + // these ops to the streams that use these constants. + llvm::SetVector constants; + // The values that are the inputs to the stream. + llvm::SetVector inputs; + // The values that will be the futures to the stream. + llvm::SetVector futures; + // The values that will be the control futures (i.e. futures with no data) to + // the stream. + llvm::SetVector control_futures; + // The values that will be the promises to the stream. + llvm::SetVector promises; + // The values that will be the control promises (i.e., promises with no data) + // to the stream. + llvm::SetVector control_promises; + // The values that are defined by the operations in the stream. Note that all + // values in `futures` will also be in `results`. + llvm::DenseSet results; + + bool contains_only_constants = true; + + bool IsRoot() const { return parent == nullptr; } +}; + +// Preprocess the block to produce StreamInfo for every stream. +llvm::DenseMap PreprocessStreamInfo( + mlir::Block& block, + const llvm::DenseMap>& + control_predecessors, + const StreamAnalysis& stream_analysis) { + llvm::DenseMap stream_map; + + // All values that will be promises in the block. + llvm::DenseSet promises; + + // All operations that will be control promises in the block. + llvm::DenseSet control_promises; + + // Keep track of all available values and controls as we traverse the stream + // tree in depth-first order. + llvm::DenseSet available_values; + llvm::DenseSet available_controls; + + struct Entry { + explicit Entry(const Stream* stream) : stream(stream) {} + + const Stream* stream = nullptr; + + // Keep track of the next operation to be processed. If all operations are + // processed, we can pop this stream from the DFS stack. + int op_idx = 0; + }; + + std::vector stack; + stack.reserve(stream_analysis.GetNumStreams()); + + // We first push the entry for the root stream. + const auto& root_stream = stream_analysis.GetRootStream(); + auto& root_stream_info = stream_map[&root_stream]; + available_values.insert(block.getArguments().begin(), + block.getArguments().end()); + root_stream_info.results.insert(block.getArguments().begin(), + block.getArguments().end()); + stack.push_back(Entry(&root_stream)); + + // The root stream's first operation a dummy operation that defines all block + // arguments. + for (auto* child_stream : root_stream.GetChildStreamsForRootOp()) { + stream_map[child_stream].parent = &root_stream; + stack.push_back(Entry(child_stream)); + } + + // The first DFS traveral populates inputs and futures for every stream but + // not promises. We only know whether a value definition is a promise only + // after traversing all streams, so it is not possible to know it in the first + // pass. + while (!stack.empty()) { + auto& [stream, op_idx] = stack.back(); + auto& stream_info = stream_map[stream]; + + auto ops = stream->ops(); + + // If we finish processing all operations in the stream, we can pop this + // stream, as well as the values defined by its operations. + if (op_idx == ops.size()) { + for (auto* op : stream->ops()) { + // Erase the values and controls produced by the current stream. + for (auto result : op->getResults()) { + available_values.erase(result); + } + available_controls.erase(op); + } + // Futures and control futures will also be available, so we erase them as + // well. + for (auto future : stream_info.futures) { + available_values.erase(future); + } + for (auto* control_future : stream_info.control_futures) { + available_controls.erase(control_future); + } + + if (!stream_info.IsRoot()) { + // Merge inputs, futures, and promises into the parent stream, as they + // will be passed down from the root in the output program. + DCHECK_GT(stream_map.count(stream_info.parent), 0); + auto& parent_info = stream_map[stream_info.parent]; + + for (const auto& input : stream_info.inputs) { + DCHECK(available_values.contains(input)); + if (!parent_info.results.contains(input)) { + // An input in the current stream will be an input in the parent + // stream only if it is not a result in the parent stream. + parent_info.inputs.insert(input); + } + } + + for (auto future : stream_info.futures) { + DCHECK(!available_values.contains(future)); + parent_info.futures.insert(future); + } + for (auto* control_future : stream_info.control_futures) { + DCHECK(!available_controls.contains(control_future)); + parent_info.control_futures.insert(control_future); + } + } + + // Update the global promise set. + promises.insert(stream_info.futures.begin(), stream_info.futures.end()); + control_promises.insert(stream_info.control_futures.begin(), + stream_info.control_futures.end()); + + stack.pop_back(); + continue; + } + + // We process the operations one by one. If the operation has child streams, + // we process the child streams first before continuing to the next + // operation. + bool has_child_streams = false; + for (; op_idx < ops.size() && !has_child_streams; ++op_idx) { + auto* op = ops[op_idx]; + + stream_info.contains_only_constants &= IsConstant(op); + + // Check every operand to see whether it is a future or input. + for (mlir::Value operand : op->getOperands()) { + // If the value is defined in the current stream, nothing needs to be + // done. + if (!stream_info.results.contains(operand)) { + if (available_values.insert(operand).second) { + // If the operand is not available in the current stream or any + // parent stream, it will be a future and then become a result. + if (IsConstant(operand.getDefiningOp())) { + stream_info.constants.insert(operand); + } else { + stream_info.futures.insert(operand); + } + stream_info.results.insert(operand); + } else { + // If the operand is not available in the current stream but + // available in the parent stream, it is an input. + if (IsConstant(operand.getDefiningOp())) { + stream_info.constants.insert(operand); + } else { + stream_info.inputs.insert(operand); + } + } + } + } + + // Insert mlrt.await_control if this op has control deps on other ops. + if (auto ctrl_iter = control_predecessors.find(op); + ctrl_iter != control_predecessors.end()) { + const auto& ctrl_deps = ctrl_iter->second; + + for (mlir::Operation* control_dep : ctrl_deps) { + if (available_controls.insert(control_dep).second) { + // If the control is not already available, it will be a control + // future and then become available. + stream_info.control_futures.insert(control_dep); + } + } + } + + // Update results of this operations. + for (mlir::Value result : op->getResults()) { + available_values.insert(result); + stream_info.results.insert(result); + } + + // Update this op as an available control. + available_controls.insert(op); + + // Pause processing the current stream to process the child streams first. + const auto& child_streams = stream->GetChildStreams(op); + has_child_streams = !child_streams.empty(); + for (auto* child_stream : child_streams) { + stream_map[child_stream].parent = stream; + stack.push_back(Entry(child_stream)); + } + } + } + + // The second pass populates promises for each stream. We also need to merge + // promises in a child to its parent stream. We can do this by traversing the + // operation in reverse program order. + for (auto& op : llvm::reverse(block)) { + const auto& stream = stream_analysis.GetStream(&op); + auto& stream_info = stream_map[&stream]; + + for (mlir::Value result : op.getResults()) { + if (promises.contains(result)) { + stream_info.promises.insert(result); + } + } + + if (control_promises.contains(&op)) { + stream_info.control_promises.insert(&op); + } + + for (const auto* child_stream : stream.GetChildStreams(&op)) { + const auto& child_info = stream_map[child_stream]; + + stream_info.promises.insert(child_info.promises.begin(), + child_info.promises.end()); + stream_info.control_promises.insert(child_info.control_promises.begin(), + child_info.control_promises.end()); + } + } + + // Special handling for the dummy operation in the root. + auto& root_info = stream_map[&root_stream]; + for (const auto* child_stream : root_stream.GetChildStreamsForRootOp()) { + const auto& child_info = stream_map[child_stream]; + + root_info.promises.insert(child_info.promises.begin(), + child_info.promises.end()); + root_info.control_promises.insert(child_info.control_promises.begin(), + child_info.control_promises.end()); + } + + return stream_map; +} + +// A custom struct that groups mappings for values, futures and promises for a +// stream during creating the corresponding stream function. +struct Mapping { + // This is the mappings for the SSA values used in the original and new + // operations. + mlir::IRMapping value_mapping; + + // Maps the original tensor value that will be a future to the corresponding + // !mlrt.future value. + mlir::IRMapping future_mapping; + + // Maps the original tensor value that will be a promise to the corresponding + // !mlrt.promise value. + mlir::IRMapping promise_mapping; + + // In addition to value mappings, we also need mappings for input control + // dependencies to the corresponding !mlrt.future and !mlrt.promise values. + llvm::DenseMap future_control_mapping; + llvm::DenseMap promise_control_mapping; +}; + +mlrt::compiler::AsyncOp CreateAsyncOp( + mlir::OpBuilder& builder, absl::string_view function_name, + const llvm::DenseMap& stream_map, + const Stream& stream, const Mapping& mapping, mlir::Location loc) { + auto iter = stream_map.find(&stream); + DCHECK(iter != stream_map.end()); + const auto& stream_info = iter->second; + + if (stream_info.contains_only_constants) return nullptr; + + const auto& [value_mapping, future_mapping, promise_mapping, + future_control_mapping, promise_control_mapping] = mapping; + + llvm::SmallVector async_operands; + + for (auto input : stream_info.inputs) { + async_operands.push_back(value_mapping.lookup(input)); + DCHECK(async_operands.back()); + } + + for (auto future : stream_info.futures) { + async_operands.push_back(future_mapping.lookup(future)); + DCHECK(async_operands.back()); + } + + for (auto* control_future : stream_info.control_futures) { + DCHECK_GT(future_control_mapping.count(control_future), 0); + async_operands.push_back(future_control_mapping.lookup(control_future)); + DCHECK(async_operands.back()); + } + + for (auto promise : stream_info.promises) { + async_operands.push_back(promise_mapping.lookup(promise)); + DCHECK(async_operands.back()); + } + + for (auto* control_promise : stream_info.control_promises) { + DCHECK_GT(promise_control_mapping.count(control_promise), 0); + async_operands.push_back(promise_control_mapping.lookup(control_promise)); + DCHECK(async_operands.back()); + } + + return mlrt::compiler::AsyncOp::create( + builder, loc, builder.getType(), + async_operands, + mlir::SymbolRefAttr::get(builder.getContext(), + GetStreamFunctionName(function_name, stream))); +} + +mlir::func::FuncOp CreateStreamFunction( + mlir::OpBuilder& builder, Mapping& mapping, absl::string_view name, + const Stream& stream, const StreamInfo& stream_info, mlir::Location loc) { + if (stream_info.contains_only_constants) return nullptr; + + auto& [value_mapping, future_mapping, promise_mapping, future_control_mapping, + promise_control_mapping] = mapping; + + llvm::SmallVector arg_types; + for (mlir::Value input : stream_info.inputs) { + arg_types.push_back(input.getType()); + } + + arg_types.append( + stream_info.futures.size() + stream_info.control_futures.size(), + builder.getType()); + arg_types.append( + stream_info.promises.size() + stream_info.control_promises.size(), + builder.getType()); + + // The stream function has no result. + auto func_type = builder.getFunctionType(arg_types, /*results=*/{}); + + auto func = mlir::func::FuncOp::create( + builder, loc, GetStreamFunctionName(name, stream), func_type); + func.setVisibility(mlir::func::FuncOp::Visibility::Private); + + // Populate the body of the stream function by copying over the operations + // in the stream. + auto* new_block = func.addEntryBlock(); + + // Replace inputs with the function arguments. + for (int i = 0; i < stream_info.inputs.size(); ++i) { + value_mapping.map(stream_info.inputs[i], new_block->getArgument(i)); + } + + // Maps the original tensor value that will be a future or a promise to + // the corresponding !mlrt.future or !mlrt.promise value. + size_t start = stream_info.inputs.size(); + for (int i = 0; i < stream_info.futures.size(); ++i) { + future_mapping.map(stream_info.futures[i], + new_block->getArgument(i + start)); + } + + start += stream_info.futures.size(); + for (int i = 0; i < stream_info.control_futures.size(); ++i) { + future_control_mapping[stream_info.control_futures[i]] = + new_block->getArgument(i + start); + } + + start += stream_info.control_futures.size(); + for (int i = 0; i < stream_info.promises.size(); ++i) { + promise_mapping.map(stream_info.promises[i], + new_block->getArgument(i + start)); + } + + start += stream_info.promises.size(); + for (int i = 0; i < stream_info.control_promises.size(); ++i) { + promise_control_mapping[stream_info.control_promises[i]] = + new_block->getArgument(i + start); + } + + return func; +} + +void CreateAllocateFuturesOp(mlir::OpBuilder& builder, Mapping& mapping, + const StreamInfo& stream_info, + mlir::Location loc) { + auto& [value_mapping, future_mapping, promise_mapping, future_control_mapping, + promise_control_mapping] = mapping; + + DCHECK_EQ(stream_info.futures.size(), stream_info.promises.size()); + + llvm::SmallVector promise_types( + stream_info.promises.size(), + builder.getType()); + llvm::SmallVector future_types( + stream_info.futures.size(), + builder.getType()); + + if (!stream_info.futures.empty()) { + auto allocate_futures = tf_mlrt::AllocateFuturesOp::create( + builder, loc, promise_types, future_types, stream_info.futures.size()); + for (int i = 0; i < stream_info.futures.size(); ++i) { + future_mapping.map(stream_info.futures[i], + allocate_futures.getFutures()[i]); + } + + for (int i = 0; i < stream_info.futures.size(); ++i) { + // Use the original values in `futures` to make sure futures[i] shares the + // state with promises[i]. + DCHECK(stream_info.promises.contains(stream_info.futures[i])); + promise_mapping.map(stream_info.futures[i], + allocate_futures.getPromises()[i]); + } + } + + DCHECK_EQ(stream_info.control_futures.size(), + stream_info.control_promises.size()); + if (!stream_info.control_futures.empty()) { + promise_types.resize(stream_info.control_promises.size(), + builder.getType()); + future_types.resize(stream_info.control_futures.size(), + builder.getType()); + + auto allocate_control_futures = + mlrt::compiler::AllocateControlFuturesOp::create( + builder, loc, promise_types, future_types, + stream_info.control_futures.size()); + for (int i = 0; i < stream_info.control_futures.size(); ++i) { + future_control_mapping[stream_info.control_futures[i]] = + allocate_control_futures.getFutures()[i]; + } + for (int i = 0; i < stream_info.control_futures.size(); ++i) { + // Use the original operations in `control_futures` to make sure + // control_futures[i] shares the state with control_promises[i]. + DCHECK(stream_info.control_promises.contains( + stream_info.control_futures[i])); + promise_control_mapping[stream_info.control_futures[i]] = + allocate_control_futures.getPromises()[i]; + } + } +} + +class TensorflowCostModel : public StreamAnalysis::CostModelInterface { + public: + explicit TensorflowCostModel(CostAnalysis* cost_analysis) + : cost_analysis_(*cost_analysis) {} + + std::optional GetOperationCost(mlir::Operation* op) const override { + return cost_analysis_.GetCost(op); + } + + private: + const CostAnalysis& cost_analysis_; +}; + +bool SkipControlDep(mlir::Operation* op) { + // TODO(chky): Consider define side effects more properly for these ops. + return llvm::isa( + op); +} + +void ParallelizeBlock( + absl::string_view name, mlir::Block& block, + const mlir::TF::SideEffectAnalysis::Info& side_effect_analysis, + const tfrt_stub::CostRecorder* cost_recorder) { + // First, we use SideEffectAnalysis to find out control predecessors for each + // operation. We use this map later to insert control futures. + llvm::DenseMap> + control_predecessors; + for (auto& op : block) { + auto& deps = control_predecessors[&op]; + for (auto* dep : side_effect_analysis.DirectControlPredecessors(&op)) { + // If we skip the control deps of `op`, then we need to use the control + // deps of these control deps instead. + if (SkipControlDep(dep)) { + for (auto* d : control_predecessors[dep]) { + DCHECK(!SkipControlDep(d)); + deps.insert(d); + } + } else { + deps.insert(dep); + } + } + } + + // Remove skipped control deps. + for (auto& op : block) { + if (SkipControlDep(&op)) { + control_predecessors.erase(&op); + } + } + + // Perform stream analysis. + CostAnalysis cost_analysis( + llvm::cast(block.getParentOp()), cost_recorder); + TensorflowCostModel cost_model(&cost_analysis); + StreamAnalysis stream_analysis(block, &cost_model); + + // Preprocess all streams to gather StreamInfos for all streams, without + // modifying the program. + llvm::DenseMap stream_map = + PreprocessStreamInfo(block, control_predecessors, stream_analysis); + + // Then we perform a DFS traversal to create stream functions and insert async + // operations. + std::vector stack; + stack.reserve(stream_analysis.GetNumStreams()); + + const auto& root_stream = stream_analysis.GetRootStream(); + stack.push_back(&root_stream); + + llvm::SmallVector to_remove; + + mlir::OpBuilder builder(block.getParentOp()); + + while (!stack.empty()) { + const auto* stream = stack.back(); + stack.pop_back(); + DCHECK(stream); + + DCHECK_GT(stream_map.count(stream), 0); + const auto& stream_info = stream_map[stream]; + + Mapping mapping; + auto& [value_mapping, future_mapping, promise_mapping, + future_control_mapping, promise_control_mapping] = mapping; + + // `async_handles` keeps the !mlrt.async_handle created in the stream. A + // mlrt.await_handle op will be inserted at the end of the stream function + // for each async handle. + llvm::SmallVector async_handles; + + mlir::func::FuncOp stream_func; + if (!stream_info.IsRoot()) { + // If it is not a root stream, we need to create a new function for this + // stream. And futures and promises are also passed as parameters. For the + // root stream, futures and promises are allocated in the body. + + // Insert the stream function before the original function. + builder.setInsertionPoint(block.getParentOp()); + + stream_func = + CreateStreamFunction(builder, mapping, name, *stream, stream_info, + block.getParentOp()->getLoc()); + + if (stream_func) { + // Set the insertion point to the start of the new block in the + // function. + builder.setInsertionPointToStart(&stream_func.front()); + } + } else { + stream_func = llvm::cast(block.getParentOp()); + + DCHECK_EQ(stream, &root_stream); + // If it is the root stream, we insert new operations in the original + // function. And we need to allocate all the futures used here. + builder.setInsertionPointToStart(&block); + + // The block arguments of the root stream are in the `results`. There will + // be no additional inputs in `inputs`. + DCHECK(stream_info.inputs.empty()); + + // Put the original arguments in the mapping as they are not changed. + for (auto arg : block.getArguments()) { + value_mapping.map(arg, arg); + } + + // Insert a tf_mlrt.allocate_futures op to allocate all futures used. + CreateAllocateFuturesOp(builder, mapping, stream_info, + block.getParentOp()->getLoc()); + + // Lastly for the root stream, we need to handle the dummy op that defines + // the arguments. + for (const auto* child_stream : stream->GetChildStreamsForRootOp()) { + stack.push_back(child_stream); + if (auto async = + CreateAsyncOp(builder, name, stream_map, *child_stream, mapping, + block.getParentOp()->getLoc())) { + async_handles.push_back(async); + } + } + } + + for (auto* op : stream->ops()) { + to_remove.push_back(op); + } + + // Skip empty streams. + if (!stream_func) continue; + + mlir::Operation* return_op = nullptr; + + // Cloning the operations in the stream. If the operand is a future, a + // tf_mlrt.Await op will be inserted. If the result is a promise, a + // tf_mlrt.Promise will be inserted. Similar to control futures and control + // promises. + for (auto* op : stream->ops()) { + // Clone the current op into the function of this stream, using the + // new operands, which can be futures. + for (mlir::Value operand : op->getOperands()) { + if (stream_info.constants.contains(operand) && + !value_mapping.contains(operand)) { + builder.clone(*operand.getDefiningOp(), value_mapping); + } else if (stream_info.futures.contains(operand) && + !value_mapping.contains(operand)) { + // Insert Await op if it is a future. + auto future_value = tf_mlrt::TFAwaitOp::create( + builder, op->getLoc(), operand.getType(), + future_mapping.lookup(operand)); + + // Now this future is available in the current stream, so it can be a + // normal value. + value_mapping.map(operand, future_value); + } + } + + if (auto ctrl_iter = control_predecessors.find(op); + ctrl_iter != control_predecessors.end()) { + const auto& ctrl_deps = ctrl_iter->second; + + for (mlir::Operation* control_dep : ctrl_deps) { + // This control may be available in the ancestors or in a previous + // AwaitControl, we only insert a new AwaitControl if it is not. + if (stream_info.control_futures.contains(control_dep)) { + if (auto iter = future_control_mapping.find(control_dep); + iter != future_control_mapping.end()) { + mlrt::compiler::AwaitControlOp::create( + builder, control_dep->getLoc(), iter->second); + + // Now we no longer need this control dep in this stream. + future_control_mapping.erase(iter); + } + } + } + } + + // Clone the op using the value mapping that includes values from futures. + auto* new_op = builder.clone(*op, value_mapping); + + // TODO(chky): Ensure the original return op is in the root stream. This + // is currently an implicit guarantee in stream analysis. + if (llvm::isa(op)) { + DCHECK(stream_info.IsRoot()) << name << " " << stream->id(); + return_op = new_op; + } + + for (mlir::Value result : op->getResults()) { + if (stream_info.promises.contains(result)) { + // Insert Promise op if the result is a promise. + tf_mlrt::TFPromiseOp::create(builder, op->getLoc(), + promise_mapping.lookup(result), + value_mapping.lookup(result)); + } + } + + if (stream_info.control_promises.contains(op)) { + // Insert Promise op if this op produce a control dependency to ops in + // other streams. + mlrt::compiler::PromiseControlOp::create(builder, op->getLoc(), + promise_control_mapping[op]); + } + + // If this op has child streams, insert mlrt.async ops. + for (auto* child_stream : stream->GetChildStreams(op)) { + stack.push_back(child_stream); + if (auto async = CreateAsyncOp(builder, name, stream_map, *child_stream, + mapping, op->getLoc())) { + async_handles.push_back(async); + } + } + } + + // Create the return op for non-root streams. + // + // TODO(chky): Ensure the original return op is in the root stream. This is + // currently an implicit guarantee in stream analysis. + if (!return_op) { + DCHECK(!stream_info.IsRoot()) << name << " " << stream->id(); + return_op = + mlir::func::ReturnOp::create(builder, block.getParentOp()->getLoc()); + } + + // We need to wait for async executions at the end of the stream function, + // in order to manage resource lifetime and handle errors properly. These + // mlrt.await_handle ops are inserted before the return op. + builder.setInsertionPoint(return_op); + for (auto handle : async_handles) { + mlrt::compiler::AwaitHandleOp::create( + builder, block.getParentOp()->getLoc(), handle); + } + } + + // Remove the operations in the original block. + for (auto* op : llvm::reverse(to_remove)) { + op->dropAllDefinedValueUses(); + op->erase(); + } +} + +class ParallelizationPass + : public mlir::PassWrapper> { + public: + ParallelizationPass() = default; + ParallelizationPass(uint64_t cost_threshold, + bool merge_inter_dependent_streams, + const tfrt_stub::CostRecorder* cost_recorder) { + cost_threshold_ = cost_threshold; + merge_inter_dependent_streams_ = merge_inter_dependent_streams; + cost_recorder_ = cost_recorder; + } + ParallelizationPass(const ParallelizationPass&) {} + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ParallelizationPass) + + private: + void getDependentDialects(mlir::DialectRegistry& registry) const override { + registry.insert(); + registry.insert(); + } + + llvm::StringRef getArgument() const final { + return "tf-mlrt-parallelization"; + } + + llvm::StringRef getDescription() const final { + return "Parallelize tf graphs by inserting mlrt async operations."; + } + + void runOnOperation() override { + auto module = getOperation(); + + mlir::Builder builder(module); + module->setAttr("tfrt.cost_threshold", + builder.getI64IntegerAttr(cost_threshold_)); + module->setAttr("tfrt.merge_inter_dependent_streams", + builder.getBoolAttr(merge_inter_dependent_streams_)); + + mlir::TF::SideEffectAnalysis side_effect_analysis(module); + + for (auto func_op : + llvm::make_early_inc_range(module.getOps())) { + ParallelizeBlock(func_op.getSymName(), func_op.front(), + side_effect_analysis.GetAnalysisForFunc(func_op), + cost_recorder_); + } + } + + Option cost_threshold_{ + *this, "tfrt-cost-threshold", + llvm::cl::desc("If a sequence of operations has a cost lower than the " + "cost-threshold, the sequence will be executed as a block " + "in the same thread."), + llvm::cl::init(1)}; + Option merge_inter_dependent_streams_{ + *this, "tfrt-merge-inter-dependent-streams", + llvm::cl::desc("If true, streams with inter data depenedencies will be " + "preferred to be merged for inline execution."), + llvm::cl::init(false)}; + const tfrt_stub::CostRecorder* cost_recorder_ = nullptr; +}; + +} // namespace + +std::unique_ptr> CreateParallelizationPass( + uint64_t cost_threshold, bool merge_inter_dependent_streams, + const tfrt_stub::CostRecorder* cost_recorder) { + return std::make_unique( + cost_threshold, merge_inter_dependent_streams, cost_recorder); +} + +std::unique_ptr> +CreateParallelizationPass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.h new file mode 100644 index 0000000000000000000000000000000000000000..61a5a746e0ac77ff81ce4ebeae549f1d85fb278f --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.h @@ -0,0 +1,37 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PARALLELIZATION_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PARALLELIZATION_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" + +namespace tensorflow { +namespace mlrt_compiler { + +std::unique_ptr> CreateParallelizationPass( + uint64_t cost_threshold, bool merge_inter_dependent_streams, + const tfrt_stub::CostRecorder* cost_recorder = nullptr); + +std::unique_ptr> +CreateParallelizationPass(); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PARALLELIZATION_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6ce9322a2efc53d39a37192e53272fa4f9ccc88 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.cc @@ -0,0 +1,83 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h" + +#include "absl/log/check.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/assign_op_key.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/async_while.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/fuse_mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/ifrt_set_tpu_host_allocator.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/parallelization.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" + +namespace tensorflow { +namespace mlrt_compiler { + +void RegisterMlrtPasses() { + mlir::registerPass([]() { return CreateAssignOpKeyPass(); }); + mlir::registerPass([]() { return CreateAsyncWhilePass(); }); + mlir::registerPass([]() { return CreateParallelizationPass(); }); + mlir::registerPass([]() { return CreateWhileToMapFnPass(); }); + mlir::registerPass([]() { return CreateRewriteIfrtLoadVariablePass(); }); + mlir::registerPass([]() { return CreateIfrtSetTpuHostAllocatorPass(); }); + mlir::registerPass( + []() { return CreateTfToMlrtPreParallelizationConversionPass({}); }); + mlir::registerPass([]() { return CreateTfToMlrtConversionPass({}); }); + mlir::registerPass([]() { return CreateFuseMlrtOpPass(); }); +} + +void CreateTfToMlrtPipeline(mlir::OpPassManager &pm, + const TfrtPipelineOptions &options, + const tfrt_stub::FallbackState *fallback_state, + const tfrt_stub::CostRecorder *cost_recorder) { + pm.addPass( + mlrt_compiler::CreateTfToMlrtPreParallelizationConversionPass(options)); + + if (options.use_tpu_host_allocator_for_inputs) { + pm.addNestedPass( + mlrt_compiler::CreateIfrtSetTpuHostAllocatorPass()); + } + pm.addPass(mlrt_compiler::CreateRewriteIfrtLoadVariablePass()); + + if (options.enable_while_parallel_iterations) { + pm.addPass(mlrt_compiler::CreateAsyncWhilePass()); + } + + pm.addPass(mlrt_compiler::CreateParallelizationPass( + options.cost_threshold, options.merge_inter_dependent_streams, + cost_recorder)); + + DCHECK(fallback_state); + pm.addPass( + mlrt_compiler::CreateTfToMlrtConversionPass(options, fallback_state)); + + // Perform optimizations in the lowered MLIR. + pm.addNestedPass(mlrt_compiler::CreateFuseMlrtOpPass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass(mlir::createCSEPass()); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h new file mode 100644 index 0000000000000000000000000000000000000000..c34f0ddebd84fed055f63b28d20d7610cebd345c --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/passes.h @@ -0,0 +1,38 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PASSES_H_ + +#include "mlir/Pass/PassOptions.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" + +namespace tensorflow { +namespace mlrt_compiler { + +void RegisterMlrtPasses(); + +// Creates a pipeline of passes that lowers MLIR TF dialect to MLRT dialects. +// The op costs from `cost_recorder` (if non-null) are used for Stream Analysis. +void CreateTfToMlrtPipeline( + mlir::OpPassManager& pm, const TfrtPipelineOptions& options, + const tfrt_stub::FallbackState* fallback_state, + const tfrt_stub::CostRecorder* cost_recorder = nullptr); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.cc new file mode 100644 index 0000000000000000000000000000000000000000..7f53ea70477c0f3bce440d0390b8eb7513659805 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.cc @@ -0,0 +1,117 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.h" + +#include +#include + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +class RewriteIfrtLoadVariablePass + : public mlir::PassWrapper> { + public: + RewriteIfrtLoadVariablePass() = default; + RewriteIfrtLoadVariablePass &operator=(const RewriteIfrtLoadVariablePass &) = + delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RewriteIfrtLoadVariablePass) + + private: + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + llvm::StringRef getArgument() const final { + return "tf-mlrt-rewrite-ifrt-load-variable"; + } + + llvm::StringRef getDescription() const final { + return "Convert tf.IfrtLoadVariable to tf_mlrt.TFIfrtLoadVariable"; + } + + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::OpBuilder builder(module); + + module->walk([&](mlir::TF::IfrtLoadVariableOp load_variable_op) { + builder.setInsertionPoint(load_variable_op); + + std::vector result_types; + result_types.push_back(load_variable_op.getArrayKey().getType()); + result_types.push_back(builder.getType()); + auto mlrt_load_variable_op = tf_mlrt::TFIfrtLoadVariableOp::create( + builder, load_variable_op->getLoc(), result_types, + load_variable_op->getOperands(), load_variable_op->getAttrs()); + tf_mlrt::TFAwaitOp await_op; + for (auto user : llvm::make_early_inc_range( + load_variable_op.getTensorFuture().getUsers())) { + // Materialize the future for the first use. Reuse it for the rest of + // the uses. + if (!await_op) { + builder.setInsertionPoint(user); + await_op = tf_mlrt::TFAwaitOp::create( + builder, user->getLoc(), + load_variable_op.getTensorFuture().getType(), + mlrt_load_variable_op.getTensorFuture()); + } else { + if (user->isBeforeInBlock(await_op)) { + await_op->moveBefore(user); + } + } + user->replaceUsesOfWith(load_variable_op.getTensorFuture(), + await_op.getResult()); + } + + for (auto user : llvm::make_early_inc_range( + load_variable_op.getArrayKey().getUsers())) { + user->replaceUsesOfWith(load_variable_op.getArrayKey(), + mlrt_load_variable_op.getArrayKey()); + } + + load_variable_op->erase(); + }); + } +}; + +} // namespace + +std::unique_ptr> +CreateRewriteIfrtLoadVariablePass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.h new file mode 100644 index 0000000000000000000000000000000000000000..e9a546cf30fe41a9571d8dc023ba6c1d67891b64 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/rewrite_ifrt_load_variable.h @@ -0,0 +1,36 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_REWRITE_IFRT_LOAD_VARIABLE_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_REWRITE_IFRT_LOAD_VARIABLE_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +// Creates a pass that converts tf.IfrtLoadVariableOp to +// tf_mlrt.TFIfrtLoadVariableOp and inserts tf_mlrt.Await on the returned future +// from tf_mlrt.TFIfrtLoadVariableOp if it is used by CPU ops. +std::unique_ptr> +CreateRewriteIfrtLoadVariablePass(); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_REWRITE_IFRT_LOAD_VARIABLE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.cc new file mode 100644 index 0000000000000000000000000000000000000000..9bfcecea17caf05de707dc361aa45f2be19acd15 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.cc @@ -0,0 +1,1410 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.h" + +#include + +#include +#include +#include +#include +#include + +#include "google/protobuf/text_format.h" +#include "absl/log/log.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Dialect/Func/Transforms/FuncConversions.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h.inc" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/translate_utils.h" +#include "tensorflow/compiler/mlir/tfrt/constants.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" +#include "xla/tsl/platform/status.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/fallback/op_kernel_runner_cache.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +constexpr char kXlaLaunchOp[] = "XlaLaunch"; + +mlir::Value CreateCustomDevice(mlir::Location loc, llvm::StringRef device_name, + mlir::ConversionPatternRewriter &rewriter) { + if (device_name == kTpuHostDevice) { + return tf_mlrt_tpu::GetTpuHostDeviceOp::create( + rewriter, loc, rewriter.getType()); + } + + return nullptr; +} + +class FuncOpSignatureConversion final + : public mlir::OpConversionPattern { + public: + explicit FuncOpSignatureConversion( + mlir::MLIRContext *context, mlir::TypeConverter *type_converter, + const llvm::DenseMap> + *function_call_site_input_types) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter), + function_call_site_input_types_(*function_call_site_input_types) {} + + mlir::LogicalResult matchAndRewrite( + mlir::func::FuncOp func_op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + auto it = function_call_site_input_types_.find(func_op.getName()); + if (it == function_call_site_input_types_.end()) { + return mlir::failure(); + } + const llvm::SmallVector &call_site_input_types = it->second; + + mlir::FunctionType func_type = func_op.getFunctionType(); + DCHECK_EQ(func_type.getNumInputs(), call_site_input_types.size()); + + mlir::TypeConverter::SignatureConversion converted_signature( + func_type.getNumInputs()); + for (const auto &[index, value] : llvm::enumerate(call_site_input_types)) { + converted_signature.addInputs(index, value); + } + + // Update the function signature in-place. + rewriter.modifyOpInPlace(func_op, [&] { + func_op.setType(mlir::FunctionType::get( + func_op.getContext(), converted_signature.getConvertedTypes(), + func_type.getResults())); + }); + + // Update the entry block + if (rewriter.applySignatureConversion(&func_op.getBody().front(), + converted_signature, + &type_converter_) == nullptr) { + return mlir::failure(); + } + + return mlir::success(); + } + + private: + mlir::TypeConverter &type_converter_; + const llvm::DenseMap> + &function_call_site_input_types_; +}; + +// Convert tf_mlrt::AsyncWhile's signature to tf_mlrt::TFTensorType +class TFAsyncWhileOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFAsyncWhileOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + auto new_op = tf_mlrt::AsyncWhileOp::create( + rewriter, op.getLoc(), op.getResultTypes(), adaptor.getOperands(), + op->getAttrs()); + rewriter.replaceOp(op, new_op.getResults()); + return mlir::success(); + } +}; + +class TFAwaitOpConversion final + : public mlir::OpConversionPattern { + public: + explicit TFAwaitOpConversion(mlir::MLIRContext *context) + : mlir::OpConversionPattern(context) {} + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFAwaitOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + auto new_op = tf_mlrt::AwaitOp::create( + rewriter, op->getLoc(), rewriter.getType(), + adaptor.getFuture()); + rewriter.replaceOp(op, new_op.getResult()); + return mlir::success(); + } +}; + +class TFPromiseOpConversion final + : public mlir::OpConversionPattern { + public: + explicit TFPromiseOpConversion(mlir::MLIRContext *context) + : mlir::OpConversionPattern(context) {} + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFPromiseOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + if (llvm::isa<::mlrt::compiler::FutureType>( + adaptor.getTensor().getType())) { + auto new_op = tf_mlrt::PromiseFutureOp::create( + rewriter, op->getLoc(), adaptor.getPromise(), adaptor.getTensor()); + rewriter.replaceOp(op, new_op->getResults()); + + } else { + auto new_op = tf_mlrt::PromiseOp::create( + rewriter, op->getLoc(), adaptor.getPromise(), adaptor.getTensor()); + rewriter.replaceOp(op, new_op->getResults()); + } + return mlir::success(); + } +}; + +// Convert tf_mlrt::MapFn's signature to tf_mlrt::TFTensorType +class TFMapFnOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFMapFnOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + llvm::SmallVector result_types; + result_types.resize(op->getResultTypes().size(), + rewriter.getType()); + + auto new_op = + tf_mlrt::MapFnOp::create(rewriter, op.getLoc(), result_types, + adaptor.getOperands(), op->getAttrs()); + rewriter.replaceOp(op, new_op.getResult()); + return mlir::success(); + } +}; + +// Convert TF call ops (eg. StatefulPartitionedCall) to call. +template +class TFCallOpConversion : public mlir::OpConversionPattern { + public: + TFCallOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter) {} + + mlir::LogicalResult matchAndRewrite( + TFCallOp op, typename TFCallOp::Adaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + if (auto xla_must_compile = + op->template getAttrOfType("_XlaMustCompile"); + xla_must_compile && xla_must_compile.getValue()) { + return mlir::failure(); + } + + auto callee = + op.getCallableForCallee().template dyn_cast(); + if (!callee) return mlir::failure(); + + llvm::SmallVector result_types; + for (auto type : op.getOperation()->getResultTypes()) { + if (failed(type_converter_.convertType(type, result_types))) + return mlir::failure(); + } + + llvm::SmallVector materialized_operands; + for (int i = 0; i < adaptor.getOperands().size(); ++i) { + auto operand = adaptor.getOperands()[i]; + auto expected_type = + type_converter_.convertType(op.getOperand(i).getType()); + if (operand.getType() != expected_type) { + auto materialized = type_converter_.materializeTargetConversion( + rewriter, op.getLoc(), expected_type, operand); + if (!materialized) { + return mlir::failure(); + } + materialized_operands.push_back(materialized); + } else { + materialized_operands.push_back(operand); + } + } + + auto new_op = mlir::func::CallOp::create( + rewriter, op.getLoc(), result_types, + callee.getRootReference().getValue(), materialized_operands); + rewriter.replaceOp(op, new_op.getResults()); + return mlir::success(); + } + + private: + mlir::TypeConverter &type_converter_; +}; + +// Convert tf.Case op to mlrt.Case. +// +// TF dialect: +// %outputs = "tf.Case"(%idx_tensor, %arg, ...) { branches = [@branch0, +// @branch1], +// ...} +// +// lowered MLRT dialect: +// %branch_idx = tf_mlrt.tensor_to_int32(%idx_tensor) +// %outputs = mlrt.case %branch_idx [@branch0, @branch1] (%arg, ...) +class CaseOpConversion : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::CaseOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::ArrayAttr branches = op.getBranches(); + + llvm::SmallVector result_types; + result_types.resize(op->getResultTypes().size(), + rewriter.getType()); + + auto index_operand = tf_mlrt::TensorToIntOp::create( + rewriter, op.getLoc(), rewriter.getI32Type(), adaptor.getBranchIndex()); + + auto new_op = mlrt::compiler::CaseOp::create( + rewriter, op.getLoc(), result_types, index_operand.getResult(), + branches, adaptor.getInput()); + + rewriter.replaceOp(op, new_op.getResults()); + return mlir::success(); + } +}; + +class AsyncOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + // Hook for derived classes to implement combined matching and rewriting. + mlir::LogicalResult matchAndRewrite( + mlrt::compiler::AsyncOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp( + op, op->getResultTypes(), adaptor.getOperands(), op.getCallee()); + return mlir::success(); + } +}; + +// SetResourceOpConversion lowers a TF SetResource op to a tf_mlrt.set_resource +// op. +class SetResourceOpConversion final + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::_TfrtSetResourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, adaptor.getArg(), + op.getIndex()); + return mlir::success(); + } +}; + +// GetResourceOpConversion lowers a TF GetResource op to a tf_mlrt.get_resource +// op. +class GetResourceOpConversion final + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::_TfrtGetResourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + llvm::SmallVector result_types( + op.getNumResults(), rewriter.getType()); + auto new_op = tf_mlrt::GetResourceOp::create(rewriter, op->getLoc(), + result_types, op.getIndices()); + rewriter.replaceOp(op, new_op->getResults()); + return mlir::success(); + } +}; + +// Convert tf_mlrt.TFIfrtLoadVariableOp to tf_mlrt.IfrtLoadVariableOp +class TFIfrtLoadVariableOpConversion + : public mlir::OpConversionPattern { + public: + TFIfrtLoadVariableOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter) {} + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFIfrtLoadVariableOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + llvm::SmallVector result_types; + for (auto type : op->getResultTypes()) { + if (failed(type_converter_.convertType(type, result_types))) + return mlir::failure(); + } + + auto new_op = tf_mlrt::IfrtLoadVariableOp::create( + rewriter, op.getLoc(), result_types, adaptor.getOperands()[0], + op.getUsedByHostAttr()); + rewriter.replaceOp(op, new_op); + + return mlir::success(); + } + + private: + mlir::TypeConverter &type_converter_; +}; + +// Convert tf.IfrtRestoreVariableOp to tf_mlrt.IfrtRestoreVariableOp +class IfrtRestoreVariableOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::IfrtRestoreVariableOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + llvm::SmallVector result_types( + op.getNumResults(), rewriter.getType()); + rewriter.replaceOpWithNewOp( + op, result_types, adaptor.getOperands()[0], adaptor.getOperands()[1], + adaptor.getOperands()[2], + adaptor.getOperands().slice(3, adaptor.getOperands().size() - 3), + op.getRestoredDtypes(), op.getTruncateInCast(), + op.getReturnedTensorNames()); + + return mlir::success(); + } +}; + +class IfrtResourceDeserializeOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::IfrtResourceDeserializeOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter& rewriter) const override { + // Transfer the tensor_name attribute; drop the unused require_matching_crc. + auto tensor_name_attr = op->getAttr("tensor_name"); + if (!tensor_name_attr) { + return op.emitError("tensor_name attribute not found"); + } + + auto new_op = tf_mlrt::MlrtIfrtResourceDeserializeOp::create( + rewriter, op.getLoc(), adaptor.getResourceVar(), adaptor.getInputDir(), + llvm::cast(tensor_name_attr)); + rewriter.replaceOp(op, new_op); + + return mlir::success(); + } +}; + +class AsyncIfrtCallOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::AsyncIfrtCallOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter& rewriter) const override { + llvm::SmallVector future_types( + op->getNumResults(), rewriter.getType()); + + auto call_op = tf_mlrt::AsyncIfrtCallOp::create( + rewriter, op.getLoc(), future_types, op.getProgramId(), + op.getVariableArgIndices(), adaptor.getOperands()); + + rewriter.replaceOp(op, call_op.getResults()); + return mlir::success(); + } +}; + +std::optional DecodeLongName(mlir::Location loc) { + if (auto name_loc = mlir::dyn_cast(loc)) { + return name_loc.getName().str(); + } + + if (auto fused_loc = mlir::dyn_cast(loc)) { + std::string fused_name; + for (auto l : fused_loc.getLocations()) { + if (auto n = DecodeLongName(l)) { + fused_name += *n; + } + } + return fused_name; + } + + return std::nullopt; +} + +std::string GetNodeName(mlir::Operation *op) { + auto name = [&]() -> std::string { + if (auto name = DecodeLongName(op->getLoc())) { + return *std::move(name); + } + + return op->getName().stripDialect().str(); + }(); + + for (char &c : name) { + if (c == ':') c = '/'; + } + return name; +} + +void CanonicalizeFunctionNameInNodeDef(const mlir::SymbolTable &symbol_table, + NodeDef &node_def) { + for (auto &p : *node_def.mutable_attr()) { + if (p.second.has_func()) { + auto *func = p.second.mutable_func(); + if (auto n = CanonicalizeTensorflowFunctionName( + symbol_table, func->name(), + /*use_mlir_func_name=*/false)) { + func->set_name(*n); + } + } + + if (p.second.has_list() && p.second.list().func_size() > 0) { + for (auto &func : *p.second.mutable_list()->mutable_func()) { + if (auto n = CanonicalizeTensorflowFunctionName( + symbol_table, func.name(), + /*use_mlir_func_name=*/false)) { + func.set_name(*n); + } + } + } + } +} + +class ExecuteOpConversion final : public mlir::ConversionPattern { + public: + ExecuteOpConversion(mlir::MLIRContext* context, + const mlir::SymbolTable* symbol_table, + mlir::TypeConverter* type_converter, + ExecuteOpRegistry* execute_op_registry, + tfrt_stub::OpKernelRunnerCache* op_kernel_cache, + const tfrt_stub::FallbackState* fallback_state) + : mlir::ConversionPattern(*type_converter, + mlir::Pattern::MatchAnyOpTypeTag(), + /*benefit=*/1, context), + symbol_table_(*symbol_table), + execute_op_registry_(*execute_op_registry), + op_kernel_cache_(*op_kernel_cache), + fallback_state_(*fallback_state) {} + + mlir::LogicalResult matchAndRewrite( + mlir::Operation *op, llvm::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const override { + // TODO(b/173017701): Avoid fallback for ops within XLA GPU clusters. + if (!UseFallback(op)) return mlir::failure(); + + if (auto const_op = llvm::dyn_cast(op)) { + tensorflow::TensorProto tensor_proto; + auto status = ConvertToTensorProto(const_op.getValue(), &tensor_proto); + if (!status.ok()) + return const_op.emitError(absl::StatusMessageAsCStr(status)); + + rewriter.replaceOpWithNewOp( + op, rewriter.getType(), + tensor_proto.SerializeAsString()); + return mlir::success(); + } + + // The assign_op_key pass should have ran. + if (!op->hasAttr(tensorflow::tfrt_compiler::kOpKeyAttrName)) + return op->emitError("does not have op_key defined"); + + std::string node_name = GetNodeName(op); + + uint32_t execute_key = op->getAttrOfType( + tensorflow::tfrt_compiler::kOpKeyAttrName) + .getInt(); + + absl::StrAppend(&node_name, "_", execute_key); + + auto statusor_node_def = tensorflow::ConvertTFDialectOpToNodeDef( + op, node_name, /*ignore_unregistered_attrs=*/false); + if (!statusor_node_def.ok()) + return op->emitWarning("failed to export NodeDef."); + auto &node_def = **statusor_node_def; + + CanonicalizeFunctionNameInNodeDef(symbol_table_, node_def); + + std::string node_def_text; + google::protobuf::TextFormat::PrintToString(node_def, &node_def_text); + + mlir::Value device; + if (auto custom_device = + op->getAttrOfType(kTfMlrtCustomDevice)) { + device = + CreateCustomDevice(op->getLoc(), custom_device.getValue(), rewriter); + if (!device) return op->emitWarning("Failed to create custom device."); + } + + mlir::Operation *new_op = nullptr; + + auto create_async_execute_ops = [&]() -> mlir::LogicalResult { + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + if (device) { + new_op = rewriter.replaceOpWithNewOp( + op, result_types, device, operands, node_def_text, execute_key); + } else { + new_op = rewriter.replaceOpWithNewOp( + op, result_types, operands, node_def_text, execute_key); + } + if (mlir::failed( + execute_op_registry_.RegisterExecuteOp(new_op, execute_key))) { + return op->emitWarning("Fail to register async op"); + } + return mlir::success(); + }; + + // TODO(b/300999257): check whether to clean up for AoT mockGpu case later + + if (node_def.op() == kXlaLaunchOp) { + // XlaLaunch Op an AsyncOpKernel, we lower it to tf_mlrt.async_executeop, + // which return !mlrt.futures. These results will be converted as + // necessary through the target materialization hook in the type + // converter. + if (mlir::failed(create_async_execute_ops())) { + return mlir::failure(); + } + } else { + auto op_kernel_runner = op_kernel_cache_.GetOrCreate( + tfrt::Location(nullptr, execute_key), node_def.op(), + node_def.device(), op->getNumOperands(), + [&](tensorflow::AttrValueMap *attr_value_map) { + *attr_value_map = node_def.attr(); + return absl::OkStatus(); + }, + fallback_state_.device_manager(), + fallback_state_.process_function_library_runtime()); + // TODO(290630314): Use LOG_IF when absl logging is available + if (!op_kernel_runner.ok()) { + LOG(WARNING) << "Failed to create op_kernel_runner for " + << node_def_text + << " with error: " << op_kernel_runner.status(); + } + + if (op_kernel_runner.ok() && (*op_kernel_runner)->IsAsync()) { + // If it is an AsyncOpKernel, we lower it to tf_mlrt.async_executeop, + // which return !mlrt.futures. These results will be converted as + // necessary through the target materialization hook in the type + // converter. + if (mlir::failed(create_async_execute_ops())) { + return mlir::failure(); + } + } else { + // Otherwise, lower to tf_mlrt.executeop. + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + if (device) { + new_op = rewriter.replaceOpWithNewOp( + op, result_types, device, operands, node_def_text, execute_key); + } else { + new_op = rewriter.replaceOpWithNewOp( + op, result_types, operands, node_def_text, execute_key); + } + + if (op_kernel_runner.ok()) { + // Only register this executeop if its opkernel can be created. + // Otherwise, it is an unused op so we don't need to create them at + // runtime. + if (mlir::failed(execute_op_registry_.RegisterExecuteOp( + new_op, execute_key))) { + return op->emitWarning("Fail to register sync op"); + } + } + } + } + + return mlir::success(); + } + + private: + const mlir::SymbolTable &symbol_table_; + ExecuteOpRegistry &execute_op_registry_; + tfrt_stub::OpKernelRunnerCache &op_kernel_cache_; + const tfrt_stub::FallbackState &fallback_state_; +}; + +mlir::Value GetPredicate(mlir::Operation *op, mlir::Value cond_operand, + mlir::ConversionPatternRewriter &rewriter) { + return tf_mlrt::PredicateOp::create(rewriter, op->getLoc(), + rewriter.getI1Type(), cond_operand); +} + +class CondOpConversion : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::IfOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::FlatSymbolRefAttr then_branch = op.getThenBranchAttr(); + mlir::FlatSymbolRefAttr else_branch = op.getElseBranchAttr(); + + llvm::SmallVector result_types( + op.getNumResults(), rewriter.getType()); + + auto bool_cond = GetPredicate(op, adaptor.getCond(), rewriter); + + auto new_op = mlrt::compiler::CondOp::create( + rewriter, op.getLoc(), result_types, bool_cond, adaptor.getInput(), + then_branch, else_branch); + + rewriter.replaceOp(op, new_op.getResults()); + + return mlir::success(); + } +}; + +// Convert TF WhileOp to mlrt.while. +// The pseudo code of mlrt.while is as follows: +// +// while(cond) { +// outputs, cond = body(inputs) +// inputs = outputs +// } +// return outputs, cond +// +// So we need to insert extra conversion kernels and merge functions when +// lowering tf.While to mlrt.while. +// +// %result = tf.While(%arg) {cond = @original_cond_fn, body = +// @original_body_fn} +// +// is converted to +// +// func @new_pred_fn(%arg) { +// %cond_tensor = func.call @original_cond_fn(%arg) +// %cond_bool = mlrt.predicate %cond_tensor +// return %cond_bool +// } +// +// func @new_while_body(%arg) { +// %result = func.call @original_body_fn(%arg) +// %cond_bool = func.call @new_pred_fn(%result) +// return%result, %cond_bool +// } +// +// %first_iter_cond = func.call @new_pred_fn(%arg) +// %result = mlrt.while %first_iter_cond @new_while_body(%arg) +// +class WhileOpConversion : public mlir::OpConversionPattern { + public: + WhileOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + mlir::SymbolTable *symbol_table) + : mlir::OpConversionPattern(*type_converter, context), + symbol_table_(*symbol_table) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::WhileOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::FlatSymbolRefAttr cond_fn = op.getCondAttr(); + mlir::FlatSymbolRefAttr body_fn = op.getBodyAttr(); + + // Create the predicate function that calls the original cond function and + // in addition convert the result to a boolean value. + mlir::func::FuncOp pred_fn = GetPredicateFunction( + op, cond_fn, adaptor.getOperands().getTypes(), rewriter); + if (!pred_fn) return mlir::failure(); + + // Insert a call op to call the pred function for the first iteration. + auto call_pred_fn = mlir::func::CallOp::create( + rewriter, op.getLoc(), pred_fn.getFunctionType().getResults(), + pred_fn.getSymName(), adaptor.getOperands()); + + if (!call_pred_fn) return mlir::failure(); + + // Create the new while body function. + mlir::func::FuncOp new_body_fn = GetWhileBodyFunction( + op, body_fn, pred_fn, adaptor.getOperands().getTypes(), rewriter); + + // mlrt.while returns one more additional boolean value than tf.while. + llvm::SmallVector while_result_types( + adaptor.getOperands().getTypes().begin(), + adaptor.getOperands().getTypes().end()); // = while_arg_types; + while_result_types.push_back(rewriter.getI1Type()); + auto new_op = mlrt::compiler::WhileOp::create( + rewriter, op.getLoc(), while_result_types, call_pred_fn.getResult(0), + adaptor.getOperands(), new_body_fn.getSymName()); + + rewriter.replaceOp(op, new_op.getResults().drop_back()); + + return mlir::success(); + } + + private: + mlir::func::FuncOp GetPredicateFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr cond_fn, + mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::func::FuncOp GetWhileBodyFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr body_fn, + mlir::func::FuncOp pred_fn, mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::SymbolTable &symbol_table_; +}; + +// Create the pred function that contains a call to the original cond function +// and a predicate kernel that converts the cond tensor to a boolean value. eg. +// +// func @pred_fn( %arg) { +// %cond_tensor = tf_mlrt.call @original_cond_fn(%arg) +// %cond_bool = tf_mlrt.predicate %cond_tensor +// return %cond_bool +// } +// +mlir::func::FuncOp WhileOpConversion::GetPredicateFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr cond_fn, + mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const { + std::string pred_fn_name = + absl::StrCat(cond_fn.getValue().str(), "/tf_mlrt_predicate"); + + if (auto pred_fn = symbol_table_.lookup(pred_fn_name)) { + return pred_fn; + } + + auto func_op = op->getParentOfType(); + + mlir::ConversionPatternRewriter::InsertionGuard insertion_guard(rewriter); + rewriter.setInsertionPointAfter(func_op); + + auto func_type = rewriter.getFunctionType(arg_types, {rewriter.getI1Type()}); + + auto pred_fn = mlir::func::FuncOp::create(rewriter, op.getLoc(), pred_fn_name, + func_type); + + auto *block = pred_fn.addEntryBlock(); + rewriter.setInsertionPointToStart(block); + + auto call_cond_fn = + mlir::func::CallOp::create(rewriter, op.getLoc(), arg_types.take_front(), + cond_fn, block->getArguments()); + mlir::Value bool_cond = GetPredicate(op, call_cond_fn.getResult(0), rewriter); + mlir::func::ReturnOp::create(rewriter, op.getLoc(), bool_cond); + + symbol_table_.insert(pred_fn); + + return pred_fn; +} + +// Create the new while body function that contains a call to original while +// body and then a call to the pred function. eg. +// +// func @while_body(%arg) { +// %result = mlrt.call @original_body(%arg) +// %cond_bool = mlrt.call @pred_function(%arg) +// mlrt.return %result, %cond_bool +// } +// +mlir::func::FuncOp WhileOpConversion::GetWhileBodyFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr original_body_fn, + mlir::func::FuncOp pred_fn, mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const { + std::string body_fn_name = + absl::StrCat(original_body_fn.getValue().str(), "/tf_mlrt_body"); + + if (auto body_fn = symbol_table_.lookup(body_fn_name)) { + return body_fn; + } + + auto func_op = op->getParentOfType(); + + mlir::ConversionPatternRewriter::InsertionGuard insertion_guard(rewriter); + rewriter.setInsertionPointAfter(func_op); + + llvm::SmallVector body_result_types(arg_types.begin(), + arg_types.end()); + // The last result of the while body function is the boolean condition. + body_result_types.push_back(rewriter.getI1Type()); + + auto func_type = rewriter.getFunctionType(arg_types, body_result_types); + auto body_fn = mlir::func::FuncOp::create(rewriter, op.getLoc(), body_fn_name, + func_type); + + auto *block = body_fn.addEntryBlock(); + rewriter.setInsertionPointToStart(block); + + // Insert a call to the original body function. + // The returned result type is also the original argument types. + auto call_original_body_fn = + mlir::func::CallOp::create(rewriter, op.getLoc(), arg_types, + original_body_fn, block->getArguments()); + + // Insert a call to the pred function, which contains a call to the original + // cond function and the predicate kernel that converts the tensor to boolean + // value. + auto call_pred_fn = mlir::func::CallOp::create( + rewriter, op.getLoc(), pred_fn.getFunctionType().getResults(), + pred_fn.getSymName(), call_original_body_fn.getResults()); + + llvm::SmallVector body_results = + call_original_body_fn.getResults(); + + // The last result should be the boolean value converted from the condition. + auto bool_cond = call_pred_fn.getResult(0); + body_results.push_back(bool_cond); + + mlir::func::ReturnOp::create(rewriter, op.getLoc(), body_results); + + symbol_table_.insert(body_fn); + + return body_fn; +} + +class BatchFunctionOpConversion + : public mlir::OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::BatchFunctionOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + std::string node_name = GetNodeName(op); + + auto statusor_node_def = tensorflow::ConvertTFDialectOpToNodeDef( + op, node_name, /*ignore_unregistered_attrs=*/true); + if (!statusor_node_def.ok()) + return op->emitWarning("failed to export NodeDef."); + const auto &node_def = **statusor_node_def; + + std::string node_def_text; + google::protobuf::TextFormat::PrintToString(node_def, &node_def_text); + + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + + if (auto custom_device = + op->getAttrOfType(kTfMlrtCustomDevice)) { + mlir::Value device = + CreateCustomDevice(op->getLoc(), custom_device.getValue(), rewriter); + if (!device) return op->emitWarning("Failed to create custom device."); + + rewriter.replaceOpWithNewOp( + op, result_types, device, adaptor.getOperands(), node_def.device(), + op.getFAttr(), node_def_text); + } else { + rewriter.replaceOpWithNewOp( + op, result_types, adaptor.getOperands(), node_def.device(), + op.getFAttr(), node_def_text); + } + + return mlir::success(); + } +}; + +void CreateFallbackInitializationFunction( + mlir::ModuleOp module, ExecuteOpRegistry &execute_op_registry) { + mlir::OpBuilder builder(&module.getBodyRegion()); + + auto func_op = mlir::func::FuncOp::create( + builder, module.getLoc(), "_tfrt_fallback_init", + mlir::FunctionType::get(module.getContext(), /*inputs=*/{}, + /*outputs=*/{})); + + auto *block = func_op.addEntryBlock(); + builder.setInsertionPointToStart(block); + + // Create operations for all fallback kernels in the module. + for (const auto &[op_index, op] : + llvm::enumerate(execute_op_registry.GetExecuteOps())) { + if (op) { + // There might be unused ops, and we don't need to create them at runtime. + // + // TODO(chky, deqiangc): Clean up unused ops before hand. + tf_mlrt::CreateOp::create( + builder, func_op.getLoc(), /*resultTypes=*/mlir::TypeRange{}, + /*operands=*/mlir::ValueRange{}, op->getAttrs()); + } + } + + mlir::func::ReturnOp::create(builder, func_op.getLoc()); +} + +// Move the tf_mlrt.await ops to right before their first uses to avoid +// unnecessary blocking. +void MoveAwaitOpToFirstUse(mlir::Block &block) { + llvm::SmallVector await_ops; + for (auto &op : block) { + if (auto await_op = llvm::dyn_cast(&op)) { + await_ops.push_back(await_op); + } + } + + for (auto op : await_ops) { + auto result = op.getResult(); + if (result.use_empty()) continue; + + mlir::Operation *first_user = *result.user_begin(); + for (auto *user : result.getUsers()) { + if (user->isBeforeInBlock(first_user)) { + first_user = user; + } + } + + op->moveBefore(first_user); + } +} + +const tfrt_stub::FallbackState &GetDefaultFallbackState() { + static const auto *const fallback_state = []() { + tensorflow::SessionOptions session_options; + tensorflow::FunctionDefLibrary fdef_lib; + auto fallback_state = + tfrt_stub::FallbackState::Create(session_options, fdef_lib).value(); + return fallback_state.release(); + }(); + + return *fallback_state; +} + +// The conversion pass that is run before 'tf-mlrt-parallelization' passes. The +// parallelization pass changes the graph content, so any rewrite/conversion +// that depends on the graph instead of individual ops should be done before +// parallelization. +class TfToMlrtPreParallelizationConversionPass + : public mlir::PassWrapper> { + public: + TfToMlrtPreParallelizationConversionPass() = default; + explicit TfToMlrtPreParallelizationConversionPass( + const TfrtPipelineOptions &options) { + // This is needed to progating user configs into this pass. + options_.copyOptionValuesFrom(options); + } + TfToMlrtPreParallelizationConversionPass( + const TfToMlrtPreParallelizationConversionPass& other) + : PassWrapper(other) { + options_.copyOptionValuesFrom(other.options_); + } + TfToMlrtPreParallelizationConversionPass &operator=( + const TfToMlrtPreParallelizationConversionPass &) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + TfToMlrtPreParallelizationConversionPass) + + private: + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + registry.insert(); + + RegisterTpuDialect(registry); + } + + llvm::StringRef getArgument() const final { + return "pre-parallel-tf-to-mlrt"; + } + llvm::StringRef getDescription() const final { + return "pre-parallel-tf-to-mlrt"; + } + + mlir::LogicalResult initialize(mlir::MLIRContext *context) override { + if (use_tpu_host_allocator_for_inputs_.hasValue()) { + options_.use_tpu_host_allocator_for_inputs = + use_tpu_host_allocator_for_inputs_; + } + + return mlir::success(); + } + + mlir::LogicalResult runOnFunction(mlir::func::FuncOp func) { + auto &context = getContext(); + mlir::ConversionTarget target(context); + mlir::RewritePatternSet patterns(&getContext()); + target.addLegalDialect(); + PopulateTpuPreParallelizationConversionPatterns(target, patterns, options_); + + return mlir::applyPartialConversion(func, target, std::move(patterns)); + } + + void maySetTpuHostAllocatorForBatch(mlir::TF::BatchFunctionOp batch_op, + mlir::ModuleOp module, + mlir::SymbolTable &symbol_table) { + mlir::func::FuncOp batched = llvm::dyn_cast( + symbol_table.lookupSymbolIn(module, batch_op.getF())); + int used_by_tpu = 0; + int num_in_tensors = batch_op.getInTensors().size(); + for (auto arg : batched.getArguments()) { + if (arg.getArgNumber() >= num_in_tensors) continue; + for (auto user : arg.getUsers()) { + if (llvm::isa(user)) { + used_by_tpu++; + break; + } + } + } + + if (used_by_tpu == num_in_tensors) { + mlir::OpBuilder builder(module); + batch_op->setAttr(kTfMlrtCustomDevice, + builder.getStringAttr(kTpuHostDevice)); + } + } + + void runOnOperation() override { + auto module = getOperation(); + mlir::SymbolTable symbol_table(module); + if (options_.use_tpu_host_allocator_for_inputs) { + module.walk([&](mlir::TF::BatchFunctionOp batch_op) { + maySetTpuHostAllocatorForBatch(batch_op, module, symbol_table); + }); + } + + for (auto func : module.getOps()) { + if (mlir::failed(runOnFunction(func))) { + signalPassFailure(); + return; + } + } + } + + Option use_tpu_host_allocator_for_inputs_{ + *this, "use-tpu-host-allocator-for-inputs", + llvm::cl::desc("If true, fallback executeops that produce inputs to tpu " + "program will use tpu host allocator."), + llvm::cl::init(false)}; + + TfrtPipelineOptions options_; +}; + +class TfToMlrtConversionPass + : public mlir::PassWrapper> { + public: + TfToMlrtConversionPass() + : TfToMlrtConversionPass({}, &GetDefaultFallbackState()) {} + explicit TfToMlrtConversionPass( + const TfrtPipelineOptions &options, + const tfrt_stub::FallbackState *fallback_state) + : fallback_state_(*fallback_state) { + // This is needed to progating user configs into this pass. + options_.copyOptionValuesFrom(options); + } + TfToMlrtConversionPass(const TfToMlrtConversionPass& other) + : fallback_state_(other.fallback_state_) { + options_.copyOptionValuesFrom(other.options_); + } + TfToMlrtConversionPass &operator=(const TfToMlrtConversionPass &) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TfToMlrtConversionPass) + + private: + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + registry.insert(); + + RegisterTpuDialect(registry); + } + + llvm::StringRef getArgument() const final { return "tf-to-mlrt"; } + llvm::StringRef getDescription() const final { return "tf-to-mlrt"; } + + mlir::LogicalResult initialize(mlir::MLIRContext *context) override { + // TODO(b/285064425): See if this and below are the right way to + // accommodate other dialects. + type_converter_.addConversion([](mlir::Type type) { return type; }); + type_converter_.addConversion( + [=](mlir::TensorType type) -> std::optional { + // Ref types are not supported in both compiler and runtime. + if (mlir::isa(type.getElementType())) + return std::nullopt; + return tf_mlrt::TFTensorType::get(context); + }); + + auto future_to_tensor_materialization = + [](mlir::OpBuilder &builder, mlir::Type desired_type, + mlir::ValueRange inputs, mlir::Location loc) -> mlir::Value { + if (inputs.size() != 1) return mlir::Value(); + + if (mlir::isa(inputs[0].getType())) { + if (mlir::isa(desired_type)) { + return tf_mlrt::AwaitOp::create(builder, loc, desired_type, + inputs[0]); + } + + return mlir::Value(); + } + + return inputs[0]; + }; + + type_converter_.addTargetMaterialization(future_to_tensor_materialization); + type_converter_.addSourceMaterialization(future_to_tensor_materialization); + + if (use_tpu_host_allocator_for_inputs_.hasValue()) { + options_.use_tpu_host_allocator_for_inputs = + use_tpu_host_allocator_for_inputs_; + } + return mlir::success(); + } + + void runOnOperation() override { + auto module = getOperation(); + mlir::SymbolTable symbol_table(module); + + // Use llvm::make_early_inc_range instead of the stock range from + // module.getOps because conversions such as WhileOpConversion could insert + // new functions into the module ops list causing the stock range to not + // able to find next OP correctly. + for (auto func : + llvm::make_early_inc_range(module.getOps())) { + if (mlir::failed(runOnFunction(func, symbol_table))) { + signalPassFailure(); + return; + } + } + + // Some mlrt kernels such as tf_mlrt_tpu.CompileAndExecute produce futures, + // but function invoked by mlrt execute op are not aware of these changes. + // We add a post process to fix up this caller-callee mismatch. + for (auto func : module.getOps()) { + CollectFunctionCallSiteInputTypes(func); + } + for (auto func : module.getOps()) { + if (mlir::failed(PostProcessFunctionSignature(func, symbol_table))) { + signalPassFailure(); + return; + } + // Move the tf_mlrt.await ops to right before their first uses to avoid + // unnecessary blocking. + MoveAwaitOpToFirstUse(func.getBlocks().front()); + } + + CreateFallbackInitializationFunction(module, execute_op_registry_); + + module.walk([&](mlir::UnrealizedConversionCastOp op) { + op->replaceAllUsesWith(op->getOperands()); + op->erase(); + }); + + AddAwaitOpToUnusedFutures(module); + } + + void AddAwaitOpToUnusedFutures(mlir::ModuleOp module) { + for (auto func : module.getOps()) { + llvm::SmallVector unused_futures; + + auto is_unused_future = [](mlir::Value result) { + return llvm::isa<::mlrt::compiler::FutureType>(result.getType()) && + result.use_empty(); + }; + + for (auto arg : func.getArguments()) { + if (is_unused_future(arg)) { + unused_futures.push_back(arg); + } + } + + for (auto &op : func.getBody().front()) { + for (mlir::Value result : op.getResults()) { + if (is_unused_future(result)) { + unused_futures.push_back(result); + } + } + } + + if (!unused_futures.empty()) { + auto builder = + mlir::OpBuilder::atBlockTerminator(&func.getBody().front()); + ::mlrt::compiler::AwaitAllControlOp::create(builder, func.getLoc(), + unused_futures); + } + } + } + + mlir::LogicalResult PostProcessFunctionSignature( + mlir::func::FuncOp func, mlir::SymbolTable &symbol_table) { + mlir::ConversionTarget target(getContext()); + mlir::RewritePatternSet patterns(&getContext()); + + target.addDynamicallyLegalOp( + [this](mlir::func::FuncOp func) { + // By default, we assume callers are well behaved. + if (function_call_site_input_types_.find(func.getName()) == + function_call_site_input_types_.end()) { + return true; + } + DCHECK_EQ(function_call_site_input_types_.at(func.getName()).size(), + func.getFunctionType().getInputs().size()); + + for (auto [expected_input_type, call_site_type] : + llvm::zip(func.getFunctionType().getInputs(), + function_call_site_input_types_.at(func.getName()))) { + if (expected_input_type != call_site_type) { + return false; + } + } + return true; + }); + + patterns.add(&getContext(), &type_converter_, + &function_call_site_input_types_); + + return mlir::applyPartialConversion(func, target, std::move(patterns)); + } + + void CollectFunctionCallSiteInputTypes(mlir::func::FuncOp func) { + func.walk([&function_call_site_input_types = + function_call_site_input_types_]( + mlir::Operation *op) mutable { + // Only collect the call-site input types when a function is invoked + // by async op. This is the only known case that the previous pass + // may left un-match types between call-site and callee. + if (auto async_op = llvm::dyn_cast(op)) { + function_call_site_input_types[async_op.getCallee() + .getLeafReference()] = + llvm::SmallVector(async_op.getOperandTypes().begin(), + async_op.getOperandTypes().end()); + } + }); + } + + mlir::LogicalResult runOnFunction(mlir::func::FuncOp func, + mlir::SymbolTable &symbol_table) { + auto &context = getContext(); + mlir::ConversionTarget target(context); + mlir::RewritePatternSet patterns(&getContext()); + target.addLegalDialect(); + target.addIllegalDialect(); + + target.addIllegalOp(); + target.addIllegalOp(); + target.addIllegalOp(); + target.addIllegalOp(); + target.addIllegalOp(); + + target.addDynamicallyLegalOp( + [this](mlir::func::FuncOp op) { + return type_converter_.isSignatureLegal(op.getFunctionType()); + }); + target.addDynamicallyLegalOp( + [this](mlir::func::ReturnOp op) { + for (auto operand : op.getOperands()) { + if (!type_converter_.isLegal(operand.getType())) return false; + } + return true; + }); + target.addDynamicallyLegalOp( + [this](mlrt::compiler::AsyncOp op) { + for (auto operand : op.getOperands()) { + if (!type_converter_.isLegal(operand.getType())) return false; + } + return true; + }); + target.addDynamicallyLegalOp( + [this](mlir::func::CallOp op) { + for (auto operand : op.getOperands()) { + if (!type_converter_.isLegal(operand.getType())) { + return false; + } + } + return true; + }); + + // LINT.IfChange + // Order the list of added ops alphabetically. + patterns.add(&context, &type_converter_, &symbol_table); + patterns + .add( + &context); + patterns.add(type_converter_, + &context); + patterns.add(&context, &symbol_table, &type_converter_, + &execute_op_registry_, &op_kernel_cache_, + &fallback_state_); + patterns.add, + TFCallOpConversion, + TFCallOpConversion>(&context, + &type_converter_); + // LINT.ThenChange(util.cc) + + mlir::populateFunctionOpInterfaceTypeConversionPattern( + patterns, type_converter_); + mlir::populateReturnOpTypeConversionPattern(patterns, type_converter_); + + PopulateTpuConversionPatterns(target, patterns, type_converter_, + execute_op_registry_, options_); + + return mlir::applyPartialConversion(func, target, std::move(patterns)); + } + + Option use_tpu_host_allocator_for_inputs_{ + *this, "use-tpu-host-allocator-for-inputs", + llvm::cl::desc("If true, fallback executeops that produce inputs to tpu " + "program will use tpu host allocator."), + llvm::cl::init(false)}; + TfrtPipelineOptions options_; + mlir::TypeConverter type_converter_; + ExecuteOpRegistry execute_op_registry_; + tfrt_stub::OpKernelRunnerCache op_kernel_cache_; + const tfrt_stub::FallbackState &fallback_state_; + + // True input argument types for a given function at call site. + llvm::DenseMap> + function_call_site_input_types_; +}; + +} // namespace + +std::unique_ptr> +CreateTfToMlrtPreParallelizationConversionPass( + const TfrtPipelineOptions &options) { + return std::make_unique(options); +} + +std::unique_ptr> +CreateTfToMlrtConversionPass(const TfrtPipelineOptions &options, + const tfrt_stub::FallbackState *fallback_state) { + return std::make_unique(options, fallback_state); +} + +std::unique_ptr> +CreateTfToMlrtConversionPass(const TfrtPipelineOptions &options) { + return CreateTfToMlrtConversionPass(options, &GetDefaultFallbackState()); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.h new file mode 100644 index 0000000000000000000000000000000000000000..53210e71b6fc13fd2c95ec562447032ef0dba9db --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tf_to_mlrt.h @@ -0,0 +1,48 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TF_TO_MLRT_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TF_TO_MLRT_H_ +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" + +namespace tensorflow { +namespace mlrt_compiler { + +// The conversion pass that is run before 'tf-mlrt-parallelization' passes. The +// parallelization pass changes the graph content, so any rewrite/conversion +// that depends on the graph instead of individual ops should be done before +// parallelization. +std::unique_ptr> +CreateTfToMlrtPreParallelizationConversionPass( + const TfrtPipelineOptions& options); + +// The conversion pass that is run after 'tf-mlrt-parallelization' passes. The +// parallelization pass changes the graph content, so this pass should only +// contain conversion that depends on individual ops. +std::unique_ptr> +CreateTfToMlrtConversionPass(const TfrtPipelineOptions& options); + +std::unique_ptr> +CreateTfToMlrtConversionPass(const TfrtPipelineOptions& options, + const tfrt_stub::FallbackState* fallback_state); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TF_TO_MLRT_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.cc new file mode 100644 index 0000000000000000000000000000000000000000..de274ecf269fbe06fc0c6a885dcdb267a333b9a8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.cc @@ -0,0 +1,169 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.h" + +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_tpu_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/mlrt_device_constants.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +class TPUCompileMlirAndExecuteOpPreParallelizationConversion + : public mlir::OpConversionPattern { + public: + TPUCompileMlirAndExecuteOpPreParallelizationConversion( + mlir::MLIRContext* context, bool use_tpu_host_allocator_for_inputs) + : OpConversionPattern(context), + use_tpu_host_allocator_for_inputs_(use_tpu_host_allocator_for_inputs) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::TPUCompileMlirAndExecuteOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter& rewriter) const override { + llvm::SmallVector constant_operand_indices; + llvm::SmallVector non_constant_operand_indices; + + for (int i = 0; i < adaptor.getArgs().size(); ++i) { + auto operand = adaptor.getOperands()[i]; + auto original_operand = op.getOperand(i); + if (IsResultVariable(original_operand, operand)) { + // NOTE: It's important to populate constant_operand_indices in + // ascending order. + constant_operand_indices.push_back(i); + } else { + non_constant_operand_indices.push_back(i); + } + } + + llvm::SmallVector operands = adaptor.getArgs(); + + size_t tensor_operands_size = operands.size(); + operands.append(adaptor.getStaticShapes().begin(), + adaptor.getStaticShapes().end()); + + auto producer_name = op->getAttrOfType("producer_name"); + + llvm::SmallVector operands_with_static_shapes; + if (adaptor.getOperandsWithStaticShape().has_value()) { + for (auto attr : adaptor.getOperandsWithStaticShapeAttr() + .getAsRange()) { + operands_with_static_shapes.push_back( + static_cast(attr.getInt())); + } + } + + if (use_tpu_host_allocator_for_inputs_) { + llvm::DenseMap replaced_ops; + + for (int i : non_constant_operand_indices) { + DCHECK_LT(i, op.getNumOperands()); + auto old_value = operands[i]; + mlir::Operation* def = old_value.getDefiningOp(); + + if (def && llvm::isa(def->getDialect())) { + auto*& op_with_device = replaced_ops[def]; + if (!op_with_device) { + mlir::ConversionPatternRewriter::InsertionGuard guard(rewriter); + rewriter.setInsertionPoint(def); + + op_with_device = rewriter.clone(*def); + op_with_device->setAttr(kTfMlrtCustomDevice, + rewriter.getStringAttr(kTpuHostDevice)); + rewriter.replaceOp(def, op_with_device->getResults()); + } + } + } + } + + auto compile_and_execute_op = tf_mlrt::TFTPUCompileAndExecuteOp::create( + rewriter, op.getLoc(), op.getResultTypes(), operands, + rewriter.getDenseI32ArrayAttr(constant_operand_indices), + op.getMetadataAttr(), op.getMlirModuleAttr(), + rewriter.getUI32IntegerAttr(tensor_operands_size), + rewriter.getDenseI32ArrayAttr(operands_with_static_shapes), + producer_name); + + rewriter.replaceOp(op, compile_and_execute_op->getResults()); + + return mlir::success(); + } + + private: + bool use_tpu_host_allocator_for_inputs_ = false; +}; + +class TPUCompileMlirAndExecuteOpConversion + : public mlir::OpConversionPattern { + public: + TPUCompileMlirAndExecuteOpConversion(mlir::TypeConverter* type_converter, + mlir::MLIRContext* context, + ExecuteOpRegistry* execute_op_registry) + : OpConversionPattern(*type_converter, context) {} + + mlir::LogicalResult matchAndRewrite( + tf_mlrt::TFTPUCompileAndExecuteOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter& rewriter) const override { + llvm::SmallVector operands = + adaptor.getOperandsAndStaticShapes(); + llvm::SmallVector result_types; + result_types.push_back(rewriter.getType()); + result_types.append(op.getResults().size(), + rewriter.getType()); + + auto compile_and_execute_op = tf_mlrt_tpu::CompileAndExecuteOp::create( + rewriter, op.getLoc(), result_types, operands, + op.getConstantOperandIndices(), op.getMetadataAttr(), + op.getMlirModuleAttr(), op.getNumOperands(), + op.getOperandsWithStaticShape(), op.getProducerName()); + + rewriter.replaceOp(op, compile_and_execute_op->getResults()); + + return mlir::success(); + } +}; + +} // namespace + +void PopulateTpuPreParallelizationConversionPatterns( + mlir::ConversionTarget& target, mlir::RewritePatternSet& patterns, + const TfrtPipelineOptions& options) { + target.addIllegalOp(); + patterns.add( + patterns.getContext(), options.use_tpu_host_allocator_for_inputs); +} + +void PopulateTpuConversionPatterns(mlir::ConversionTarget& target, + mlir::RewritePatternSet& patterns, + mlir::TypeConverter& type_converter, + ExecuteOpRegistry& execute_op_registry, + const TfrtPipelineOptions& options) { + target.addIllegalOp(); + target.addLegalDialect(); + + patterns.add( + &type_converter, patterns.getContext(), &execute_op_registry); +} + +void RegisterTpuDialect(mlir::DialectRegistry& registry) { + registry.insert(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.h new file mode 100644 index 0000000000000000000000000000000000000000..7f0fbe2d28735ce39f0f729f1835abe3e42f081a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/tpu_conversion_patterns.h @@ -0,0 +1,42 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TPU_CONVERSION_PATTERNS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TPU_CONVERSION_PATTERNS_H_ + +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/execute_op_registry.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" + +namespace tensorflow { +namespace mlrt_compiler { + +void RegisterTpuDialect(mlir::DialectRegistry& registry); + +void PopulateTpuPreParallelizationConversionPatterns( + mlir::ConversionTarget& target, mlir::RewritePatternSet& patterns, + const TfrtPipelineOptions& options); + +void PopulateTpuConversionPatterns(mlir::ConversionTarget& target, + mlir::RewritePatternSet& patterns, + mlir::TypeConverter& type_converter, + ExecuteOpRegistry& execute_op_registry, + const TfrtPipelineOptions& options); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_TPU_CONVERSION_PATTERNS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.cc new file mode 100644 index 0000000000000000000000000000000000000000..eb941aa8322b698bfb30ad5d1fede7b5a486bef3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.cc @@ -0,0 +1,45 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h" + +#include "llvm/Support/Casting.h" +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h.inc" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.h.inc" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h.inc" + +namespace tensorflow { +namespace mlrt_compiler { + +bool UseFallback(mlir::Operation* op) { + if (!llvm::isa(op->getDialect())) return false; + + // TODO(b/173017701): have a centralized place to hold the information + // whether a TF op should be lowered to FallbackExecute op. + // TODO(b/319045348): Define trait to reflect that IfrtLoadVariableOp has no + // TF kernels so that we don't need to check every op here. + return !llvm::isa< + mlir::TF::_TfrtSetResourceOp, mlir::TF::_TfrtGetResourceOp, + mlir::TF::BatchFunctionOp, mlir::TF::CaseOp, + mlir::TF::IfrtRestoreVariableOp, mlir::TF::StatefulPartitionedCallOp, + mlir::TF::PartitionedCallOp, mlir::TF::LegacyCallOp, mlir::TF::IfOp, + mlir::TF::WhileOp, mlir::TF::TPUCompileMlirAndExecuteOp, + mlir::TF::AsyncIfrtCallOp>(op); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h new file mode 100644 index 0000000000000000000000000000000000000000..064ef99fb217be3475f0cf73be5e00b0d800d75a --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/util.h @@ -0,0 +1,30 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_UTIL_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_UTIL_H_ + +#include "mlir/IR/Operation.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +// Use fallback by default for anything that does not have a native kernel +// with some exceptions. +bool UseFallback(mlir::Operation* op); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_UTIL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.cc new file mode 100644 index 0000000000000000000000000000000000000000..7e4680f0e91142c200e5d308aed3955ab9a29d46 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.cc @@ -0,0 +1,1004 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/log/check.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/IRMapping.h" // from @llvm-project +#include "mlir/IR/Matchers.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/WalkResult.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/tf_mlrt_ops.h" + +namespace tensorflow { +namespace mlrt_compiler { +namespace { + +std::optional GetConstantValue( + mlir::Value value, const mlir::SymbolTable& symbol_table) { + if (!value) return std::nullopt; + + if (auto const_op = + llvm::dyn_cast_or_null(value.getDefiningOp())) { + llvm::APInt val; + if (mlir::matchPattern(const_op.getOperation(), + mlir::m_ConstantInt(&val))) { + return val; + } + } + + if (auto get_resource_op = + llvm::dyn_cast_or_null( + value.getDefiningOp())) { + auto indices_attr = get_resource_op.getIndices(); + if (indices_attr.size() != 1) return std::nullopt; + int64_t index = llvm::cast(indices_attr[0]).getInt(); + + auto init_func = + symbol_table.lookup("_tfrt_resource_init"); + if (!init_func) return std::nullopt; + + mlir::TF::_TfrtSetResourceOp set_resource_op; + init_func.walk([&](mlir::TF::_TfrtSetResourceOp op) { + if (op.getIndex() == index) { + set_resource_op = op; + return mlir::WalkResult::interrupt(); + } + return mlir::WalkResult::advance(); + }); + + if (set_resource_op) { + return GetConstantValue(set_resource_op.getArg(), symbol_table); + } + } + + return std::nullopt; +} + +void RemoveIdentityOp(mlir::func::FuncOp func) { + auto &block = func.getBody().front(); + llvm::SmallVector identity_ops; + for (auto &op : block) { + if (auto identity_op = llvm::dyn_cast(&op)) { + identity_ops.push_back(identity_op); + } + } + + for (auto op : llvm::reverse(identity_ops)) { + op.getOutput().replaceAllUsesWith(op.getInput()); + } + + for (auto op : identity_ops) { + op->erase(); + } + + auto return_op = llvm::cast(block.getTerminator()); + + auto func_type = mlir::FunctionType::get( + func.getContext(), func.getArgumentTypes(), return_op.getOperandTypes()); + + func.setType(func_type); +} + +// tf.map_fn (https://www.tensorflow.org/api_docs/python/tf/map_fn) is converted +// to tf.while during lowering. tf.map_fn expects parallel execution of its body +// function but not all tf.while can guarantee parallel executions. The tf.while +// op that is converted from tf.map_fn has distinct programming patterns. This +// pass matches those patterns to convert applicable tf.while to tf_mlrt.map_fn +// for parallel execution of the body function. +// +// For example, tf.map_fn(fn, elems, ...) can be converted to the following: +// +// %tensor_list = "tf.TensorListReserve"(%per_iteration_shape, %max_iterations) +// +// %while_outputs:7 = "tf.While"(%loop_counter, +// %tensor_list_index, %other_args, %tensor_list) {body = @while_body, cond = +// @while_cond} +// +// %outputs = "tf.TensorListStack"(%while_outputs#2, %output_shape) +// +// in which +// +// while_cond: check loop_counter and tensor_list_index both smaller than +// max_iterations. +// +// while_body: loop_counter and tensor_list_index is incremented and returned; +// also gather input from elems based on un-incremented tensor_list_index, +// call fn and set output into a TensorList at tensor_list_index. +// +// This pass additionally assumes the following patterns to identify a tf.While +// that are converted from tf.map_fn: +// 1. Arguments have one loop_counter and one element_index that are initialized +// to be 0. +// 2. TensorList or TensorArray is reserved with max_iterations size. The +// max_iterations shall be a constant. +// 3. The predicate function check both loop_counter and element_index is less +// than max_iterations. +// 4. The body function increase loop_counter and element_index by 1 and use +// element_index to stores its result into tensor list or tensor array such +// that there is no overlap in write between iterations +// 5. The body function does not have side effects such that one iteration will +// impact the next iteration outside #4. +// +// After conversion, the pseudocode is +// +// %tensor_list = "tf.TensorListReserve"(%per_iteration_shape, %max_iterations) +// +// %updated_tensor_list = "tf_mlrt.map_fn" (%max_iterations, %tensor_list, +// %other_args) {body = @map_fn_body} +// +// %outputs = "tf.TensorListStack"(%updated_tensor_list, %output_shape) +// +// where +// +// tf_mlrt.map_fn leads to a blocking call and +// the argument list of tf_mlrt.map_fn is (%max_iterations, %tensor_list, +// tf.while's argument list minus loop_counter, tensor_list_index and +// tensor_list). tf_mlrt.map_fn is a block call and returns the updated tensor +// list. +// +// map_fn_body has an input signature of (%in_tensor_list_future, +// %out_tensor_list_promise, %loop_counter, %tensor_list_index, %other_args) and +// has not return values (the updated_tensor_list is delivered through +// %out_tensor_list_promise). +// +class WhileToMapFnPass + : public mlir::PassWrapper> { + public: + WhileToMapFnPass() = default; + WhileToMapFnPass &operator=(const WhileToMapFnPass &) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(WhileToMapFnPass) + + private: + struct LoopInfo { + // Argument indices in while op of key loop variables. + int loop_counter = -1; + int element_index = -1; + std::vector tensor_list_or_flow_in; + // Max iteration may be passed in as an argument to while op. + std::optional max_iterations_arg_idx; + // Max itertions may be hard coded as constant inside while predicate + // function. + std::optional max_iterations_value; + // Defining Op of max_iterations. + mlir::Value max_iterations; + }; + + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + registry.insert(); + registry.insert(); + } + + llvm::StringRef getArgument() const final { + return "tf-mlrt-while-to-map-fn"; + } + + llvm::StringRef getDescription() const final { + return "Convert tf.while to tf_mlrt.map_fn when possible for parallel " + "execution."; + } + + void runOnOperation() override { + mlir::ModuleOp module = getOperation(); + mlir::SymbolTable symbol_table(module); + + // Use make_early_inc_range because the processing might insert new node + // into the list + for (auto func_op : + llvm::make_early_inc_range(module.getOps())) { + MayConvertWhileToMapFn(func_op, symbol_table); + } + } + + // We match while op's predicate function and body function with known + // patterns from tf.map_fn. If matched, tf.while is converted to + // tf_mlrt.map_fn. + void MayConvertWhileToMapFn(mlir::func::FuncOp op, + mlir::SymbolTable &symbol_table) { + mlir::OpBuilder builder(op); + for (mlir::Operation &op : llvm::make_early_inc_range(op.front())) { + auto while_op = llvm::dyn_cast(&op); + if (!while_op) continue; + LoopInfo loop_info; + if (mlir::succeeded(MatchPredicate(while_op.getCondAttr(), symbol_table, + loop_info)) && + mlir::succeeded( + MatchBody(while_op.getBodyAttr(), symbol_table, loop_info)) && + mlir::succeeded( + MatchInputSource(while_op, symbol_table, loop_info)) && + mlir::succeeded(MatchOutputUse(while_op, loop_info))) { + // Input, predicate function, body function and output are all following + // patterns, we can convert it to tf_mlrt.map_fn. + mlir::func::FuncOp while_body_func = + symbol_table.lookup(while_op.getBody()); + auto map_fn_body_func = CreateMapFnBodyFunction( + builder, while_body_func, symbol_table, loop_info); + + mlir::OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointAfter(while_op); + std::vector invariant_arguments; + invariant_arguments.reserve(while_op->getNumOperands()); + + absl::flat_hash_set variant_arguments = {loop_info.loop_counter, + loop_info.element_index}; + variant_arguments.insert(loop_info.tensor_list_or_flow_in.begin(), + loop_info.tensor_list_or_flow_in.end()); + for (int i = 0; i < while_op->getNumOperands(); ++i) { + if (variant_arguments.contains(i)) { + continue; + } + invariant_arguments.push_back(while_op.getOperand(i)); + } + + llvm::SmallVector result_types; + llvm::SmallVector tensor_list_operands; + for (int i = 0; i < loop_info.tensor_list_or_flow_in.size(); ++i) { + tensor_list_operands.push_back( + while_op.getOperand(loop_info.tensor_list_or_flow_in[i])); + result_types.push_back( + while_op.getResult(loop_info.tensor_list_or_flow_in[i]) + .getType()); + } + + auto map_fn_op = tf_mlrt::TFMapFnOp::create( + builder, while_op.getLoc(), result_types, loop_info.max_iterations, + tensor_list_operands, invariant_arguments, + map_fn_body_func.getSymName(), + loop_info.tensor_list_or_flow_in.size()); + + // MatchOutputUse already makes sure only the tensor_list or + // tensor_array output is used. + absl::flat_hash_map old_arg_indx_to_new_index; + for (int i = 0; i < loop_info.tensor_list_or_flow_in.size(); ++i) { + old_arg_indx_to_new_index.insert( + {loop_info.tensor_list_or_flow_in[i], i}); + } + for (int i = 0; i < while_op.getResults().size(); ++i) { + if (!old_arg_indx_to_new_index.contains(i)) { + while_op.getResult(i).dropAllUses(); + } else { + while_op.getResult(i).replaceAllUsesWith( + map_fn_op.getResult()[old_arg_indx_to_new_index[i]]); + } + } + + while_op.erase(); + } + } + } + + // Match that (a) the tensor list or tensor array are reserved with + // max_iterations size such that parallel operations on tensor list or tensor + // array is thread safe; (b) loop_counter and element_index starts with 0. + // Also may identify source of max_iterations. + mlir::LogicalResult MatchInputSource(mlir::TF::WhileOp while_op, + const mlir::SymbolTable& symbol_table, + LoopInfo& loop_info) { + // Element index and loop counter should start from 0. + std::optional loop_counter_val = GetConstantValue( + while_op.getOperand(loop_info.loop_counter), symbol_table); + std::optional element_index_val = GetConstantValue( + while_op.getOperand(loop_info.element_index), symbol_table); + if (!loop_counter_val.has_value() || !loop_counter_val->isZero() || + !element_index_val.has_value() || !element_index_val->isZero()) { + return mlir::failure(); + } + + DCHECK_GE(loop_info.tensor_list_or_flow_in.size(), 1); + // Tensor list or a tensor array are reserved + + for (auto tensor_list_index : loop_info.tensor_list_or_flow_in) { + mlir::Operation *tensor_list_or_flow_in_defining_op = + while_op.getOperand(tensor_list_index).getDefiningOp(); + if (tensor_list_or_flow_in_defining_op == nullptr) { + return mlir::failure(); + } + + mlir::Operation *max_iterations = nullptr; + if (loop_info.max_iterations_arg_idx.has_value()) { + max_iterations = + while_op.getOperand(loop_info.max_iterations_arg_idx.value()) + .getDefiningOp(); + } + if (auto tensor_list_reserve = + llvm::dyn_cast( + tensor_list_or_flow_in_defining_op)) { + // Tensor list should resever for max_iterations. + mlir::Operation *tensor_list_reserve_size = + tensor_list_reserve.getNumElements().getDefiningOp(); + + if (tensor_list_reserve_size != max_iterations) { + // if tensor list is not reserved by max_iteration variable, then + // another acceptable case is that both contain same constant values. + std::optional reserved_cst = GetConstantValue( + tensor_list_reserve.getNumElements(), symbol_table); + if (!reserved_cst.has_value() || + !loop_info.max_iterations_value.has_value() || + *reserved_cst != loop_info.max_iterations_value.value()) { + return mlir::failure(); + } + } + // TensorListReserveOp has only one result and is already in used by + // while. + loop_info.max_iterations = tensor_list_reserve.getNumElements(); + } else if (auto tensor_array = llvm::dyn_cast( + tensor_list_or_flow_in_defining_op)) { + mlir::Operation *tensor_array_size = + tensor_array.getOperand().getDefiningOp(); + if (tensor_array_size != max_iterations) { + // if tensor array is not reserved by max_iteration variable, then + // another acceptable case is that both contain same constant values. + std::optional reserved_cst = + GetConstantValue(tensor_array.getOperand(), symbol_table); + if (!reserved_cst.has_value() || + !loop_info.max_iterations_value.has_value() || + *reserved_cst != loop_info.max_iterations_value.value()) { + return mlir::failure(); + } + } + + // Other than flow_in, the tensor array should be used by while as well. + if (!llvm::is_contained(while_op.getOperands(), + tensor_array.getHandle())) { + return mlir::failure(); + } + loop_info.max_iterations = tensor_array.getSize(); + } else { + return mlir::failure(); + } + } + return mlir::success(); + } + + // Match the map_attern that output of while op is subsequentially stacked. + mlir::LogicalResult MatchOutputUse(mlir::TF::WhileOp &while_op, + const LoopInfo &loop_info) { + absl::flat_hash_set used_results; + used_results.insert(loop_info.tensor_list_or_flow_in.begin(), + loop_info.tensor_list_or_flow_in.end()); + for (int i = 0; i < while_op->getResults().size(); ++i) { + if (used_results.contains(i)) { + // Tensor list or flow in should be used next. + if (!while_op->getResult(i).hasOneUse()) { + return mlir::failure(); + } + } else { + // No other result should be used. + if (!while_op->getResult(i).use_empty()) { + return mlir::failure(); + } + } + } + + for (auto result_index : loop_info.tensor_list_or_flow_in) { + // Finds the use of the tensor list or flow in is a tensor list stack or + // tensor array gather. This maybe over-conservative, but we rather be + // correct than sorry. + mlir::Operation *use_op = + *while_op->getResult(result_index).getUsers().begin(); + if (llvm::isa(use_op)) { + use_op = *use_op->getUsers().begin(); + } + + if (!llvm::isa(use_op)) { + return mlir::failure(); + } + } + return mlir::success(); + } + + // Match that the while predicate function is doing just + // loop_counter < max iterations && element_index < max_iterations. + // Through this pattern, we also update the argument index of + // loop_counter, element_index and possibly max_iterations. + mlir::LogicalResult MatchPredicate(mlir::FlatSymbolRefAttr predicate_fn, + const mlir::SymbolTable &symbol_table, + LoopInfo &loop_info) { + mlir::func::FuncOp predicate_fn_op = + symbol_table.lookup(predicate_fn.getValue()); + + // The body of the predicate function should have two LessOp and one + // LogicalAndOp. It can optionally has IdentityOp and ToBoolOp. + enum class PredicateBodyExpectingOp { + kExpectFirstLess, + kExpectSecondLess, + kExpectLogicalAnd, + kExpectTerminator + }; + std::vector less_ops; + less_ops.reserve(2); + PredicateBodyExpectingOp expecting_op = + PredicateBodyExpectingOp::kExpectFirstLess; + for (auto &body_op : predicate_fn_op.getBody().front()) { + switch (expecting_op) { + case PredicateBodyExpectingOp::kExpectFirstLess: + if (llvm::isa(body_op)) { + expecting_op = PredicateBodyExpectingOp::kExpectSecondLess; + less_ops.push_back(&body_op); + } else if (!llvm::isa( + body_op)) { + return mlir::failure(); + } + break; + case PredicateBodyExpectingOp::kExpectSecondLess: + if (llvm::isa(body_op)) { + expecting_op = PredicateBodyExpectingOp::kExpectLogicalAnd; + less_ops.push_back(&body_op); + } else if (!llvm::isa( + body_op)) { + return mlir::failure(); + } + break; + case PredicateBodyExpectingOp::kExpectLogicalAnd: + if (llvm::isa(body_op)) { + expecting_op = PredicateBodyExpectingOp::kExpectTerminator; + } else if (!llvm::isa(body_op)) { + return mlir::failure(); + } + break; + case PredicateBodyExpectingOp::kExpectTerminator: + if (!llvm::isa(body_op)) { + return mlir::failure(); + } + break; + default: + return mlir::failure(); + } + } + + // Identify loop_counter + int counter_index = -1; + auto counter_iter = + llvm::find(predicate_fn_op.getArguments(), less_ops[0]->getOperand(0)); + if (counter_iter != predicate_fn_op.getArguments().end()) { + counter_index = counter_iter->getArgNumber(); + if (!IsScalarOrUnrankedI32Tensor( + predicate_fn_op.getArgument(counter_index))) { + return mlir::failure(); + } + } + + // Find upper bound on loop_counter. + int max_iter_index_from_counter = -1; + int max_iter_value_from_counter = -1; + if (auto max_iter_iter = llvm::find(predicate_fn_op.getArguments(), + less_ops[0]->getOperand(1)); + max_iter_iter != predicate_fn_op.getArguments().end()) { + // Upper bound on loop_counter is from one argument. + max_iter_index_from_counter = max_iter_iter->getArgNumber(); + // Argument has to be int32 + if (!IsScalarOrUnrankedI32Tensor( + predicate_fn_op.getArgument(max_iter_index_from_counter))) { + return mlir::failure(); + } + } else { + // If upper bound is not passed in, it has to be a constant + std::optional value = + GetConstantValue(less_ops[0]->getOperand(1), symbol_table); + if (!value.has_value()) { + return mlir::failure(); + } + max_iter_value_from_counter = value->getZExtValue(); + } + + // Identify element_index + int element_index = -1; + auto element_index_iter = + llvm::find(predicate_fn_op.getArguments(), less_ops[1]->getOperand(0)); + if (element_index_iter != predicate_fn_op.getArguments().end()) { + element_index = element_index_iter->getArgNumber(); + if (!IsScalarOrUnrankedI32Tensor( + predicate_fn_op.getArgument(element_index))) { + return mlir::failure(); + } + } + + // Find upper bound on element_index. + int max_iter_index_from_element = -1; + int max_iter_value_from_element = -1; + if (auto max_iter_iter = llvm::find(predicate_fn_op.getArguments(), + less_ops[1]->getOperand(1)); + max_iter_iter != predicate_fn_op.getArguments().end()) { + // Upper bound on element_index is from one argument. + max_iter_index_from_element = max_iter_iter->getArgNumber(); + // Upper bound argument needs to be int32 + if (!IsScalarOrUnrankedI32Tensor( + predicate_fn_op.getArgument(max_iter_index_from_element))) { + return mlir::failure(); + } + } else { + // If upper bound is not passed in, it has to be a constant + std::optional value = + GetConstantValue(less_ops[1]->getOperand(1), symbol_table); + if (!value.has_value()) { + return mlir::failure(); + } + max_iter_value_from_element = value->getZExtValue(); + } + + // Loop_counter is always available. + if (counter_index < 0) return mlir::failure(); + // element_index can change its location, but will always be provided. + if (element_index < 0) return mlir::failure(); + + std::optional max_iter_const; + std::optional max_iter_index; + if (max_iter_index_from_counter < 0 && max_iter_index_from_element < 0) { + // If both loop counter and element index are not upper bounded by passing + // in arguments, they shall be upper bounded by constants of same value. + if (max_iter_value_from_element != max_iter_value_from_counter || + max_iter_value_from_element < 0 || max_iter_value_from_counter < 0) { + return mlir::failure(); + } else { + max_iter_const = max_iter_value_from_element; + } + } else if (max_iter_index_from_counter >= 0 && + max_iter_index_from_element >= 0) { + // Loop counter or element are upper bounded by pass-in arguments. + // They need to be upper bounded by the same argument + if (max_iter_index_from_element != max_iter_index_from_counter) { + return mlir::failure(); + } else { + max_iter_index = max_iter_index_from_counter; + } + } else { + // TODO(deqiangc): remove this clause after verifying grappler pass remove + // the case that one of them is bounded by pass-in argument and the other + // is bounded by constants. + max_iter_index = + std::max(max_iter_index_from_counter, max_iter_index_from_element); + max_iter_const = + std::max(max_iter_value_from_element, max_iter_value_from_counter); + } + + // Update hypothesis + loop_info.loop_counter = counter_index; + loop_info.element_index = element_index; + loop_info.max_iterations_arg_idx = max_iter_index; + loop_info.max_iterations_value = max_iter_const; + return mlir::success(); + } + + // Match that the current hypothesis of current loop_counter and element_index + // in the while body function based on the following simple pattern: + // %updated_loop_counter = %loop_counter + 1 + // %updated_element_index = %element_index + 1 + // %loaded_elem = tf.Gather(.., %element_index,... ) + // DoSomething + // tf.TensorListSetItem(.., %element_index) + // return %update_loop_counter, %updated_element_index, + // %tensor_array_list, %max_iterations, %other_args + mlir::LogicalResult MatchLoopCounterElementIndexInBody( + mlir::func::FuncOp while_body_func, const mlir::SymbolTable& symbol_table, + LoopInfo& loop_info) { + mlir::Block &block = while_body_func.getBlocks().front(); + + // Verify argument loop_counter is +1 and returned at the same location. + mlir::BlockArgument loop_counter = + block.getArgument(loop_info.loop_counter); + llvm::SmallVector loop_counter_users = + GetUsersIgnoringIdentityOp(loop_counter); + if (loop_counter_users.size() != 1 || + !llvm::isa( + loop_counter_users.front())) { + return mlir::failure(); + } + std::optional loop_counter_add_val = GetConstantValue( + loop_counter_users.front()->getOperand(1), symbol_table); + if (!loop_counter_add_val.has_value() || *loop_counter_add_val != 1) { + return mlir::failure(); + } + + // loop_counter + 1 is in ReturnOp's operand. + if (loop_counter_users.front() != + GetDefiningOpIgnoringIdentityOp( + GetReturnedOperand(while_body_func, loop_info.loop_counter))) { + return mlir::failure(); + } + + // Verify element_index's usage and also identify the argument index of + // tensor list or tensor array flow_in. + std::vector tensor_list_or_flow_in_index; + mlir::BlockArgument element_index = + block.getArgument(loop_info.element_index); + for (auto *element_index_use : GetUsersIgnoringIdentityOp(element_index)) { + if (llvm::isa(element_index_use)) { + // One use of element_index is +1 and then returned at the same + // location. + std::optional element_index_add_val = + GetConstantValue(element_index_use->getOperand(1), symbol_table); + if (!element_index_add_val.has_value() || *element_index_add_val != 1 || + element_index_use != + GetDefiningOpIgnoringIdentityOp(GetReturnedOperand( + while_body_func, loop_info.element_index))) { + return mlir::failure(); + } + } else if (llvm::isa(element_index_use)) { + if (auto tensor_list_index = MayGetArgumentIndexIgnoringIdentityOp( + while_body_func, + llvm::dyn_cast(element_index_use) + .getInputHandle()); + !tensor_list_index.has_value()) { + return mlir::failure(); + } else { + tensor_list_or_flow_in_index.push_back(tensor_list_index.value()); + } + } else if (llvm::isa(element_index_use)) { + if (auto flow_in_index = MayGetArgumentIndexIgnoringIdentityOp( + while_body_func, llvm::dyn_cast( + element_index_use) + .getFlowIn()); + !flow_in_index.has_value()) { + return mlir::failure(); + } else { + tensor_list_or_flow_in_index.push_back(flow_in_index.value()); + } + } else if (!llvm::isa(element_index_use)) { + // The only other use is to either gather the input or set output. + return mlir::failure(); + } + } + + if (tensor_list_or_flow_in_index.empty()) { + return mlir::failure(); + } + + // Update hypothesis + loop_info.tensor_list_or_flow_in = std::move(tensor_list_or_flow_in_index); + + return mlir::success(); + } + + // Match that the while body function is the following simple pattern: + // %updated_loop_counter = %loop_counter + 1 + // %updated_element_index = %element_index + 1 + // %loaded_elem = tf.Gather(.., %element_index,... ) + // DoSomething + // tf.TensorListSetItem(.., %element_index) + // return %update_loop_counter, %updated_element_index, + // %tensor_array_list, %max_iterations, %other_args + // + // in which + // DoSomething has no side-effect on the next iteration. + // + // Also identify argument index for TensorList or TensorArray flow_in. + mlir::LogicalResult MatchBody(mlir::FlatSymbolRefAttr while_body_func_name, + const mlir::SymbolTable &symbol_table, + LoopInfo &loop_info) { + mlir::func::FuncOp while_body_func = + symbol_table.lookup( + while_body_func_name.getValue()); + + if (mlir::failed(MatchLoopCounterElementIndexInBody( + while_body_func, symbol_table, loop_info))) { + // Swap the order of loop_counter and element_index in the current + // hypothesis and try again + int swap = loop_info.loop_counter; + loop_info.loop_counter = loop_info.element_index; + loop_info.element_index = swap; + if (mlir::failed(MatchLoopCounterElementIndexInBody( + while_body_func, symbol_table, loop_info))) { + return mlir::failure(); + } + } + + // The next iteration of while_body does not depend on the previous + // iteration except loop_counter, element_index, tensor_list_or_flow_in, and + // max_iterations. + absl::flat_hash_set allowed_variable_between_iterations; + allowed_variable_between_iterations.insert(loop_info.loop_counter); + allowed_variable_between_iterations.insert(loop_info.element_index); + if (loop_info.max_iterations_arg_idx.has_value()) { + allowed_variable_between_iterations.insert( + loop_info.max_iterations_arg_idx.value()); + } + allowed_variable_between_iterations.insert( + loop_info.tensor_list_or_flow_in.begin(), + loop_info.tensor_list_or_flow_in.end()); + for (int j = 0; j < while_body_func.getNumArguments(); j++) { + if (!allowed_variable_between_iterations.contains(j)) { + if (GetReturnedOperand(while_body_func, j) != + while_body_func.getArgument(j)) { + return mlir::failure(); + } + } + } + + return mlir::success(); + } + + // The map_fn body function is a clone of the while_body_func that + // canonicalize loop_counter and tensor_list_index to be the first two + // arguments. + mlir::func::FuncOp CreateMapFnBodyFunction(mlir::OpBuilder &builder, + mlir::func::FuncOp while_body_func, + mlir::SymbolTable &symbol_table, + const LoopInfo &loop_info) { + std::string map_fn_body_name = + absl::StrCat(while_body_func.getSymName().str(), "/MapFnBody"); + + if (auto func = symbol_table.lookup(map_fn_body_name)) { + return func; + } + + RemoveIdentityOp(while_body_func); + + absl::flat_hash_set variant_arguments = {loop_info.loop_counter, + loop_info.element_index}; + variant_arguments.insert(loop_info.tensor_list_or_flow_in.begin(), + loop_info.tensor_list_or_flow_in.end()); + llvm::SmallVector remapped_input_type; + + for (int i = 0; i < loop_info.tensor_list_or_flow_in.size(); i++) { + remapped_input_type.push_back( + builder.getType()); + remapped_input_type.push_back( + builder.getType()); + } + + remapped_input_type.push_back( + while_body_func.getFunctionType().getInput(loop_info.loop_counter)); + remapped_input_type.push_back( + while_body_func.getFunctionType().getInput(loop_info.element_index)); + for (int i = 0; i < while_body_func.getFunctionType().getNumInputs(); i++) { + if (!variant_arguments.contains(i)) { + remapped_input_type.push_back( + while_body_func.getFunctionType().getInput(i)); + } + } + mlir::OpBuilder::InsertionGuard insertion_guard(builder); + builder.setInsertionPointAfter(while_body_func); + auto map_fn_body_func = mlir::func::FuncOp::create( + builder, while_body_func.getLoc(), map_fn_body_name, + mlir::FunctionType::get(while_body_func.getContext(), + remapped_input_type, {})); + + map_fn_body_func->setAttr( + "tfrt.cost_threshold", + builder.getI64IntegerAttr(std::numeric_limits::max())); + + if (while_body_func.getArgAttrs().has_value()) { + llvm::SmallVector remapped_input_attributes; + // No attributes carry over for tensor list future/promise. + for (int i = 0; i < loop_info.tensor_list_or_flow_in.size(); i++) { + remapped_input_attributes.push_back(mlir::Attribute()); + remapped_input_attributes.push_back(mlir::Attribute()); + } + auto args_attrs = while_body_func.getArgAttrs().value(); + remapped_input_attributes.push_back(args_attrs[loop_info.loop_counter]); + remapped_input_attributes.push_back(args_attrs[loop_info.element_index]); + for (int i = 0; i < args_attrs.size(); i++) { + if (!variant_arguments.contains(i)) { + remapped_input_attributes.push_back(args_attrs[i]); + } + } + map_fn_body_func.setAllArgAttrs(remapped_input_attributes); + } + auto future_index = [](int i) { return 2 * i; }; + auto promise_index = [](int i) { return 2 * i + 1; }; + + if (while_body_func.getResAttrs().has_value()) { + // The order and types of results remain the same; so does attributes. + map_fn_body_func.setAllResultAttrs(while_body_func.getResAttrs().value()); + } + map_fn_body_func.setVisibility(mlir::func::FuncOp::Visibility::Private); + + builder.setInsertionPointToEnd(map_fn_body_func.addEntryBlock()); + + mlir::IRMapping mapping; + std::vector await_ops; + for (int i = 0; i < loop_info.tensor_list_or_flow_in.size(); i++) { + await_ops.push_back(tf_mlrt::TFAwaitOp::create( + builder, while_body_func.getLoc(), + while_body_func.getArgument(loop_info.tensor_list_or_flow_in.at(i)) + .getType(), + map_fn_body_func.getArgument(future_index(i)))); + + mapping.map( + while_body_func.getArgument(loop_info.tensor_list_or_flow_in.at(i)), + await_ops.at(i)); + } + // Rest of argument start after promise + int map_fn_argument_index = + promise_index(loop_info.tensor_list_or_flow_in.size() - 1); + mapping.map(while_body_func.getArgument(loop_info.loop_counter), + map_fn_body_func.getArgument(++map_fn_argument_index)); + mapping.map(while_body_func.getArgument(loop_info.element_index), + map_fn_body_func.getArgument(++map_fn_argument_index)); + for (int i = 0; i < while_body_func.getNumArguments(); i++) { + if (!variant_arguments.contains(i)) { + mapping.map(while_body_func.getArgument(i), + map_fn_body_func.getArgument(++map_fn_argument_index)); + } + } + + for (auto &op : while_body_func.getBody().front()) { + builder.clone(op, mapping); + } + + auto return_op = map_fn_body_func.getBody().front().getTerminator(); + + mlir::Operation *first_write = nullptr; + // Move tensor list write to the end of the block. + for (int index : loop_info.tensor_list_or_flow_in) { + auto *def = return_op->getOperand(index).getDefiningOp(); + CHECK(def); // Crash OK + def->moveBefore(return_op); + if (!first_write) first_write = def; + } + + // Move the await op before the first write. + for (auto tensor_list_or_flow_in : await_ops) { + tensor_list_or_flow_in->moveBefore(first_write); + } + + // Insert promise right before return + builder.setInsertionPoint(return_op); + for (int i = 0; i < await_ops.size(); i++) { + tf_mlrt::TFPromiseOp::create( + builder, return_op->getLoc(), + map_fn_body_func.getArgument(promise_index(i)), + return_op->getOperand(loop_info.tensor_list_or_flow_in.at(i))); + } + mlir::func::ReturnOp::create(builder, return_op->getLoc()); + return_op->erase(); + + symbol_table.insert(map_fn_body_func); + + return map_fn_body_func; + } + + std::optional MayGetArgumentIndexIgnoringIdentityOp( + mlir::func::FuncOp func, mlir::Value value) const { + // Value may go through some identify chains. + while (value.getDefiningOp()) { + if (!llvm::isa(value.getDefiningOp())) { + return std::nullopt; + } + value = value.getDefiningOp()->getOperand(0); + } + + // Value is directly from argument since it has no defining op. + auto argument_iter = llvm::find(func.getArguments(), value); + if (argument_iter == func.getArguments().end()) { + return std::nullopt; + } + return argument_iter->getArgNumber(); + } + + // Given a value, find its use ignoring identify op. + // For example, given the below chains: + // + // %original_value = OriginalDefinedOp() + // %value1 = tf.IdentifyOp(original_value) + // %value2 = tf.IdentifyOp(value1) + // UseOp(%value2) + // + // GetUseIgnroningIdentifyOp(%original_value) will return UseOp + llvm::SmallVector GetUsersIgnoringIdentityOp( + mlir::Value value) { + llvm::SmallVector users; + std::vector users_stack; + + for (auto *direct_user : value.getUsers()) { + users_stack.push_back(direct_user); + } + + while (!users_stack.empty()) { + mlir::Operation *descendent_user = users_stack.back(); + users_stack.pop_back(); + + if (!llvm::isa(descendent_user)) { + users.push_back(descendent_user); + } else { + // User of identify op is considered as user. + for (auto *user : descendent_user->getResult(0).getUsers()) { + users_stack.push_back(user); + } + } + } + return users; + } + + // Given a value, find its source defined op ignoring identify op. + // For example, given the below chains: + // + // %original_value = OriginalDefinedOp() + // %value1 = tf.IdentifyOp(original_value) + // %value2 = tf.IdentifyOp(value1) + // UseOp(%value2) + // + // GetDefiningOpIgnroningIdentifyOp(%value2) will return OriginalDefinedOp + mlir::Operation *GetDefiningOpIgnoringIdentityOp(mlir::Value value) { + mlir::Operation *source_op = value.getDefiningOp(); + while (llvm::isa(source_op)) { + source_op = source_op->getOperand(0).getDefiningOp(); + } + return source_op; + } + + mlir::Value GetReturnedOperand(const mlir::func::FuncOp func, + uint32_t result_index) { + auto return_op = llvm::dyn_cast( + func->getRegion(0).front().getTerminator()); + DCHECK_NE(return_op, nullptr); + return return_op->getOperand(result_index); + } + + bool IsScalarI32Tensor(mlir::Value value) const { + if (auto value_type = llvm::dyn_cast(value.getType())) { + if (value_type.getElementType().isInteger(32) && value_type.hasRank() && + value_type.getRank() == 0) { + return true; + } + } + return false; + } + + bool IsScalarOrUnrankedI32Tensor(mlir::Value value) const { + if (auto value_type = llvm::dyn_cast(value.getType())) { + if (value_type.getElementType().isInteger(32) && + ((value_type.hasRank() && value_type.getRank() == 0) || + !value_type.hasRank())) { + return true; + } + } + return false; + } +}; +} // namespace + +std::unique_ptr> CreateWhileToMapFnPass() { + return std::make_unique(); +} + +} // namespace mlrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h new file mode 100644 index 0000000000000000000000000000000000000000..47e4acca7687d2fb49e69df9495744778dade2df --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/mlrt/while_to_map_fn.h @@ -0,0 +1,31 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_WHILE_TO_MAP_FN_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_WHILE_TO_MAP_FN_H_ + +#include + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace mlrt_compiler { + +std::unique_ptr> CreateWhileToMapFnPass(); + +} // namespace mlrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_MLRT_WHILE_TO_MAP_FN_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize.cc new file mode 100644 index 0000000000000000000000000000000000000000..ac601c9b725213430e81522e5e6081850f2c46c7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize.cc @@ -0,0 +1,181 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Block.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Rewrite/FrozenRewritePatternSet.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +// Fold tf.DeviceIndex to tf.Const if it has device assigned. +class FoldDeviceIndex : public mlir::OpRewritePattern { + public: + using mlir::OpRewritePattern::OpRewritePattern; + + mlir::LogicalResult matchAndRewrite( + mlir::TF::DeviceIndexOp op, + mlir::PatternRewriter &rewriter) const override { + auto device = op->getAttrOfType("device"); + if (!device) return mlir::failure(); + + DeviceNameUtils::ParsedName parsed_name; + if (!DeviceNameUtils::ParseFullName(device.getValue().str(), + &parsed_name) || + !parsed_name.has_type) + return mlir::failure(); + + int32_t i = 0; + mlir::ArrayAttr device_names = op.getDeviceNames(); + for (; i < device_names.size(); ++i) { + auto device_name = + mlir::cast(device_names[i]).getValue(); + if (device_name == parsed_name.type) break; + } + + rewriter.replaceOpWithNewOp( + op, + mlir::DenseIntElementsAttr::get( + mlir::RankedTensorType::get(/*shape=*/{}, rewriter.getI32Type()), + i)); + + return mlir::success(); + } +}; + +// A custom hash and compare function for finding out common ops. +struct SimpleOperationInfo : public llvm::DenseMapInfo { + static unsigned getHashValue(const mlir::Operation *opC) { + return mlir::OperationEquivalence::computeHash( + const_cast(opC), + /*hashOperands=*/mlir::OperationEquivalence::directHashValue, + /*hashResults=*/mlir::OperationEquivalence::ignoreHashValue, + mlir::OperationEquivalence::IgnoreLocations); + } + static bool isEqual(const mlir::Operation *lhsC, + const mlir::Operation *rhsC) { + auto *lhs = const_cast(lhsC); + auto *rhs = const_cast(rhsC); + if (lhs == rhs) return true; + return mlir::OperationEquivalence::isEquivalentTo( + const_cast(lhsC), + const_cast(rhsC), + mlir::OperationEquivalence::IgnoreLocations); + } +}; + +void EliminateCommonMultinomialOps(mlir::Block &block) { + llvm::SmallDenseMap, 2, + SimpleOperationInfo> + multinomial_to_eliminate; + + auto eliminate = [&]() { + auto &list = multinomial_to_eliminate.begin()->second; + auto first = list.front(); + for (auto op : llvm::drop_begin(list)) { + op.getOutput().replaceAllUsesWith(first.getOutput()); + op->erase(); + } + multinomial_to_eliminate.clear(); + }; + + for (auto &op : block) { + auto multinomial_op = llvm::dyn_cast(&op); + // Conservatively, we only eliminate back-to-back tf.Multinomial ops. + if (multinomial_op) { + if (multinomial_to_eliminate.find(multinomial_op) == + multinomial_to_eliminate.end() && + !multinomial_to_eliminate.empty()) { + // If the current op is a tf.Multinomial but it is different from the + // preiously found tf.Multinomial, then we eliminate the prviously found + // tf.Multinomial. + eliminate(); + } + multinomial_to_eliminate[multinomial_op].push_back(multinomial_op); + } else if (!multinomial_to_eliminate.empty()) { + // If the current op is not a tf.Multinomial, then we eliminate previously + // found tf.Multinomial + eliminate(); + } + } +} + +// Optimization pass for TFRT-specific rewrite patterns. +class OptimizeTfForTfrt + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OptimizeTfForTfrt) + + llvm::StringRef getArgument() const final { return "optimize-tf-for-tfrt"; } + + llvm::StringRef getDescription() const final { + return "optmize TF MLIR for TFRT workflow."; + } + + mlir::LogicalResult initialize(mlir::MLIRContext *context) override { + mlir::RewritePatternSet pattern_list(context); + pattern_list.add(context); + patterns_ = std::move(pattern_list); + return mlir::success(); + } + + void runOnOperation() override { + auto func = getOperation(); + + EliminateCommonMultinomialOps(func.getBody().front()); + + if (mlir::failed(mlir::applyPatternsGreedily(func, patterns_))) + signalPassFailure(); + } + + private: + mlir::FrozenRewritePatternSet patterns_; +}; + +} // namespace + +std::unique_ptr> +CreateOptimizeTfForTfrtPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass; + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize_tf_control_flow_side_effect.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize_tf_control_flow_side_effect.cc new file mode 100644 index 0000000000000000000000000000000000000000..2abbdc7d6a09fabf571846e1bad43c51d72c5130 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/optimize_tf_control_flow_side_effect.cc @@ -0,0 +1,186 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +bool IsOpNonSideEffectingOrReadOnly(mlir::Operation* op, + bool include_read_only = true) { + // Though tf.Assert and tf.Timestamp are side-effecting, they do not + // interfere with any other side-effecting ops. For now, if control flow + // ops' callee functions contain them, we treat them as non-side-effecting. + if (llvm::isa(op)) return true; + + // TensorArray ops have explicit operand/result for handling side effects. + if (IsTensorArrayOp(op)) return true; + + if (include_read_only) { + if (mlir::hasSingleEffect(op) || + mlir::hasSingleEffect(op) || + mlir::hasSingleEffect(op)) + return true; + } + + return mlir::isMemoryEffectFree(op); +} + +bool FunctionHasSideEffect( + mlir::func::FuncOp func_op, bool include_read_only, + llvm::DenseMap& function_side_effect) { + auto iter = function_side_effect.find(func_op); + if (iter != function_side_effect.end()) return iter->second; + + auto& block = func_op.front(); + + auto op_has_side_effect = [&](mlir::Operation* op) { + if (auto while_op = llvm::dyn_cast(op)) { + if (while_op.getIsStateless()) return false; + + return FunctionHasSideEffect(while_op.cond_function(), include_read_only, + function_side_effect) || + FunctionHasSideEffect(while_op.body_function(), include_read_only, + function_side_effect); + } + + if (auto if_op = llvm::dyn_cast(op)) { + if (if_op.getIsStateless()) return false; + + return FunctionHasSideEffect(if_op.else_function(), include_read_only, + function_side_effect) || + FunctionHasSideEffect(if_op.then_function(), include_read_only, + function_side_effect); + } + + return !IsOpNonSideEffectingOrReadOnly(op, include_read_only); + }; + + // Speculatively setting the function to have no side effect to avoid infinite + // recursion. The correct side effect will be updated later once more + // operations in the block are checked. + function_side_effect[func_op] = false; + + for (mlir::Operation& op : block) { + if (op_has_side_effect(&op)) { + function_side_effect[func_op] = true; + return true; + } + } + + function_side_effect[func_op] = false; + return false; +} + +// This pass sets `is_stateless` attribute of tf.If and tf.While ops to true if +// their callee functions contains only non-side-effecting ops. +class OptimizeTfControlFlowSideEffectPass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID( + OptimizeTfControlFlowSideEffectPass) + + private: + llvm::StringRef getArgument() const final { + return "tfrt-optimize-tf-control-flow-side-effect"; + } + llvm::StringRef getDescription() const final { + return "Set tf control flow ops to stateless if their callee functions " + "contains only non-side-effecting ops"; + } + void runOnOperation() override { + auto module = getOperation(); + + llvm::SmallVector functions; + + bool include_read_only = true; + for (auto func : module.getOps()) { + if (func.isPublic() && !IsSessionInitializer(func)) { + functions.push_back(func); + + for (auto& op : func.front()) { + // Skip control flow ops in the first pass. + if (llvm::isa(&op)) continue; + + if (!IsOpNonSideEffectingOrReadOnly(&op)) { + include_read_only = false; + } + } + } + } + + llvm::DenseMap function_side_effect; + + mlir::Builder builder(module.getContext()); + + for (auto func : functions) { + func.walk([&](mlir::Operation* op) { + if (auto while_op = llvm::dyn_cast(op)) { + if (while_op.getIsStateless()) return; + + if (!FunctionHasSideEffect(while_op.cond_function(), + include_read_only, function_side_effect) && + !FunctionHasSideEffect(while_op.body_function(), + include_read_only, function_side_effect)) { + while_op->setAttr("is_stateless", builder.getBoolAttr(true)); + } + } + + if (auto if_op = llvm::dyn_cast(op)) { + if (if_op.getIsStateless()) return; + + if (!FunctionHasSideEffect(if_op.else_function(), include_read_only, + function_side_effect) && + !FunctionHasSideEffect(if_op.then_function(), include_read_only, + function_side_effect)) { + if_op->setAttr("is_stateless", builder.getBoolAttr(true)); + } + } + }); + } + } +}; + +} // namespace + +std::unique_ptr> +CreateOptimizeTfControlFlowSideEffectPass() { + return std::make_unique(); +} + +static mlir::PassRegistration + register_pass(CreateOptimizeTfControlFlowSideEffectPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.cc new file mode 100644 index 0000000000000000000000000000000000000000..bc6c160f778be873a136bd0fd0f3b25e0257222d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.cc @@ -0,0 +1,282 @@ + +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Pass/PassOptions.h" +#include "mlir/Transforms/Passes.h" +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/bridge_logger.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/util/device_name_utils.h" +#include "tsl/platform/errors.h" + +namespace tensorflow { +namespace { + +// Assigns devices so that later passes can utilize device information. +// Device assignment might have not been done by the upstream pipeline, or get +// removed by previous passes. However, we assume most of the device assignment +// has been done by the upstream pipeline, so we simply assign the default +// device to unassigned ops. Specifically, we do assignment for ConstOp first to +// place it on the same device as its user operation, instead of placing it on +// the default device blindly. +// TODO(b/221297389): Figure out a more robust way to handle dropped device +// assignment. +void AddTfDeviceAssignmentPasses(mlir::OpPassManager &pm, + const TfrtPipelineOptions &options) { + pm.addPass(mlir::TF::CreateConstantOpDeviceAssignmentPass()); + pm.addNestedPass( + mlir::TF::CreateTFDeviceAssignmentByFuncAttrPass()); + pm.addNestedPass( + mlir::TF::CreateSimpleTFDeviceAssignmentPass(options.default_device)); +} + +} // namespace + +void CreateTFExecutorToTFPreInvariantOptimizationPipelineHelper( + mlir::OpPassManager &pm, const TfrtPipelineOptions &options) { + // Due to b/191304670, functionalized while ops might not have the + // shape_invariant attribute set correctly, which leads to failure in shape + // inference. As a workaround, we conservatively (e.g., we place less + // restrictions on tf.while which will avoid failures but lead to potentially + // less exact shape inference) set the shape_invariant attribute in all + // tf.While ops before performing shape inference. + // + // Note that this pass might not work well with TF XLA bridge, but this is + // fine as TF XLA bridge is run before this pipeline. For CPU ops, less exact + // shape inference may lead to fewer optimizations but it should be fine as it + // is limited to while ops currently. + // + // TODO(b/191304670): Remove this pass once the shape_invariant attribute is + // set correctly in the upstream. + pm.addNestedPass( + tfrt_compiler::CreateSetShapeInvariantInWhileOps()); + + // We pass the MLIR module through the TF standard pipeline, which for + // instances does shape inference, canonicalization, inlining, etc. + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass()); + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorIslandCoarseningPass()); + + AddTfDeviceAssignmentPasses(pm, options); + + if (options.allow_xla_cpu) { + pm.addPass(tfrt_compiler::CreateTfrtXlaRewritePass()); + } + + // Here we perform TFRT specific optimization before standard TF optimization, + // as TFRT-specific optimization may create more opportunities. + pm.addNestedPass( + tfrt_compiler::CreateOptimizeTfForTfrtPass()); + + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + pm.addNestedPass(mlir::createCanonicalizerPass()); + // Guarantee all functions have one use, which enables more exact shape + // inference. + pm.addPass(mlir::TF::CreateGuaranteeAllFuncsOneUsePass()); + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addPass(mlir::createInlinerPass()); + pm.addPass(mlir::createSymbolDCEPass()); + pm.addNestedPass(mlir::TF::CreateTFOptimizePass()); + pm.addNestedPass(mlir::createCSEPass()); + + AddTfDeviceAssignmentPasses(pm, options); + + // After the standard pass, we now have MLIR in TF dialect, and now we convert + // reference variable to resource variables, which is best effort. + pm.addPass(CreateConvertReferenceVariableToResourceVariablePass()); + + // Move the tf.Assert op to the end of the function, so that it does not + // impose unnecessary control dependencies on other ops. + pm.addPass(tfrt_compiler::CreateReorderTfAssertPass()); + + // Optimize the side-effects of control flow ops by examining the ops in its + // callees. + pm.addPass(tfrt_compiler::CreateOptimizeTfControlFlowSideEffectPass()); + + // Remove tf.If ops' operands that are produced by tf.Const ops. + pm.addPass(tfrt_compiler::CreateRemoveTfIfConstArgsPass()); + + // Merge non-side-effecting tf.If ops if their operands are the same. + pm.addPass(tfrt_compiler::CreateMergeTfIfOpsPass()); + + pm.addPass(tfrt_compiler::CreateReconfigBatchOpPass({ + .min_num_batch_threads = options.min_num_batch_threads, + .min_max_enqueued_batches = options.min_max_enqueued_batches, + .batch_padding_policy = options.batch_padding_policy, + .num_batch_threads = options.num_batch_threads, + .max_batch_size = options.max_batch_size, + .batch_timeout_micros = options.batch_timeout_micros, + .allowed_batch_sizes = options.allowed_batch_sizes, + .max_enqueued_batches = options.max_enqueued_batches, + .low_priority_max_batch_size = options.low_priority_max_batch_size, + .low_priority_batch_timeout_micros = + options.low_priority_batch_timeout_micros, + .low_priority_allowed_batch_sizes = + options.low_priority_allowed_batch_sizes, + .low_priority_max_enqueued_batches = + options.low_priority_max_enqueued_batches, + .num_warmup_batch_threads = options.num_warmup_batch_threads, + .enable_large_batch_splitting = options.enable_large_batch_splitting, + .mixed_priority_batching_policy = options.mixed_priority_batching_policy, + .batch_queue_global_prioritization_num_threads = + options.batch_queue_global_prioritization_num_threads, + .enable_priority_aware_batch_scheduler = + options.enable_priority_aware_batch_scheduler, + .enable_priority_aware_batch_scheduler_resplit = + options.enable_priority_aware_batch_scheduler_resplit, + .enable_batching_task_lazy_cancellation = + options.enable_batching_task_lazy_cancellation, + })); + + // Deduplicate functions invoked by tf.BatchFunction with the same + // shared_name + pm.addPass( + tfrt_compiler::CreateDeduplicateFunctionsInovkedByBatchFunctionPass()); + + // RemoveUnusedWhileResultsPass operates on the region-based control flow, so + // the functional control flow is first converted to region-based control + // flow, which is converted back after the optimization passes are performed. + pm.addPass(mlir::TF::CreateTFFunctionalControlFlowToRegions()); + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass( + mlir::TF::CreateRemoveUnusedWhileResultsPass()); + + // Apply standard optimization after optimizing control flow ops. + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass(mlir::createCSEPass()); + + // TODO(b/187876545): An extra shape inference pass is added because it does + // not work well with tf.Identity op that remove ref type. So we work around + // by performing shape inference again after reference variable to resource + // variable conversion. We should remove this after b/187876545 is fixed. + pm.addPass(mlir::TF::CreateTFShapeInferencePass()); + pm.addPass(mlir::TF::CreateTFRegionControlFlowToFunctional()); + + pm.addNestedPass( + mlir::TFDevice::CreateLaunchToDeviceAttributePass()); + + // After all standard passes run layout optimization to assign optimal data + // format for all layout sensitive operations. + mlir::TF::LayoutOptimizationPipelineOptions layout_optimization_options; + layout_optimization_options.force_data_format = + options.force_data_format.getValue(); + // TODO(b/191304261): Folding transpose in ops is buggy in the layout + // optimization pass. Disable it to avoid errors in b/191304261. This should + // not affect CPU performance as it does not change the number of ops, nor + // does it change the types of the ops. + layout_optimization_options.skip_fold_transpose_in_ops = true; + mlir::TF::CreateLayoutOptimizationPipeline(pm.nest(), + layout_optimization_options); + + // Run canonicalization pipeline to remove unused constants and bypassed + // transpose operations left in the IR after layout optimization. + pm.addNestedPass(mlir::createCanonicalizerPass()); + + // Decompose resource ops as resource variables will be converted to tensors + // directly. + if (options.decompose_resource_ops) + pm.addNestedPass( + mlir::TFDevice::CreateDecomposeResourceOpsPass()); + + AddTfDeviceAssignmentPasses(pm, options); + + pm.addNestedPass( + mlir::TF::CreateTensorDeviceCopyConversionPass()); + + // Rewriter operation sequences to device specific fusions. + DeviceNameUtils::ParsedName parsed_name; + + // Ignore error. + bool success = + DeviceNameUtils::ParseFullName(options.default_device, &parsed_name); + assert(success && "default device is invalid"); + (void)success; + + if (parsed_name.has_type && parsed_name.type == DEVICE_GPU) + pm.addNestedPass(mlir::TF::CreateGpuOpFusionPass()); + + if (parsed_name.has_type && parsed_name.type == DEVICE_CPU) + pm.addNestedPass( + mlir::TF::CreateFusedKernelMatcherPass()); + + if (options.tpu_fuse_ops) { + pm.addNestedPass( + tfrt_compiler::CreateFuseTpuCompileAndExecutePass()); + // Remove ops for the input to _TPUCompileMlirOp, which are no longer needed + // after CreateFuseTpuCompileAndExecutePass + pm.addNestedPass(mlir::createCanonicalizerPass()); + } + + AddTfDeviceAssignmentPasses(pm, options); +} + +void CreateTFInvariantOptimizationPipelineHelper( + mlir::OpPassManager& pm, const TfrtPipelineOptions& options) { + if (options.sink_in_invariant_ops) { + pm.addPass(CreateSinkInInvariantOpsPass()); + } + + if (!options.saved_model_dir.empty()) { + pm.addPass( + mlir::tf_saved_model::CreateAssetSinkingPass(options.saved_model_dir)); + } + + pm.addPass(CreateLowerTFSavedModelPass( + options.hoist_invariant_ops, options.fuse_get_resource_ops_in_hoisting)); +} + +absl::Status ValidateTfrtPipelineOptions(const TfrtPipelineOptions &options) { + if (options.target_tpurt && options.target_gpu) { + return absl::InternalError( + "Invalid pipeline options. Targeting both TPU and GPU is not " + "supported."); + } + return absl::OkStatus(); +} + +absl::Status CreateTFExecutorToTFPreInvariantOptimizationPipeline( + mlir::PassManager &pm, const TfrtPipelineOptions &options) { + TF_RETURN_IF_ERROR(ValidateTfrtPipelineOptions(options)); + if (VLOG_IS_ON(1)) { + // Print the whole module after each pass, which requires disabling + // multi-threading as well. + pm.getContext()->disableMultithreading(); + pm.enableIRPrinting(std::make_unique( + /*print_module_scope=*/true)); + } + CreateTFExecutorToTFPreInvariantOptimizationPipelineHelper(pm, options); + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.h new file mode 100644 index 0000000000000000000000000000000000000000..e2b681c7fd8b969c78e7e413a0c341f876fce79d --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/passes.h @@ -0,0 +1,182 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_PASSES_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassOptions.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "tensorflow/core/platform/status.h" + +namespace mlir { +class PassManager; +} + +namespace tensorflow { + +namespace tfrt_compiler { + +// Create a pass to insert kernels that copy fallback tensors when they are +// passed to multiple threads, to avoid atomic contention on their refcounts. +std::unique_ptr> +CreateInsertFallbackTensorCopyPass(); + +// Create a pass to reorder tf.Assert ops or tf.If ops that contains only +// tf.Assert ops to the end of the function, to avoid unnecessary control +// dependencies to other ops. +std::unique_ptr> +CreateReorderTfAssertPass(); + +// Create a pass to optimize the side-effect of control flow ops. eg. if both +// branches of a tf.If op contains only non-side-effecting ops, its +// `is_stateless` attribute will be set to true. +std::unique_ptr> +CreateOptimizeTfControlFlowSideEffectPass(); + +// Create a pass to remove tf.If ops' operands that are produced by tf.Const +// ops. +std::unique_ptr> +CreateRemoveTfIfConstArgsPass(); + +// Create a pass to merge non-side-effecting tf.If ops that have the same +// operands. +std::unique_ptr> CreateMergeTfIfOpsPass(); + +// Create a pass to deduplicate the function invoked by tf.BatchFunction with +// the same shared_name. +std::unique_ptr> +CreateDeduplicateFunctionsInovkedByBatchFunctionPass(); + +// Create a pass to lower bound the number of threads in tf.BatchFunction. +struct ReconfigBatchOpPassOptions { + int64_t min_num_batch_threads = 1; + int64_t min_max_enqueued_batches = 1; + std::string batch_padding_policy = ""; + int64_t num_batch_threads = 0; + int64_t max_batch_size = 0; + int64_t batch_timeout_micros = 0; + llvm::ArrayRef allowed_batch_sizes = {}; + int64_t max_enqueued_batches = 0; + int64_t low_priority_max_batch_size = 0; + int64_t low_priority_batch_timeout_micros = 0; + llvm::ArrayRef low_priority_allowed_batch_sizes = {}; + int64_t low_priority_max_enqueued_batches = 0; + int64_t num_warmup_batch_threads = 0; + bool enable_large_batch_splitting = false; + std::string mixed_priority_batching_policy = ""; + int64_t batch_queue_global_prioritization_num_threads = 0; + bool enable_priority_aware_batch_scheduler = false; + bool enable_priority_aware_batch_scheduler_resplit = false; + bool enable_batching_task_lazy_cancellation = false; +}; +std::unique_ptr> CreateReconfigBatchOpPass( + ReconfigBatchOpPassOptions options); + +// Create a pass to fuse the TPU Ops for TFRT. +std::unique_ptr> +CreateFuseTpuCompileAndExecutePass(); + +// Create a pass to optimize TF dialect for TFRT workflow. +std::unique_ptr> +CreateOptimizeTfForTfrtPass(); + +std::unique_ptr> CreateTfrtXlaRewritePass(); + +// Create a pass to deduplicate results of tf.If ops. +std::unique_ptr> +CreateDeduplicateIfResultPass(); + +} // namespace tfrt_compiler + +class CoreRTConverter; + +// Create a pass that sink in the var handle op to the callee function when +// proper. +std::unique_ptr> +CreateSinkInInvariantOpsPass(); + +// Create a pass that rewrites tf_saved_model dialect's ops according to TFRT's +// requirements. +std::unique_ptr> +CreateLowerTFSavedModelPass(bool hoist_invariant_ops, + bool fuse_get_resource_ops); + +// Create a pass that converts ref variables to resource variables in a limited +// number of cases. +std::unique_ptr> +CreateConvertReferenceVariableToResourceVariablePass(); + +// Run *ToCoreRTConversionPassRun as free functions. Useful for +// reusing the pass logic in a custom pass with additional conversions. +mlir::LogicalResult TFSavedModelToCoreRTConversionPassRun( + mlir::MLIRContext* context, mlir::func::FuncOp func, + mlir::ConversionTarget* target, mlir::RewritePatternSet* patterns, + CoreRTConverter* corert_converter); + +// Create an operation pass that removes the device attribute from every +// corert.executeop. +std::unique_ptr> +CreateRemoveDeviceAttributePass(); + +// Create an operation pass that inserts corert.transfer op to make sure any +// argument of any op is on the same device of the op itself. +std::unique_ptr> +CreateCrossDeviceTransferPass(); + +// Create a pass that converts MLIR TF dialect to MLIR TFRT dialect. +std::unique_ptr> +CreateTfToTfrtConversionPass(const TfrtPipelineOptions& options); + +// Creates a pipeline of passes that lowers MLIR TF dialect to TFRT dialects. +void CreateTfToTfrtPipeline(mlir::OpPassManager& pm, + const TfrtPipelineOptions& options); + +// Creates a pipeline of passes that lowers MLIR TF dialect from tf.function to +// TFRT dialect. SavedModel related conversions are not included. +absl::Status CreateTfExecutorToTfrtPipeline(mlir::PassManager& pm, + const TfrtPipelineOptions& options); + +// Creates a pipeline of passes that lowers MLIR TF Executor dialect to TF +// dialect for CoreRT purposes. +absl::Status CreateTFExecutorToTFPipeline(mlir::PassManager& pm, + const TfrtPipelineOptions& options); + +// TODO(deqiangc): refactor below helpers once mlrt is OSSed. +void CreateTFExecutorToTFPreInvariantOptimizationPipelineHelper( + mlir::OpPassManager& pm, const TfrtPipelineOptions& options); +void CreateTFInvariantOptimizationPipelineHelper( + mlir::OpPassManager& pm, const TfrtPipelineOptions& options); + +absl::Status CreateTFExecutorToTFPreInvariantOptimizationPipeline( + mlir::PassManager& pm, const TfrtPipelineOptions& options); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reconfig_batch_op.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reconfig_batch_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..bf49bebb8a0102cee5daa20aaf031576fdb76cb9 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reconfig_batch_op.cc @@ -0,0 +1,273 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/core/platform/cpu_info.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +class ReconfigBatchOpPass + : public mlir::PassWrapper> { + public: + explicit ReconfigBatchOpPass(ReconfigBatchOpPassOptions options) + : mlir::PassWrapper>() { + min_num_batch_threads_ = options.min_num_batch_threads; + min_max_enqueued_batches_ = options.min_max_enqueued_batches; + batch_padding_policy_ = options.batch_padding_policy; + num_batch_threads_ = options.num_batch_threads; + max_batch_size_ = options.max_batch_size; + batch_timeout_micros_ = options.batch_timeout_micros; + allowed_batch_sizes_ = options.allowed_batch_sizes; + max_enqueued_batches_ = options.max_enqueued_batches; + low_priority_max_batch_size_ = options.low_priority_max_batch_size; + low_priority_batch_timeout_micros_ = + options.low_priority_batch_timeout_micros; + low_priority_allowed_batch_sizes_ = + options.low_priority_allowed_batch_sizes; + low_priority_max_enqueued_batches_ = + options.low_priority_max_enqueued_batches; + num_warmup_batch_threads_ = options.num_warmup_batch_threads; + enable_large_batch_splitting_ = options.enable_large_batch_splitting; + mixed_priority_batching_policy_ = options.mixed_priority_batching_policy; + batch_queue_global_prioritization_num_threads_ = + options.batch_queue_global_prioritization_num_threads; + enable_priority_aware_batch_scheduler_ = + options.enable_priority_aware_batch_scheduler; + enable_priority_aware_batch_scheduler_resplit_ = + options.enable_priority_aware_batch_scheduler_resplit; + enable_batching_task_lazy_cancellation_ = + options.enable_batching_task_lazy_cancellation; + } + ReconfigBatchOpPass() + : mlir::PassWrapper>() {} + ReconfigBatchOpPass(const ReconfigBatchOpPass& other) + : mlir::PassWrapper>(other) {} + + ReconfigBatchOpPass& operator=(const ReconfigBatchOpPass& other) = delete; + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ReconfigBatchOpPass) + + private: + llvm::StringRef getArgument() const final { return "tfrt-reconfig-batch-op"; } + + llvm::StringRef getDescription() const final { + return "Reconfig batch op such as num_batch_threads and " + "max_enqueued_batches."; + } + + void runOnOperation() override { + if (min_num_batch_threads_ == 0 && min_max_enqueued_batches_ == 0 && + batch_padding_policy_.empty() && num_batch_threads_ == 0 && + max_batch_size_ == 0 && batch_timeout_micros_ == 0 && + allowed_batch_sizes_.empty() && max_enqueued_batches_ == 0 && + low_priority_max_batch_size_ == 0 && + low_priority_batch_timeout_micros_ == 0 && + low_priority_allowed_batch_sizes_.empty() && + low_priority_max_enqueued_batches_ == 0 && + num_warmup_batch_threads_ == 0 && + !enable_priority_aware_batch_scheduler_ && + !enable_priority_aware_batch_scheduler_resplit_ && + !enable_batching_task_lazy_cancellation_) { + return; + } + mlir::ModuleOp module = getOperation(); + module.walk([&](mlir::TF::BatchFunctionOp batch_op) { + int64_t num_batch_threads = batch_op.getNumBatchThreads(); + num_batch_threads = + std::max(num_batch_threads, min_num_batch_threads_.getValue()); + batch_op.setNumBatchThreads(num_batch_threads); + + int64_t max_enqueued_batches = batch_op.getMaxEnqueuedBatches(); + max_enqueued_batches = + std::max(max_enqueued_batches, min_max_enqueued_batches_.getValue()); + batch_op.setMaxEnqueuedBatches(max_enqueued_batches); + + if (!batch_padding_policy_.empty()) { + batch_op.setBatchPaddingPolicy(batch_padding_policy_); + } + if (num_batch_threads_ > 0) { + batch_op.setNumBatchThreads(num_batch_threads_); + } + if (max_batch_size_ > 0) { + batch_op.setMaxBatchSize(max_batch_size_); + } + if (batch_timeout_micros_ > 0) { + batch_op.setBatchTimeoutMicros(batch_timeout_micros_); + } + if (!allowed_batch_sizes_.empty()) { + batch_op.setAllowedBatchSizesAttr( + mlir::Builder(module.getContext()) + .getI64ArrayAttr(allowed_batch_sizes_)); + } + if (max_enqueued_batches_ > 0) { + batch_op.setMaxEnqueuedBatches(max_enqueued_batches_); + } + if (low_priority_max_batch_size_ > 0) { + batch_op.setLowPriorityMaxBatchSize(low_priority_max_batch_size_); + } + if (low_priority_batch_timeout_micros_ > 0) { + batch_op.setLowPriorityBatchTimeoutMicros( + low_priority_batch_timeout_micros_); + } + if (!low_priority_allowed_batch_sizes_.empty()) { + batch_op.setLowPriorityAllowedBatchSizesAttr( + mlir::Builder(module.getContext()) + .getI64ArrayAttr(low_priority_allowed_batch_sizes_)); + } + if (low_priority_max_enqueued_batches_ > 0) { + batch_op.setLowPriorityMaxEnqueuedBatches( + low_priority_max_enqueued_batches_); + } + if (num_warmup_batch_threads_ > 0) { + batch_op.setNumWarmupBatchThreads(num_warmup_batch_threads_); + } + if (enable_large_batch_splitting_) { + batch_op.setEnableLargeBatchSplittingAttr( + mlir::Builder(module.getContext()).getBoolAttr(true)); + } + if (!mixed_priority_batching_policy_.empty()) { + batch_op.setMixedPriorityPolicy(mixed_priority_batching_policy_); + } + if (batch_queue_global_prioritization_num_threads_ > 0) { + batch_op.setNumBatchThreads( + batch_queue_global_prioritization_num_threads_); + batch_op.setNumWarmupBatchThreads(port::MaxParallelism()); + batch_op.setEnablePriorityAwareBatchScheduler(true); + } + if (enable_priority_aware_batch_scheduler_) { + batch_op.setEnablePriorityAwareBatchScheduler(true); + } + if (enable_priority_aware_batch_scheduler_resplit_) { + batch_op.setEnablePriorityAwareBatchSchedulerResplit(true); + } + if (enable_batching_task_lazy_cancellation_) { + batch_op.setEnableBatchingTaskLazyCancellation(true); + } + }); + } + + protected: + mlir::Pass::Option min_num_batch_threads_{ + *this, "tfrt-min-num-batch-threads", llvm::cl::init(1), + llvm::cl::desc("Minimum number of batch threads")}; + mlir::Pass::Option min_max_enqueued_batches_{ + *this, "tfrt-min-max-enqueued-batches", llvm::cl::init(1), + llvm::cl::desc( + "Minimum of the maximum number of outstanding enqueued batches")}; + mlir::Pass::Option batch_padding_policy_{ + *this, "tfrt-batch-padding-policy", llvm::cl::init(""), + llvm::cl::desc("The policy used when padding (or splitting) batches.")}; + mlir::Pass::Option num_batch_threads_{ + *this, "tfrt-num-batch-threads", llvm::cl::init(0), + llvm::cl::desc( + "The number of threads for processing batches in parallel")}; + mlir::Pass::Option max_batch_size_{ + *this, "tfrt-max-batch-size", llvm::cl::init(0), + llvm::cl::desc("The maximum allowed batch size")}; + mlir::Pass::Option batch_timeout_micros_{ + *this, "tfrt-batch-timeout-micros", llvm::cl::init(0), + llvm::cl::desc("The maximum number of microseconds before outputting an " + "incomplete batch")}; + mlir::Pass::ListOption allowed_batch_sizes_{ + *this, "tfrt-allowed-batch-sizes", + llvm::cl::desc("Allowed sizes for padding (or splitting) batches")}; + mlir::Pass::Option max_enqueued_batches_{ + *this, "tfrt-max-enqueued-batches", llvm::cl::init(0), + llvm::cl::desc("The maximum number of batches enqueued for processing " + "before requests are failed fast")}; + mlir::Pass::Option low_priority_max_batch_size_{ + *this, "tfrt-low-priority-max-batch-size", llvm::cl::init(0), + llvm::cl::desc( + "The maximum allowed batch size for low priority requests")}; + mlir::Pass::Option low_priority_batch_timeout_micros_{ + *this, "tfrt-low-priority-batch-timeout-micros", llvm::cl::init(0), + llvm::cl::desc("The maximum number of microseconds before outputting an " + "incomplete batch for low priority requests")}; + mlir::Pass::ListOption low_priority_allowed_batch_sizes_{ + *this, "tfrt-low-priority-allowed-batch-sizes", + llvm::cl::desc("Allowed sizes for padding (or splitting) batches for low " + "priority requests")}; + mlir::Pass::Option low_priority_max_enqueued_batches_{ + *this, "tfrt-low-priority-max-enqueued-batches", llvm::cl::init(0), + llvm::cl::desc("The maximum number of batches enqueued for processing " + "before low priority requests are failed fast")}; + // TODO(b/516818455): Remove warmup threads after in-lining warmup requests. + mlir::Pass::Option num_warmup_batch_threads_{ + *this, "tfrt-num-warmup-batch-threads", llvm::cl::init(0), + llvm::cl::desc( + "The number of threads for processing warmup requests. " + "Useful to process warmup requests without starving the " + "regular batch threads when global scheduler is enabled.")}; + mlir::Pass::Option enable_large_batch_splitting_{ + *this, "tfrt-enable-large-batch-splitting", llvm::cl::init(false), + llvm::cl::desc("If true, enables large batch splitting to reduce " + "padding inefficiency")}; + mlir::Pass::Option mixed_priority_batching_policy_{ + *this, "tfrt-mixed-priority-batching-policy", llvm::cl::init(""), + llvm::cl::desc("Policy for mixed priority batching")}; + mlir::Pass::Option batch_queue_global_prioritization_num_threads_{ + *this, "tfrt-batch-queue-global-prioritization-num-threads", + llvm::cl::init(0), + llvm::cl::desc("If non-zero, all models on this server are switched to " + "use a prioritized batching function using this number of " + "global threads.")}; + mlir::Pass::Option enable_priority_aware_batch_scheduler_{ + *this, "tfrt-enable-priority-aware-batch-scheduler", + llvm::cl::init(false), + llvm::cl::desc("If true, the queue implementation will have a separate " + "subqueue for each criticality.")}; + mlir::Pass::Option enable_priority_aware_batch_scheduler_resplit_{ + *this, "tfrt-enable-priority-aware-batch-scheduler-resplit", + llvm::cl::init(false), + llvm::cl::desc("If true, the queue implementation will allow task " + "resplit for priority aware batch scheduler.")}; + mlir::Pass::Option enable_batching_task_lazy_cancellation_{ + *this, "tfrt-enable-batching-task-lazy-cancellation", + llvm::cl::init(false), + llvm::cl::desc("If true, enable lazy cancellation filtering in the " + "priority-aware batch scheduler.")}; +}; + +} // namespace + +std::unique_ptr> CreateReconfigBatchOpPass( + ReconfigBatchOpPassOptions options) { + return std::make_unique(options); +} + +static mlir::PassRegistration register_pass; + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_device_attribute.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_device_attribute.cc new file mode 100644 index 0000000000000000000000000000000000000000..f706d02f5f620b0a8a81645a3babda3798c52395 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_device_attribute.cc @@ -0,0 +1,87 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This pass removes the device attribute from every corert.executeop. + +#include +#include + +#include "llvm/ADT/StringRef.h" +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime + +namespace tensorflow { + +namespace { + +constexpr const char* kDevice = "device"; + +struct RemoveDeviceAttributePass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RemoveDeviceAttributePass) + + llvm::StringRef getArgument() const final { + return "tfrt-remove-device-attribute"; + } + llvm::StringRef getDescription() const final { + return "This pass removes the device attribute from every corert.executeop"; + } + + void runOnOperation() override; +}; + +void RemoveDeviceAttributePass::runOnOperation() { + ModuleOp module = getOperation(); + + module.walk([&](tfrt::corert::ExecuteOp execute_op) { + SmallVector, 4> op_func_attrs; + SmallVector, 4> op_attrs; + SmallVector, 4> new_op_attrs; + execute_op.getOpFuncAttrs(&op_func_attrs); + execute_op.getOpAttrs(&op_attrs); + for (std::pair attr : op_attrs) { + if (attr.first != kDevice) { + new_op_attrs.push_back(attr); + } + } + if (op_attrs.size() == new_op_attrs.size()) return WalkResult::advance(); + + OpBuilder builder(execute_op); + auto new_execute_op = tfrt::corert::ExecuteOp::create( + builder, execute_op.getLoc(), execute_op.getResultTypes(), + execute_op.getOpHandler(), execute_op.getArguments(), new_op_attrs, + op_func_attrs, execute_op.getOpName()); + execute_op.replaceAllUsesWith(new_execute_op.getResults()); + execute_op.erase(); + return WalkResult::advance(); + }); +} + +} // namespace + +std::unique_ptr> CreateRemoveDeviceAttributePass() { + return std::make_unique(); +} + +static PassRegistration pass; + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_tf_if_const_args.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_tf_if_const_args.cc new file mode 100644 index 0000000000000000000000000000000000000000..66769045fb2b960b7670c6454ea2691dbc9d9922 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/remove_tf_if_const_args.cc @@ -0,0 +1,196 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +// This pass removes tf.If ops' operands that are produced by tf.Const ops. +// These constants can be moved into branches' function body for further +// optimziation. +class RemoveTfIfConstArgs + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RemoveTfIfConstArgs) + + private: + llvm::StringRef getArgument() const final { + return "tfrt-remove-tf-if-const-args"; + } + llvm::StringRef getDescription() const final { + return "Remove const args from tf.If ops"; + } + + void runOnOperation() override { + auto module = getOperation(); + for (auto func_op : + llvm::make_early_inc_range(module.getOps())) { + ProcessFunction(func_op); + } + } + + void ProcessFunction(mlir::func::FuncOp op) { + // Set the insertion point to the current function, as we will insert new + // functions here. + mlir::OpBuilder builder(op); + for (mlir::Operation &op : op.front()) { + auto if_op = llvm::dyn_cast(&op); + if (!if_op) continue; + + // Record the operands that are produced by tf.Const ops. + llvm::SmallVector const_args; + // Record these operands's corresponding operand indices. + llvm::SmallVector const_arg_indices; + // Record the remaining operands that won't be removed. + llvm::SmallVector remaining_args; + for (auto iter : llvm::enumerate(if_op.getInput())) { + mlir::Value operand = iter.value(); + if (auto const_op = operand.getDefiningOp()) { + const_args.push_back(const_op); + const_arg_indices.push_back(iter.index()); + } else { + remaining_args.push_back(operand); + } + } + + if (const_args.empty()) continue; + + RemoveConstArgsFromTfIfOp(builder, if_op, const_args, const_arg_indices, + remaining_args); + } + } + + void RemoveConstArgsFromTfIfOp(mlir::OpBuilder &builder, mlir::TF::IfOp if_op, + llvm::ArrayRef const_args, + llvm::ArrayRef const_arg_indices, + llvm::ArrayRef remaining_args) { + auto branch_suffix = absl::StrCat("_removed_const_args_", id_++); + + // Create wrapper functions with the new arguments (as const args are + // removed) for both then function and else function. + auto new_then_function_name = + CreateBranchFunction(builder, if_op.then_function(), branch_suffix, + const_args, const_arg_indices); + auto new_else_function_name = + CreateBranchFunction(builder, if_op.else_function(), branch_suffix, + const_args, const_arg_indices); + + // Change the if_op's argumetns to the new arguments, branches to new + // branches. Note that the outputs are not changed. + if_op.getInputMutable().assign(remaining_args); + if_op.setThenBranchAttr( + mlir::SymbolRefAttr::get(builder.getContext(), new_then_function_name)); + if_op.setElseBranchAttr( + mlir::SymbolRefAttr::get(builder.getContext(), new_else_function_name)); + } + + llvm::StringRef CreateBranchFunction( + mlir::OpBuilder &builder, mlir::func::FuncOp branch, + absl::string_view branch_suffix, + llvm::ArrayRef const_args, + llvm::ArrayRef const_arg_indices) { + // Get the new function type as const args are removed. + llvm::BitVector const_arg_indices_bv(branch.getNumArguments()); + for (auto i : const_arg_indices) const_arg_indices_bv.set(i); + auto new_branch_type = branch.getFunctionType().getWithoutArgsAndResults( + const_arg_indices_bv, {}); + std::string new_branch_name = + absl::StrCat(branch.getSymName().str(), branch_suffix); + // Create the wrapper function with the new arguments that calls the + // original branch. + auto new_branch = mlir::func::FuncOp::create( + builder, branch.getLoc(), new_branch_name, new_branch_type); + new_branch.setVisibility(mlir::func::FuncOp::Visibility::Private); + + // In its function body, we will add the corresponding const ops and call + // the original branch. + + mlir::OpBuilder::InsertionGuard guard(builder); + auto *block = new_branch.addEntryBlock(); + builder.setInsertionPointToStart(block); + + // Prepare the function arguments of the original branch. + llvm::SmallVector call_args(branch.getNumArguments()); + + // For those removed const args, we copy the tf.Const op, and use that as + // the corresponding argument when calling the original branch. + for (const auto &iter : llvm::zip(const_args, const_arg_indices)) { + auto const_op = + llvm::cast(builder.clone(*std::get<0>(iter))); + unsigned index = std::get<1>(iter); + call_args[index] = const_op; + } + + // For the rest, they are now coming from the wrapper function's arguments + // in the original order. + for (int i = 0, j = 0; i < call_args.size(); ++i) { + if (!call_args[i]) { + assert(j < block->getNumArguments()); + call_args[i] = block->getArgument(j++); + } + } + + // Now create the call op to the original branch. + auto call_op = mlir::TF::StatefulPartitionedCallOp::create( + builder, new_branch.getLoc(), new_branch_type.getResults(), call_args, + /*args_attrs=*/nullptr, /*res_attrs=*/nullptr, branch.getSymName(), "", + "", ""); + // Note that the outputs are not changed. + mlir::func::ReturnOp::create(builder, new_branch.getLoc(), + call_op.getOutput()); + + return new_branch.getSymName(); + } + + int id_ = 0; +}; + +} // namespace + +std::unique_ptr> +CreateRemoveTfIfConstArgsPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateRemoveTfIfConstArgsPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reorder_assert.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reorder_assert.cc new file mode 100644 index 0000000000000000000000000000000000000000..0a49e0cce711ac8236b08850bdcd2d01b36ee6f8 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/reorder_assert.cc @@ -0,0 +1,114 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +// Reorder tf.Assert ops or tf.If ops that contains only tf.Assert ops to the +// end of the function, in order to avoid unnecessary control dependencies +// between tf.Assert and other ops. +class ReorderTfAssertPass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ReorderTfAssertPass) + + llvm::StringRef getArgument() const final { return "tfrt-reorder-tf-assert"; } + llvm::StringRef getDescription() const final { + return "Move tf.Assert to the end of the function to avoid unnecessary " + "control dependencies"; + } + + void runOnOperation() override { + auto module = getOperation(); + for (auto func_op : module.getOps()) { + ProcessFunction(func_op); + } + } + + void ProcessFunction(mlir::func::FuncOp func_op) { + auto& block = func_op.front(); + + llvm::SmallVector assert_ops; + for (mlir::Operation& op : block) { + if (auto assert_op = llvm::dyn_cast(&op)) { + assert_ops.push_back(assert_op); + } + + if (auto if_op = llvm::dyn_cast(&op)) { + if (IsAssertOnlyIfOp(if_op)) { + assert_ops.push_back(if_op); + } + } + } + + auto& return_op = block.back(); + + for (auto assert_op : assert_ops) { + assert_op->moveBefore(&return_op); + } + } + + bool IsAssertOnlyIfOp(mlir::TF::IfOp op) { + // If the results of the if op are used by some other ops, we cannot reorder + // it. + if (!op->use_empty()) return false; + + // Only reorder if both branches are non-side-effecting or containing only + // Assert ops. + if (IsFunctionNonSideEffectingOrAssert(op.then_function()) && + IsFunctionNonSideEffectingOrAssert(op.else_function())) + return true; + + return false; + } + + bool IsFunctionNonSideEffectingOrAssert(mlir::func::FuncOp func_op) { + auto& block = func_op.front(); + for (mlir::Operation& op : block) { + if (!llvm::isa(&op) && !mlir::isMemoryEffectFree(&op)) + return false; + } + return true; + } +}; + +} // namespace + +std::unique_ptr> +CreateReorderTfAssertPass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateReorderTfAssertPass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..73e8edd9169fc371dde844afc38e590aebb1feeb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.cc @@ -0,0 +1,57 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.h" + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +class SetShapeInvariantInWhileOps + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SetShapeInvariantInWhileOps) + + void runOnOperation() override { + mlir::func::FuncOp func_op = getOperation(); + + auto shape_invariant = mlir::UnitAttr::get(&getContext()); + + func_op.walk([&](mlir::TF::WhileOp op) { + // Skip tf.While op on TPU. + if (!op->hasAttr("_tpu_replicate")) { + op.setShapeInvariantAttr(shape_invariant); + } + }); + } +}; + +} // namespace + +std::unique_ptr> +CreateSetShapeInvariantInWhileOps() { + return std::make_unique(); +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.h new file mode 100644 index 0000000000000000000000000000000000000000..96617fcad56e30e5a2773e190ffcd08a475ef1f1 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/set_shape_invariant_in_while_ops.h @@ -0,0 +1,35 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_SET_SHAPE_INVARIANT_IN_WHILE_OPS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_SET_SHAPE_INVARIANT_IN_WHILE_OPS_H_ + +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace tensorflow { +namespace tfrt_compiler { + +// Create a pass to set shape_invariant attribute for all tf.While ops except +// those are on TPU. +std::unique_ptr> +CreateSetShapeInvariantInWhileOps(); + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_SET_SHAPE_INVARIANT_IN_WHILE_OPS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/sink_in_invariant_ops.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/sink_in_invariant_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..41346bc846a4c8aadbadb82038d124ad2a65f018 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/sink_in_invariant_ops.cc @@ -0,0 +1,239 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace tensorflow { +namespace { + +// TODO(b/262610234): Generalize the sinking conditions. +// Check if the op qualifies to sink to the callee. +bool IsSinkCandidate(mlir::Operation *op) { + return op && llvm::isa(op); +} + +// Check if the op is allowed to be sinked. We are being conservative here to +// whilelist very limited set of ops here. +struct AllowSinkHelper { + explicit AllowSinkHelper(mlir::Operation* sinked_op, mlir::Operation* user, + int arg_index) { + if (llvm::isa(user)) { + allow_sink_to = true; + callee_arg_index = arg_index; + return; + } + + // We tend to limit this support on WhileOp to only VarHandleOp to satisfy + // IFRT lowering requirements. + // Sinking other invariants like ConstOp is error-prone because it requires + // non-trivial effort to avoid sinking Consts when they are used by cond + // function and we don't need such support. + if (llvm::isa(sinked_op) && + llvm::isa(user)) { + allow_sink_to = true; + callee_arg_index = arg_index; + return; + } + + if (llvm::isa(user) && arg_index > 0) { + allow_sink_to = true; + callee_arg_index = arg_index - 1; + return; + } + } + + bool allow_sink_to = false; + int callee_arg_index = 0; +}; + +llvm::SmallVector FindValueInCallees( + const mlir::SymbolTable &symbol_table, + const mlir::SymbolUserMap &symbol_users, mlir::Operation *caller, + int arg_index) { + llvm::SmallVector values; + llvm::SmallDenseSet callees; + for (const auto &named_attr : caller->getAttrs()) { + if (auto symbol_attr = + mlir::dyn_cast(named_attr.getValue())) { + auto symbol = symbol_attr.getValue(); + + auto callee = symbol_table.lookup(symbol); + if (!callee) continue; + + // One callee invoked by multiple caller is skipped for simplicity. + // Consider adding support if more usage are observed from production. + if (llvm::ArrayRef users = + symbol_users.getUsers(callee); + users.size() > 1) + continue; + + // Invoked by same caller multiple times, only process the first one. + if (!callees.insert(symbol).second) continue; + + values.push_back(callee.getArgument(arg_index)); + } + } + return values; +} + +void FindSinkTarget( + const mlir::SymbolTable &symbol_table, + const mlir::SymbolUserMap &symbol_users, mlir::OpResult original, + mlir::Value value, + llvm::DenseMap> + &targets) { + for (mlir::OpOperand &use : value.getUses()) { + auto *user = use.getOwner(); + + AllowSinkHelper helper(original.getDefiningOp(), user, + use.getOperandNumber()); + + if (helper.allow_sink_to) { + auto values = FindValueInCallees(symbol_table, symbol_users, user, + helper.callee_arg_index); + for (auto value : values) { + FindSinkTarget(symbol_table, symbol_users, original, value, targets); + } + } else if (value != original) { + // If the sinked op is directly used by ReturnOp, we don't sink it. + // One example is for tf.WhileOp, the input and output of the cond + // function and the body function must be the same. If the cond function + // has an input of type tf.VarHandleOp and it just return the VarHandleOp, + // we don't need to sink it. + if (llvm::isa(user)) { + continue; + } + targets[&use].insert(original); + } + } +} + +// Sink in invariant ops like tf.Const, tf.VarHandleOp and tf.HashTableV2 ops +// into sinkable calls like tf.BatchFunction and tf.If. If there are nested +// calls, the invariant ops will only be copied at the target. +void SinkInInvariantOps(mlir::ModuleOp module) { + mlir::SymbolTable symbol_table(module); + mlir::SymbolTableCollection symbol_table_collection; + mlir::SymbolUserMap symbol_users(symbol_table_collection, module); + + // TODO(b/263191534): Replace with CallOpInterface to handle callees. + // Identify the invariant Op, Caller, Callee FuncOp to update. + + llvm::DenseMap> + targets; + module.walk([&](mlir::Operation *op) { + if (IsSinkCandidate(op)) { + for (auto value : op->getOpResults()) { + FindSinkTarget(symbol_table, symbol_users, value, value, targets); + } + } + }); + + // Clone the sinkable op associated with the func op to the func op + mlir::OpBuilder builder(module); + for (const auto &p : targets) { + if (p.second.size() != 1) continue; + + auto *use = p.first; + + builder.setInsertionPointToStart(use->getOwner()->getBlock()); + + mlir::OpResult original = *p.second.begin(); + auto *new_op = builder.clone(*original.getDefiningOp()); + // Use `use->set()` instead of `replaceAllUsesWith()` on the block argument. + // `replaceAllUsesWith()` would replace ALL uses of the block argument in + // the region, including those that were explicitly skipped (e.g., + // `ReturnOp` uses). Replacing a `ReturnOp` use with a cloned ranked + // resource/constant would change the type of the returned value without + // updating the function signature, leading to a verifier crash. + // + // For example, in the following IR: + // + // func.func private @body( + // %arg0: tensor<*x!tf_type.resource> + // ) -> tensor<*x!tf_type.resource> { + // %0 = "tf.ReadVariableOp"(%arg0) + // : (tensor<*x!tf_type.resource>) -> tensor + // func.return %arg0 : tensor<*x!tf_type.resource> + // } + // + // If %arg0 is a constant with known rank, we only want to sink the use in + // `ReadVariableOp`. Using `use->set()` ensures only that specific use is + // replaced with the cloned ranked constant, while the `ReturnOp` continues + // to use the unranked `%arg0`, preserving signature compliance. + // + // Since the `targets` map is constructed by recursively traversing all uses + // of the invariant value (via `FindSinkTarget`), it is guaranteed to + // capture every single safe leaf use. By iterating over `targets` and + // updating them individually, we ensure all eligible uses are successfully + // sunk without missing any, while safely leaving the ineligible ones (like + // `ReturnOp`) untouched. + use->set(new_op->getResult(original.getResultNumber())); + } +} + +class SinkInInvariantOpsPass + : public mlir::PassWrapper> { + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SinkInInvariantOpsPass) + + llvm::StringRef getArgument() const final { + return "tfrt-sink-in-invariant-ops"; + } + llvm::StringRef getDescription() const final { + return "Sink in the invariant ops to facilitate invariant ops hoisting."; + } + + void runOnOperation() override { + auto module = getOperation(); + SinkInInvariantOps(module); + } +}; + +} // namespace + +std::unique_ptr> +CreateSinkInInvariantOpsPass() { + return std::make_unique(); +} + +static mlir::PassRegistration + sink_in_invariant_ops_pass; + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tf_to_tfrt.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tf_to_tfrt.cc new file mode 100644 index 0000000000000000000000000000000000000000..48cd607619dcacadbf584f89958236adc2796cdb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tf_to_tfrt.cc @@ -0,0 +1,1913 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements lowering of TF dialect to TFRT CoreRuntime ExecuteOp. +// This lowering pass is heavily experimental and incomplete. External code +// should not depend on the code here. And please do not take example on it as +// "the path forward" for this. + +#include +#include +#include +#include +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Attributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OperationSupport.h" +#include "mlir/IR/Types.h" +#include "mlir/Pass/PassOptions.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/Transforms/Passes.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Interfaces/SideEffectInterfaces.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_dialect.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/convert_tensor.h" +#include "tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h" +#include "tensorflow/compiler/mlir/tfrt/analysis/tensor_array_side_effect_analysis.h" +#include "tensorflow/compiler/mlir/tfrt/ir/gpu_ops.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback.h" +#include "tensorflow/compiler/mlir/tfrt/ir/tfrt_fallback_async.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/attr_lowering_utils.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/corert_converter.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/fallback_converter.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/gpu_passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/compiler/tf2xla/tf2xla_defs.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/platform/status.h" +#include "tsl/platform/errors.h" +#include "tfrt/basic_kernels/opdefs/basic_kernels.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/tfrt_base.h" // from @tf_runtime +#include "tfrt/basic_kernels/opdefs/types.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/core_runtime.h" // from @tf_runtime +#include "tfrt/core_runtime/opdefs/types.h" // from @tf_runtime +#include "tfrt/test_kernels/opdefs/test_kernels.h" // from @tf_runtime + +namespace tensorflow { +namespace { + +constexpr unsigned kFallbackBenefit = 1; +constexpr unsigned kCoreRTBenefit = 2; +constexpr char kGpuDeviceName[] = + "/job:localhost/replica:0/task:0/device:GPU:0"; +constexpr char kTFDeviceAttr[] = "tf.device"; +constexpr char kTFRTDeviceAttr[] = "tfrt.device"; +constexpr char kDeviceAttr[] = "device"; +constexpr char kHostAttr[] = "host"; +constexpr char kDeviceTypeTpu[] = "TPU"; +constexpr int64_t kDefaultCheapCost = 1; + +void getDependentConversionDialects(mlir::DialectRegistry ®istry) { + registry.insert(); + mlir::func::registerAllExtensions(registry); +} + +mlir::Value GetFunctionInputChain(mlir::Operation *op) { + auto func_op = op->getParentOfType(); + return func_op.getArgument(0); +} + +llvm::SmallVector AddGpuVariableAndInputTensorTransferOps( + mlir::Operation *op, llvm::SmallVector operands, + mlir::ConversionPatternRewriter &rewriter) { + llvm::SmallVector new_operands; + assert(op->getOperands().size() == operands.size()); + for (int i = 0; i < op->getOperands().size(); ++i) { + if (IsResultVariable(op->getOperand(i), operands[i])) { + auto transfer_variable_op = tfrt::gpu::MaybeTransferVariableOp::create( + rewriter, op->getLoc(), + rewriter.getType(), + mlir::ValueRange{operands[i]}, ArrayRef()); + new_operands.push_back(transfer_variable_op); + } else { + auto transfer_to_device_op = tfrt::gpu::TransferToDeviceOp::create( + rewriter, op->getLoc(), + rewriter.getType(), + mlir::ValueRange{operands[i]}, ArrayRef()); + new_operands.push_back(transfer_to_device_op); + } + } + return new_operands; +} + +llvm::SmallVector AddGpuTransferFromDeviceOps( + mlir::Operation *op, llvm::SmallVector results, + mlir::ConversionPatternRewriter &rewriter) { + assert(results.size() == op->getNumResults()); + llvm::SmallVector new_results; + for (int idx = 0; idx < results.size(); ++idx) { + if (op->getResult(idx).use_empty()) { + // If the result is not used, it is not transferred. + new_results.push_back(results[idx]); + } else { + auto transfer_from_device_op = tfrt::gpu::TransferFromDeviceOp::create( + rewriter, op->getLoc(), + rewriter.getType(), results[idx]); + new_results.push_back(transfer_from_device_op); + } + } + return new_results; +} + +mlir::ArrayAttr ToArrayAttr(const llvm::SmallVector &int_collection, + ::mlir::MLIRContext *context) { + mlir::Builder builder(context); + std::vector array_attr; + array_attr.reserve(int_collection.size()); + for (int int_element : int_collection) { + array_attr.push_back(builder.getI64IntegerAttr(int_element)); + } + return builder.getArrayAttr(llvm::ArrayRef(array_attr)); +} + +class GpuCompileAndExecuteOpConversion + : public mlir::OpConversionPattern { + public: + explicit GpuCompileAndExecuteOpConversion(mlir::MLIRContext *context) + : mlir::OpConversionPattern(context) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::XlaLaunchOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + const auto &xla_compile_device_type = + op->getAttrOfType(tensorflow::kCompileDeviceTypeAttr); + if (!xla_compile_device_type || + xla_compile_device_type.getValue().empty()) { + return op->emitWarning("failed to find the XLA compile device type"); + } + if (xla_compile_device_type.getValue().str() != tensorflow::kGpuDevice) { + return failure(); + } + + llvm::SmallVector resource_indices; + for (int idx : llvm::seq(0, adaptor.getArgs().size())) { + auto operand = adaptor.getOperands()[idx]; + auto original_operand = op.getOperand(idx); + if (IsResultVariable(original_operand, operand)) { + resource_indices.push_back(idx); + } + } + + const auto &xla_function = + op->getAttrOfType("function"); + if (!xla_function) { + return op->emitWarning("failed to find 'function' attribute"); + } + auto func_attr = mlir::dyn_cast(xla_function); + if (!func_attr || func_attr.getValue().empty()) { + return op->emitWarning("failed to find a non-empty 'function' attribute"); + } + + llvm::SmallVector used_output_indices; + for (int idx = 0; idx < op.getNumResults(); ++idx) { + if (!op.getResult(idx).use_empty()) { + used_output_indices.push_back(idx); + } + } + + llvm::SmallVector result_types( + op.getNumResults(), rewriter.getType()); + + auto compile_and_execute_op = tfrt::gpu::CompileAndExecuteOp::create( + rewriter, op.getLoc(), result_types, adaptor.getArgs(), + func_attr.getValue(), ToArrayAttr(resource_indices, getContext()), + ToArrayAttr(used_output_indices, getContext())); + + rewriter.replaceOp(op, compile_and_execute_op->getResults()); + + return success(); + } +}; + +// Convert TF dialect ops to tfrt_fallback.executeop for non-side-effecting ops +// and tfrt_fallback.executeop.seq for side-effecting ops. +// +// For example, +// +// %0 = "tf.MatMul"(%arg0, %arg1) {device = "cpu"} : +// (tensor<3x1xf32>, tensor<1x3xf32>) -> tensor<3x3xf32> +// +// is converted to +// +// %result = tfrt_fallback.executeop device("cpu") +// "tf.MatMul"(%arg0, %arg1) : 1 +// +class FallbackExecuteOpConversion : public mlir::ConversionPattern { + public: + FallbackExecuteOpConversion( + mlir::MLIRContext *context, CoreRTConverter *corert_converter, + tfrt_compiler::FallbackConverter *fallback_converter, + const mlir::SymbolTable *symbol_table, + const tfrt_compiler::CostAnalysis *cost_analysis, + bool tpu_lower_to_fallback, bool target_tpurt, + bool use_gpu_compile_and_execute_op) + : mlir::ConversionPattern(mlir::Pattern::MatchAnyOpTypeTag(), + kFallbackBenefit, context), + corert_converter_(*corert_converter), + fallback_converter_(*fallback_converter), + symbol_table_(*symbol_table), + cost_analysis_(*cost_analysis), + tpu_lower_to_fallback_(tpu_lower_to_fallback), + target_tpurt_(target_tpurt), + use_gpu_compile_and_execute_op_(use_gpu_compile_and_execute_op) {} + + LogicalResult matchAndRewrite( + mlir::Operation *op, ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const override { + if (!UseFallback(op)) return failure(); + + if (target_tpurt_ && IsTpuCompileAndExecuteOps(op)) return failure(); + + corert_converter_.MaterializeDerivedAttributes(op); + + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) + return op->emitWarning("failed to find a non-empty 'device' attribute"); + op->removeAttr(kDeviceAttr); + auto parsed_device_name = + corert_converter_.ParseDeviceName(device.getValue()); + if (!parsed_device_name) + return op->emitWarning("failed to parse the device name"); + + // Convert the function (symbol) attributes to an array of string + // attributes, which represents the function names. + llvm::SmallVector func_attr_keys; + + // If the op is XlaLaunch on GPU, the function attribute will use the + // function name in MLIR, instead of the original function name in the + // function library, because the function could have been changed by bridge, + // e.g., variable lifting. The new MLIR function will need to be exported to + // the function library for runtime to use. + bool use_mlir_func_name = + parsed_device_name->device_type == DEVICE_GPU && + op->getName().getStringRef().str() == "tf.XlaLaunch"; + + mlir::ArrayAttr op_func_attrs = corert_converter_.CreateOpFuncAttrs( + symbol_table_, op->getAttrs(), &func_attr_keys, use_mlir_func_name); + if (!op_func_attrs) { + return op->emitWarning("failed to create func attributes."); + } + + // Remove the function attributes, which have already been processed. + for (const auto &key : func_attr_keys) op->removeAttr(key); + + Builder builder(op->getContext()); + mlir::ArrayAttr op_attrs = CreateTfrtOpAttrs(op->getAttrs(), builder); + if (!op_attrs) return op->emitWarning("failed to lower attributes."); + + mlir::StringAttr op_name = + rewriter.getStringAttr(op->getName().getStringRef()); + + // Ops with _tpu_replicate attribute are TPU ops. + bool is_tpu_op = op->hasAttr("_tpu_replicate") || + llvm::isa(op); + + if ((parsed_device_name->device_type == kDeviceTypeTpu && + !tpu_lower_to_fallback_) || + // Convert ops running on TPU to CoreRT dialect to prevent the creation + // of tfrt_fallback_async.createop for them. + // These ops will be encountered here only when using fallback to run + // TPU models, in which case, these ops are assumed to be in a function + // called by a TPUPartitionedCall op and will be compiled in + // TPUPartitionedCall op via FunctionLibraryRuntime and not be processed + // by BEFExecutor. + // + // We also avoid creating tfrt_fallback_async.createop for all GPU ops + // except for tf.XlaLaunch. This is correct as long as we only run XLA + // clusters on GPU and all other ops on CPU. + is_tpu_op || + (parsed_device_name->device_type == DEVICE_GPU && + op->getName().getStringRef().str() != "tf.XlaLaunch")) { + return ConvertToCoreRTExecuteOp( + op, operands, parsed_device_name->op_handler_name, op_attrs, + op_func_attrs, op_name, rewriter); + } + + // For DEVICE_CPU, DEVICE_DEFAULT, and DEVICE_TPU_SYSTEM, we use fallback. + // Note that TPU bridge should handle all ops that are required to be + // executed on TPU. So if there are still ops that are placed on TPU at this + // stage of lowering TF to TFRT, then these ops are supposed to be executed + // on host. + return ConvertToFallbackExecuteOp(op, operands, device, op_attrs, + op_func_attrs, op_name, + fallback_converter_, rewriter); + } + + private: + // Return true if this op can be lowered to fallback ops. + bool UseFallback(mlir::Operation *op) const { + if (!llvm::isa(op->getDialect())) return false; + + // Below is a blocklist of ops that should not go through CoreRTExecuteOp + // conversion. + // TODO(b/173017701): have a centralized place to hold the information + // whether a TF op should be lowered to FallbackExecute op. + return !llvm::isa(op); + } + + bool IsTpuCompileAndExecuteOps(mlir::Operation *op) const { + return llvm::isa(op); + } + + mlir::LogicalResult ConvertToFallbackExecuteOp( + mlir::Operation *op, ValueRange operands, mlir::StringAttr device, + mlir::ArrayAttr op_attrs, mlir::ArrayAttr op_func_attrs, + mlir::StringAttr op_name, + tfrt_compiler::FallbackConverter &fallback_converter, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::LogicalResult ConvertToCoreRTExecuteOp( + mlir::Operation *op, ValueRange operands, llvm::StringRef op_handler_name, + mlir::ArrayAttr op_attrs, mlir::ArrayAttr op_func_attrs, + mlir::StringAttr op_name, + mlir::ConversionPatternRewriter &rewriter) const; + + CoreRTConverter &corert_converter_; + tfrt_compiler::FallbackConverter &fallback_converter_; + const mlir::SymbolTable &symbol_table_; + const tfrt_compiler::CostAnalysis &cost_analysis_; + bool tpu_lower_to_fallback_; + bool target_tpurt_; + // TODO(b/294895431): Remove the flag and default to the fused op. + bool use_gpu_compile_and_execute_op_; +}; + +mlir::LogicalResult FallbackExecuteOpConversion::ConvertToFallbackExecuteOp( + mlir::Operation *op, ValueRange operands, mlir::StringAttr device, + mlir::ArrayAttr op_attrs, mlir::ArrayAttr op_func_attrs, + mlir::StringAttr op_name, + tfrt_compiler::FallbackConverter &fallback_converter, + mlir::ConversionPatternRewriter &rewriter) const { + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + + // Convert the operands to tfrt_fallback.tf_tensor if needed. + llvm::SmallVector new_operands; + if (mlir::failed(tfrt_compiler::ConvertFallbackOperands( + op, device.getValue(), operands, &new_operands, rewriter))) + return failure(); + + auto fallback_key = + rewriter.getI64IntegerAttr(fallback_converter.GetNextFallbackKey()); + + // Query cost analysis to assign costs. + IntegerAttr cost; + auto parsed_device_name = + corert_converter_.ParseDeviceName(device.getValue()); + bool is_gpu_op = + parsed_device_name && parsed_device_name->device_type == DEVICE_GPU; + if (is_gpu_op) { + // For GPU ops, the host only needs to dispatch them to GPUs, which should + // be relatively cheap for the host. + cost = rewriter.getI64IntegerAttr(kDefaultCheapCost); + } else { + cost = rewriter.getI64IntegerAttr(cost_analysis_.GetCost(op)); + } + + // For now, we only consider GPU XLA clusters in the form of XlaLaunch for + // simplicity. We could extend to support other GPU ops that cann't be XLAed. + bool is_xla_launch_on_gpu = + is_gpu_op && !use_gpu_compile_and_execute_op_ && + op->getName().getStringRef().str() == "tf.XlaLaunch"; + if (is_xla_launch_on_gpu) { + new_operands = + AddGpuVariableAndInputTensorTransferOps(op, new_operands, rewriter); + } + + if (mlir::isMemoryEffectFree(op)) { + auto new_op = tfrt::fallback_async::ExecuteOp::create( + rewriter, op->getLoc(), result_types, new_operands, device, op_attrs, + op_func_attrs, fallback_key, op_name, cost); + fallback_converter.RegisterFallbackOp(new_op); + rewriter.replaceOp(op, new_op.getResults()); + } else { + auto in_chain = corert_converter_.GetLocalSideEffectChain(op, &rewriter); + auto out_chain = in_chain; + + if (tfrt_compiler::IsTensorArrayOp(op)) { + // If it is a tensor array op, we don't need to use + // tfrt_fallback_async.executeop.seq because its operands/results already + // take care of control dependencies. + auto new_op = tfrt::fallback_async::ExecuteOp::create( + rewriter, op->getLoc(), result_types, new_operands, device, op_attrs, + op_func_attrs, fallback_key, op_name, cost); + fallback_converter.RegisterFallbackOp(new_op); + rewriter.replaceOp(op, new_op.getResults()); + } else { + // Create tfrt_fallback.executeop.seq if it is a side-effecting op. + auto new_op = tfrt::fallback_async::ExecuteOpSeq::create( + rewriter, op->getLoc(), corert_converter_.chain_type(), result_types, + in_chain, new_operands, device, op_attrs, op_func_attrs, fallback_key, + op_name, cost); + fallback_converter.RegisterFallbackOp(new_op); + out_chain = new_op.getOutOpChain(); + if (is_xla_launch_on_gpu) { + // TODO(b/262280565): Remove unnecessary data transfers when there are + // multiple XLA clusters. + auto results = + AddGpuTransferFromDeviceOps(op, new_op.getResults(), rewriter); + rewriter.replaceOp(op, results); + } else { + rewriter.replaceOp(op, new_op.getResults()); + } + } + + // Register the converted op so that it can be retrieved by successors. + corert_converter_.RegisterLocalSideEffectChain(op, out_chain); + } + + return success(); +} + +mlir::LogicalResult FallbackExecuteOpConversion::ConvertToCoreRTExecuteOp( + mlir::Operation *op, ValueRange operands, llvm::StringRef op_handler_name, + mlir::ArrayAttr op_attrs, mlir::ArrayAttr op_func_attrs, + mlir::StringAttr op_name, mlir::ConversionPatternRewriter &rewriter) const { + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + + // Convert the operands to tensorhandles if needed. + llvm::SmallVector new_operands; + if (mlir::failed(tfrt_compiler::ConvertCoreRTOperands( + op, operands, &new_operands, rewriter))) + return failure(); + + // Get the op handler, or create one if there does not exist one. Note that + // ConvertOpHandler changes internal state so it can only be called if the + // rewrite is guaranteed to succeed afterwards. + auto op_handler = + corert_converter_.ConvertOpHandler(op, op_handler_name, &rewriter); + if (!op_handler) return failure(); + + if (mlir::isMemoryEffectFree(op)) { + auto new_op = tfrt::corert::ExecuteOp::create( + rewriter, op->getLoc(), result_types, op_handler, new_operands, + op_attrs, op_func_attrs, op_name); + rewriter.replaceOp(op, new_op.getResults()); + } else { + // Create corert.executeop.seq if it is a side-effecting op. + auto new_op = tfrt::corert::ExecuteOpSeq::create( + rewriter, op->getLoc(), corert_converter_.chain_type(), result_types, + op_handler, corert_converter_.GetLocalSideEffectChain(op, &rewriter), + new_operands, op_attrs, op_func_attrs, op_name); + rewriter.replaceOp(op, new_op.getResults()); + + // Register the converted op so that it can be retrieved by successors. + corert_converter_.RegisterLocalSideEffectChain(op, new_op.getOutOpChain()); + } + + return success(); +} + +class FallbackConstOpConversion + : public mlir::OpConversionPattern { + public: + FallbackConstOpConversion(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + corert_converter_(*corert_converter) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::ConstOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + // Some data types are handled separately using a fast path. + if (IsSupportedTfrtNumericDType(op.getDtype()) || + mlir::isa(op.getDtype())) + return failure(); + + // For other data types that do not have a fast path (eg. quantized types), + // we convert it to serialized tensor proto. + + tensorflow::TensorProto tensor_proto; + auto status = ConvertToTensorProto(op.getValue(), &tensor_proto); + if (!status.ok()) return op.emitError(absl::StatusMessageAsCStr(status)); + + rewriter.replaceOpWithNewOp( + op, rewriter.getType(), + tensor_proto.SerializeAsString()); + + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +class FallbackSetResourceOp + : public mlir::OpConversionPattern { + public: + FallbackSetResourceOp(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + corert_converter_(*corert_converter) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::_TfrtSetResourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) + return op->emitWarning("failed to find a non-empty 'device' attribute"); + + // Currently the static resource is always on host CPU. + // + // TODO(chky): Support resource on other devices. + llvm::SmallVector new_operands; + if (mlir::failed(tfrt_compiler::ConvertFallbackOperands( + op, tfrt_compiler::GetDefaultCpuDeviceName(), adaptor.getOperands(), + &new_operands, rewriter))) + return failure(); + + assert(new_operands.size() == 1); + + auto new_op = tfrt::fallback_async::SetResourceOp::create( + rewriter, op.getLoc(), corert_converter_.chain_type(), + corert_converter_.GetLocalSideEffectChain(op, &rewriter), + new_operands[0], device.getValue(), op.getIndex()); + + // Register the converted op so that it can be retrieved by successors. + corert_converter_.RegisterLocalSideEffectChain(op, new_op.getOutCh()); + + rewriter.eraseOp(op); + + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +class FallbackGetResourceOp + : public mlir::OpConversionPattern { + public: + FallbackGetResourceOp(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + corert_converter_(*corert_converter) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::_TfrtGetResourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) + return op->emitWarning("failed to find a non-empty 'device' attribute"); + + llvm::SmallVector result_types( + op.getNumResults(), rewriter.getType()); + + auto ready_chain = tfrt::compiler::NewChainOp::create( + rewriter, op.getLoc(), rewriter.getType()); + + auto new_op = tfrt::fallback_async::GetResourceOp::create( + rewriter, op.getLoc(), corert_converter_.chain_type(), result_types, + ready_chain, device.getValue(), op.getIndices()); + + rewriter.replaceOp(op, new_op.getResults()); + + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +// Convert a tf_device.remote_run op to a tfrt_dist.remote_execute_func op. +// +// For example, +// +// %result = tf_device.remote_run "/job:worker/replica:0/task:0" +// @remote_func(%arg0) : (tensor) -> (tensor) +// +// is converted to +// +// %0 = tfrt_dist.get_distributed_context +// %1 = tfrt_dist.get_task_handle %0 +// {task_name = "/job:worker/replica:0/task:0"} +// %2 = tfrt_dist.test_create_remote_chain_manager %0 +// %3 = tfrt_dist.get_chain_for_task_handle %in_chain, %2, %1 +// %out_op_chain, %results:2 = tfrt_dist.remote_execute_func[%in_chain, %0, %1] +// @remote_func(%3, %arg1): (!tfrt_dist.remote_object_id, !corert.tensorhandle) +// -> (!tfrt_dist.remote_object_id, !corert.tensorhandle) +// %4 = tfrt_dist.set_chain_for_task_handle %out_op_chain, %2, %1, %results#0 +// +class TFDeviceRemoteRunOpConversion + : public mlir::OpConversionPattern { + public: + TFDeviceRemoteRunOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context, + kCoreRTBenefit), + type_converter_(*type_converter), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + tf_device::RemoteRunOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + ModuleOp module = op->getParentOfType(); + SymbolTable symtab(module); + func::FuncOp callee = symtab.lookup(op.getCallee()); + if (!callee) { + op.emitOpError("callee function ") << op.getCallee() << " is not found"; + return failure(); + } + StringAttr host = callee->getAttrOfType(kHostAttr); + if (!host) { + op.emitOpError("callee function ") + << op.getCallee() << " should have the host attribute"; + return failure(); + } + + llvm::SmallVector arguments; + + for (mlir::Value argument : op.getCalleeArgs()) { + arguments.push_back(argument); + } + + llvm::SmallVector result_types; + // The first result of the remote function should be a remote chain which + // is added to the function signature when it is lowered from TF dialect to + // TFRT dialect. + for (mlir::Type type : op.getResultTypes()) { + (void)type_converter_.convertType(type, result_types); + } + + return success(); + } + + private: + mlir::TypeConverter &type_converter_; + CoreRTConverter &corert_converter_; +}; + +// Lowers a tf.BatchFunction to tfrt_fallback.batch_function. +class FallbackBatchFunctionOpConversion + : public mlir::OpConversionPattern { + public: + FallbackBatchFunctionOpConversion(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context, + kFallbackBenefit), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + mlir::TF::BatchFunctionOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + corert_converter_.MaterializeDerivedAttributes(op); + + llvm::SmallVector attr_array; + for (auto &key_and_value : op->getAttrs()) { + StringRef name = key_and_value.getName(); + if (name == "_output_shapes" || name == "f") { + continue; + } + attr_array.push_back(key_and_value); + } + + Builder builder(op.getContext()); + ArrayAttr op_attrs = CreateTfrtOpAttrs(attr_array, builder); + if (!op_attrs) return op.emitWarning("failed to lower attributes."); + + mlir::StringAttr device = op->getAttrOfType(kDeviceAttr); + if (!device || device.getValue().empty()) + return op->emitWarning("failed to find a non-empty 'device' attribute"); + auto parsed_device_name = + corert_converter_.ParseDeviceName(device.getValue()); + if (!parsed_device_name) + return op->emitWarning("failed to parse the device name"); + + llvm::SmallVector new_operands; + if (mlir::failed(tfrt_compiler::ConvertFallbackOperands( + op, device.getValue(), adaptor.getOperands(), &new_operands, + rewriter))) + return failure(); + + llvm::SmallVector result_types( + op->getNumResults(), rewriter.getType()); + + auto new_op = tfrt::fallback_async::BatchFunctionOp::create( + rewriter, op.getLoc(), result_types, new_operands, device, + op.getFAttr(), op_attrs); + rewriter.replaceOp(op, new_op.getResults()); + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +// Lower a tf.Const op that creates a string tensor to a native +// corert.create_string_tensor op. +class FallbackConstDenseTensorOpConversion + : public mlir::OpConversionPattern { + public: + FallbackConstDenseTensorOpConversion(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context, kCoreRTBenefit), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + mlir::TF::ConstOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + if (!IsSupportedTfrtNumericDType(op.getDtype())) return failure(); + + // Only CPU ops can be lowered using this conversion. If there is no device + // assignment, this op is treated as a CPU op and can be lowered. + if (auto parsed_device_name = corert_converter_.ParseDeviceName(op)) + if (parsed_device_name->device_type != DEVICE_CPU) return failure(); + + auto new_op = tfrt::fallback_async::ConstDenseTensorOp::create( + rewriter, op.getLoc(), rewriter.getType(), + mlir::cast(op.getValue())); + rewriter.replaceOp(op, new_op->getResult(0)); + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +// Convert the FuncOp with the following changes to meet TFRT's requirements: +// 1) Convert types for the arguments and the results. +// 2) Add a chain to the arguments. +// 3) Add a chain to the results for side-effects. +// 4) If any argument has a tf.device attribute, change the attribute name +// to tfrt.device. +// 5) If any result has a tf.device attribute, change the attribute name +// to tfrt.device. +// +// The input chain is used to signal visibility of all side-effects before +// calling this function. The output chain is used to signal visibility of all +// side-effects of this function. +class TFRTFuncOpSignatureConversion + : public mlir::OpConversionPattern { + public: + TFRTFuncOpSignatureConversion(mlir::MLIRContext *ctx, + mlir::TypeConverter *type_converter) + : OpConversionPattern(ctx), type_converter_(*type_converter) {} + + LogicalResult matchAndRewrite( + mlir::func::FuncOp func_op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + mlir::FunctionType type = func_op.getFunctionType(); + + // Convert the original function arguments. + mlir::TypeConverter::SignatureConversion converted_signature( + type.getNumInputs()); + // Add a chain as the first input. + converted_signature.addInputs( + rewriter.getType()); + + // Convert the original function results. + SmallVector converted_results; + // Add a chain as the first result. + converted_results.push_back(rewriter.getType()); + + if (failed(type_converter_.convertSignatureArgs(type.getInputs(), + converted_signature)) || + failed(type_converter_.convertTypes(type.getResults(), + converted_results)) || + failed(rewriter.convertRegionTypes(&func_op.getBody(), type_converter_, + &converted_signature))) { + return failure(); + } + + llvm::SmallVector arg_attrs; + // The first input, which is a chain added by this pass, has no attribute. + arg_attrs.emplace_back(); + func_op.getAllArgAttrs(arg_attrs); + // If any argument has a tf.device attribute, change the attribute name to + // tfrt.device. + for (auto &arg_attr : arg_attrs) { + mlir::NamedAttrList arg_attr_list(arg_attr); + if (Attribute device = arg_attr_list.erase(kTFDeviceAttr)) { + arg_attr_list.set(kTFRTDeviceAttr, device); + arg_attr = arg_attr_list.getDictionary(device.getContext()); + } + } + arg_attrs.resize(converted_signature.getConvertedTypes().size()); + + // The first result, which is a chain added by this pass, has no attribute. + llvm::SmallVector result_attrs; + result_attrs.emplace_back(); + func_op.getAllResultAttrs(result_attrs); + // If any result has a tf.device attribute, change the attribute name to + // tfrt.device. + for (auto &result_attr : result_attrs) { + mlir::NamedAttrList result_attr_list(result_attr); + if (Attribute device = result_attr_list.erase(kTFDeviceAttr)) { + result_attr_list.set(kTFRTDeviceAttr, device); + result_attr = result_attr_list.getDictionary(device.getContext()); + } + } + result_attrs.resize(converted_results.size()); + + // Update the function signature in-place. + rewriter.modifyOpInPlace(func_op, [&] { + func_op.setType(mlir::FunctionType::get( + func_op.getContext(), converted_signature.getConvertedTypes(), + converted_results)); + func_op.setAllArgAttrs(arg_attrs); + func_op.setAllResultAttrs(result_attrs); + }); + + return success(); + } + + private: + mlir::TypeConverter &type_converter_; +}; + +// Lower a tf.Const op that creates a string tensor to a native +// corert.create_string_tensor op. +class FallbackConstStringTensorOpConversion + : public mlir::OpConversionPattern { + public: + FallbackConstStringTensorOpConversion(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context, kCoreRTBenefit), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + mlir::TF::ConstOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { // NOLINT + if (!mlir::isa(op.getDtype())) return failure(); + + DenseStringElementsAttr attr = + mlir::cast(op.getValue()); + + llvm::SmallVector values; + values.reserve(attr.getNumElements()); + for (const auto &element : attr.getRawStringData()) + values.push_back(rewriter.getStringAttr( + llvm::StringRef(element.data(), element.size()))); + + // Create the shape attribute from the tensor shape. + ArrayRef shape = op.getValue().getShapedType().getShape(); + llvm::SmallVector dims; + dims.reserve(shape.size()); + auto i64_type = rewriter.getIntegerType(64); + for (auto dim : shape) + dims.push_back(rewriter.getIntegerAttr(i64_type, dim)); + + auto new_op = tfrt::fallback_async::ConstStringTensorOp::create( + rewriter, op.getLoc(), rewriter.getType(), + rewriter.getArrayAttr(dims), rewriter.getArrayAttr(values)); + + rewriter.replaceOp(op, new_op.getResult()); + + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +LogicalResult ConvertFunctionCallOperands( + mlir::Operation *op, ValueRange operands, + llvm::SmallVectorImpl *new_operands, + mlir::ConversionPatternRewriter &rewriter) { + // TODO(b/182232457): Support other devices. + return tfrt_compiler::ConvertFallbackOperands( + op, tfrt_compiler::GetDefaultCpuDeviceName(), operands, new_operands, + rewriter); +} + +// Convert TF call ops (eg. StatefulPartitionedCall) to tfrt.call. +template +class TFRTCallOpConversion : public mlir::OpConversionPattern { + public: + TFRTCallOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + CallOp op, typename CallOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + if (auto xla_must_compile = + op->template getAttrOfType("_XlaMustCompile"); + xla_must_compile && xla_must_compile.getValue()) { + return mlir::failure(); + } + + auto callee = + op.getCallableForCallee().template dyn_cast(); + if (!callee) return failure(); + + llvm::SmallVector new_operands; + new_operands.push_back( + corert_converter_.GetLocalSideEffectChain(op, &rewriter)); + // Currently the converted functions always use !corert.tensorhandle types + // for tensor arguments and results. + // + // TODO(b/175881042): We should avoid the tensor conversion here if the + // operand is !tfrt_fallback.tf_tensor, and it is also used as fallback + // tensor inside the callee function. + if (mlir::failed(ConvertFunctionCallOperands(op, adaptor.getOperands(), + &new_operands, rewriter))) + return failure(); + + llvm::SmallVector result_types; + result_types.push_back(rewriter.getType()); + for (auto type : op.getOperation()->getResultTypes()) { + if (failed(type_converter_.convertType(type, result_types))) + return failure(); + } + + auto new_op = tfrt::compiler::CallOp::create( + rewriter, op.getLoc(), result_types, + callee.getRootReference().getValue(), new_operands); + rewriter.replaceOp(op, new_op.getResults().drop_front()); + + if (!mlir::isMemoryEffectFree(op)) { + // Register the converted op so that it can be retrieved by successors. + // TODO(chky): Add OpTraits or OpInterface, rather than assume first + // result is a chain. + corert_converter_.RegisterLocalSideEffectChain(op, new_op.getResult(0)); + } + + return success(); + } + + private: + mlir::TypeConverter &type_converter_; + CoreRTConverter &corert_converter_; +}; + +// Convert func ReturnOp to tfrt.return. +// +// TODO(chky): conversion to tfrt kernels should come from a common tf_to_tfrt +// library. +class TFRTReturnOpConversion + : public mlir::OpConversionPattern { + public: + TFRTReturnOpConversion(mlir::MLIRContext *context, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + mlir::func::ReturnOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + llvm::SmallVector new_operands; + + // Currently in mlir::TF::SideEffectAnalysis, all terminator ops are treated + // as side-effect ops and they have predecessors but not successors. + // + // TODO(chky): ReturnOp has no side effect. Once the special handling in + // mlir::TF::SideEffectAnalysis is removed, the chains should come from + // side-effecting ops with no successors in the function. + new_operands.push_back( + corert_converter_.GetLocalSideEffectChain(op, &rewriter)); + if (mlir::failed(ConvertFunctionCallOperands(op, adaptor.getOperands(), + &new_operands, rewriter))) + return failure(); + + rewriter.replaceOpWithNewOp(op, new_operands); + return success(); + } + + private: + CoreRTConverter &corert_converter_; +}; + +// Convert tf.Case op to tfrt.Case. +// +// TF dialect: +// %outputs = "tf.Case"(%arg, ...) { branches = [@branch0, @branch1], ...} +// +// lowered TFRT CoreRT dialect: +// %idx_int = corert.tensorhandle_to_int32 %idx +// %out_chain, %outputs = tfrt.case %idx_int [@branch0, @branch1] (%chain, ...) +class TFRTCaseOpConversion : public mlir::OpConversionPattern { + public: + TFRTCaseOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter), + corert_converter_(*corert_converter) {} + + LogicalResult matchAndRewrite( + TF::CaseOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + mlir::ArrayAttr branches = op.getBranches(); + + llvm::SmallVector result_types; + result_types.push_back(corert_converter_.chain_type()); + for (mlir::Type type : op->getResultTypes()) { + if (failed(type_converter_.convertType(type, result_types))) + return failure(); + } + + llvm::SmallVector branch_operands; + branch_operands.push_back( + corert_converter_.GetLocalSideEffectChain(op, &rewriter)); + if (mlir::failed( + ConvertFunctionCallOperands(op, adaptor.getOperands().drop_front(), + &branch_operands, rewriter))) + return failure(); + + mlir::Value index_operand = adaptor.getOperands()[0]; + // TODO(b/182233401): Support TF tensor; remove the conversion op here. + if (mlir::isa(index_operand.getType())) { + // TODO(b/182232457): Support other devices. + index_operand = + + tfrt::fallback_async::FallbackTensorToCoreRTTensorHandleOp::create( + rewriter, op.getLoc(), + rewriter.getType(), + adaptor.getOperands()[0], + tfrt_compiler::GetDefaultCpuDeviceName()) + .getResult(0); + } + if (!mlir::isa(index_operand.getType())) + return op.emitError( + "branch index operand is expected to be a TensorHandle."); + mlir::Value index_value = tfrt::corert::TensorHandleToIntOp::create( + rewriter, op.getLoc(), rewriter.getI32Type(), index_operand); + + auto new_op = + tfrt::compiler::CaseOp::create(rewriter, op.getLoc(), result_types, + index_value, branches, branch_operands); + + rewriter.replaceOp(op, new_op.getBranchOutputs().drop_front()); + return success(); + } + + private: + mlir::TypeConverter &type_converter_; + CoreRTConverter &corert_converter_; +}; + +static mlir::Value GetPredicate(mlir::Operation *op, mlir::Value cond_operand, + mlir::ConversionPatternRewriter &rewriter) { + if (!mlir::isa(cond_operand.getType())) { + cond_operand = tfrt_compiler::ConvertCoreRTTensorHandleToFallbackTensor( + op->getLoc(), tfrt_compiler::GetDefaultCpuDeviceName(), cond_operand, + rewriter); + if (!cond_operand) { + op->emitWarning("failed to convert the cond operand to fallback tensor."); + return {}; + } + } + + return tfrt::fallback_async::PredicateOp::create( + rewriter, op->getLoc(), rewriter.getI1Type(), cond_operand); +} + +class TFRTCondOpConversion : public mlir::OpConversionPattern { + public: + TFRTCondOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + CoreRTConverter *corert_converter) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter), + corert_converter_(*corert_converter) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::IfOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::FlatSymbolRefAttr then_branch = op.getThenBranchAttr(); + mlir::FlatSymbolRefAttr else_branch = op.getElseBranchAttr(); + + llvm::SmallVector result_types; + result_types.push_back(rewriter.getType()); + for (mlir::Type type : op.getOperation()->getResultTypes()) { + if (failed(type_converter_.convertType(type, result_types))) + return failure(); + } + + // Convert the cond tensor to a boolean value so that it can be used by + // tfrt.cond kernel. + auto bool_cond = GetPredicate(op, adaptor.getOperands()[0], rewriter); + if (!bool_cond) return failure(); + + llvm::SmallVector new_operands; + // The first arg of converted branch functions should be !tfrt.chain. + new_operands.push_back( + corert_converter_.GetLocalSideEffectChain(op, &rewriter)); + + if (mlir::failed(ConvertFunctionCallOperands( + op, adaptor.getOperands().drop_front(), &new_operands, rewriter))) + return failure(); + + auto new_op = tfrt::compiler::CondOp::create( + rewriter, op.getLoc(), result_types, bool_cond, then_branch, + else_branch, new_operands); + + // The first result is a !tfrt.chain. + rewriter.replaceOp(op, new_op.getResults().drop_front(1)); + + if (!mlir::isMemoryEffectFree(op)) { + // Register the converted op so that it can be retrieved by successors. + // TODO(chky): Add OpTraits or OpInterface, rather than assume first + // result is a chain. + corert_converter_.RegisterLocalSideEffectChain(op, new_op.getResult(0)); + } + + return success(); + } + + private: + mlir::TypeConverter &type_converter_; + CoreRTConverter &corert_converter_; +}; + +// Convert TF WhileOp to tfrt.while. tfrt.while use a boolean condition and has +// slightly different semantics from tf.While for performance and generality. +// The pseudo code of tfrt.while is as follows: +// +// while(cond) { +// outputs, cond = body(inputs) +// inputs = outputs +// } +// return outputs +// +// So we need to insert extra convertion kernels and merge functions when +// lowering tf.While to tfrt.while. +// +// %result = tf.While(%arg) {cond = @original_cond_fn, body = +// @original_body_fn} +// +// is converted to +// +// func @new_pred_fn(%ch, %arg) { +// %ch0, %cond_tensor = tfrt.call @original_cond_fn(%ch, %arg) +// %cond_bool = tfrt_fallback_async.predicate %cond_tensor +// tfrt.return %ch0, %cond_bool +// } +// +// func @new_while_body(%ch, %arg) { +// %ch0, %result = tfrt.call @original_body_fn(%ch, %arg) +// %ch1, %cond_bool = tfrt.call @new_pred_fn(%ch0, %result) +// tfrt.return %ch1, %result, %cond_bool +// } +// +// %ch0, %first_iter_cond = tfrt.call @new_pred_fn(%ch, %arg) +// %ch1, %result = tfrt.while %first_iter_cond @new_while_body(%ch0, %arg) +// +class TFRTWhileOpConversion + : public mlir::OpConversionPattern { + public: + TFRTWhileOpConversion(mlir::MLIRContext *context, + mlir::TypeConverter *type_converter, + CoreRTConverter *corert_converter, + mlir::SymbolTable *symbol_table, + const tfrt_compiler::TensorArraySideEffectAnalysis + *tensor_array_side_effect_analysis, + bool enable_while_parallel_iterations) + : mlir::OpConversionPattern(context), + type_converter_(*type_converter), + corert_converter_(*corert_converter), + symbol_table_(*symbol_table), + tensor_array_side_effect_analysis_(*tensor_array_side_effect_analysis), + enable_while_parallel_iterations_(enable_while_parallel_iterations) {} + + mlir::LogicalResult matchAndRewrite( + mlir::TF::WhileOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + mlir::FlatSymbolRefAttr cond_fn = op.getCondAttr(); + mlir::FlatSymbolRefAttr body_fn = op.getBodyAttr(); + + llvm::SmallVector while_arg_result_types; + // Insert a chain for side effects as the first argument/result. + while_arg_result_types.push_back( + rewriter.getType()); + for (mlir::Type type : op.getOperation()->getResultTypes()) { + if (failed(type_converter_.convertType(type, while_arg_result_types))) + return failure(); + } + + // Convert the operands to either fallback tensor or corert tensors as + // specified in the option. + llvm::SmallVector new_operands; + if (mlir::failed(ConvertFunctionCallOperands(op, adaptor.getOperands(), + &new_operands, rewriter))) + return failure(); + + // Create the predicate function that calls the original cond function and + // in addition convert the result to a boolean value. + mlir::func::FuncOp pred_fn = + GetPredicateFunction(op, cond_fn, while_arg_result_types, rewriter); + if (!pred_fn) return failure(); + + auto in_chain = corert_converter_.GetLocalSideEffectChain(op, &rewriter); + auto out_chain = in_chain; + + bool has_at_most_tensor_array_effect = HasAtMostTensorArrayEffect(op); + + // Prepare the arguments to call the pred function for the first iteration. + llvm::SmallVector pred_args; + pred_args.push_back( + has_at_most_tensor_array_effect ? GetFunctionInputChain(op) : in_chain); + pred_args.append(new_operands.begin(), new_operands.end()); + + // Insert a call op to call the pred function for the first iteration. + auto call_pred_fn = tfrt::compiler::CallOp::create( + rewriter, op.getLoc(), pred_fn.getFunctionType().getResults(), + pred_fn.getSymName(), pred_args); + + auto pred_chain = call_pred_fn.getResult(0); + auto first_iteration_bool_cond = call_pred_fn.getResult(1); + + mlir::func::FuncOp new_body_fn = GetWhileBodyFunction( + op, body_fn, pred_fn, while_arg_result_types, rewriter); + + // Use the pred chain as the chain to the while body. The rest args should + // be the same as the pred_args. + auto &while_args = pred_args; + while_args[0] = pred_chain; + + int64_t parallel_iterations = + enable_while_parallel_iterations_ ? op.getParallelIterations() : 1; + + auto new_op = tfrt::compiler::WhileOp::create( + rewriter, op.getLoc(), while_arg_result_types, + first_iteration_bool_cond, while_args, new_body_fn.getSymName(), + parallel_iterations); + + rewriter.replaceOp(op, new_op.getResults().drop_front()); + + if (!has_at_most_tensor_array_effect) out_chain = new_op.getResult(0); + + if (!mlir::isMemoryEffectFree(op)) { + // Register the converted op so that it can be retrieved by successors. + // TODO(chky): Add OpTraits or OpInterface, rather than assume first + // result is a chain. + corert_converter_.RegisterLocalSideEffectChain(op, out_chain); + } + return success(); + } + + private: + bool HasAtMostTensorArrayEffect(mlir::TF::WhileOp op) const { + return tensor_array_side_effect_analysis_.HasAtMostTensorArrayEffect( + op.cond_function()) && + tensor_array_side_effect_analysis_.HasAtMostTensorArrayEffect( + op.body_function()); + } + + mlir::func::FuncOp GetPredicateFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr cond_fn, + mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::func::FuncOp GetWhileBodyFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr body_fn, + mlir::func::FuncOp pred_fn, mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::TypeConverter &type_converter_; + CoreRTConverter &corert_converter_; + mlir::SymbolTable &symbol_table_; + const tfrt_compiler::TensorArraySideEffectAnalysis + &tensor_array_side_effect_analysis_; + bool enable_while_parallel_iterations_; +}; + +// Create the pred function that contains a call to the original cond function +// and a predicate kernel that converts the cond tensor to a boolean value. eg. +// +// func @pred_fn(%ch, %arg) { +// %ch0, %cond_tensor = tfrt.call @original_cond_fn(%ch, %arg) +// %cond_bool = tfrt_fallback_async.predicate %cond_tensor +// return %ch0, %cond_bool +// } +// +mlir::func::FuncOp TFRTWhileOpConversion::GetPredicateFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr cond_fn, + mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const { + std::string pred_fn_name = cond_fn.getValue().str() + "/tfrt_predicate"; + + if (auto pred_fn = symbol_table_.lookup(pred_fn_name)) { + return pred_fn; + } + + auto func_op = op->getParentOfType(); + + mlir::ConversionPatternRewriter::InsertionGuard insertion_guard(rewriter); + rewriter.setInsertionPointAfter(func_op); + + std::array pred_result_types = { + rewriter.getType(), rewriter.getI1Type()}; + + auto func_type = rewriter.getFunctionType(arg_types, pred_result_types); + + auto pred_fn = mlir::func::FuncOp::create(rewriter, op.getLoc(), pred_fn_name, + func_type); + + auto *block = pred_fn.addEntryBlock(); + rewriter.setInsertionPointToStart(block); + + // There are at least two arguments, with the first being !tfrt.chain and the + // second being either !tfrt_fallback.tf_tensor or !corert.tensorhandle. + // cond_fn must have two results. The first of them must also be !tfrt.chain + // and the second must have the same tensor type as arguments. So we can just + // use the first two arg types as the result types. + assert(arg_types.size() >= 2); + auto cond_result_types = arg_types.take_front(2); + + auto call_cond_fn = tfrt::compiler::CallOp::create( + rewriter, op.getLoc(), cond_result_types, cond_fn, block->getArguments()); + + auto chain = call_cond_fn.getResult(0); + auto cond = call_cond_fn.getResult(1); + + auto bool_cond = GetPredicate(op, cond, rewriter); + if (!bool_cond) return {}; + + llvm::SmallVector results = {chain, bool_cond}; + + tfrt::compiler::ReturnOp::create(rewriter, op.getLoc(), results); + + symbol_table_.insert(pred_fn); + + return pred_fn; +} + +// Create the new while body function that contains a call to original while +// body and then a call to the pred function. eg. +// +// func @new_while_body(%ch, %arg) { +// %ch0, %result = tfrt.call @original_body(%ch, %arg) +// %ch1, %cond_bool = tfrt.call @pred_function(%ch0, %arg) +// tfrt.return %ch1, %result, %cond_bool +// } +// +mlir::func::FuncOp TFRTWhileOpConversion::GetWhileBodyFunction( + mlir::TF::WhileOp op, mlir::FlatSymbolRefAttr original_body_fn, + mlir::func::FuncOp pred_fn, mlir::TypeRange arg_types, + mlir::ConversionPatternRewriter &rewriter) const { + int64_t parallel_iterations = + enable_while_parallel_iterations_ ? op.getParallelIterations() : 1; + + std::string body_fn_name = original_body_fn.getValue().str() + "/tfrt_body_" + + absl::StrCat(parallel_iterations); + + if (auto body_fn = symbol_table_.lookup(body_fn_name)) { + return body_fn; + } + + auto func_op = op->getParentOfType(); + + mlir::ConversionPatternRewriter::InsertionGuard insertion_guard(rewriter); + rewriter.setInsertionPointAfter(func_op); + + auto while_result_types = arg_types; + + llvm::SmallVector body_result_types(arg_types.begin(), + arg_types.end()); + + // The last result of the while body function is the boolean condition. + body_result_types.push_back(rewriter.getI1Type()); + auto func_type = rewriter.getFunctionType(arg_types, body_result_types); + auto body_fn = mlir::func::FuncOp::create(rewriter, op.getLoc(), body_fn_name, + func_type); + if (parallel_iterations > 1) { + // Disable stream merging by setting cost threshold to 1. The key to + // parallelize while iterations is to execute iteration index handling (e.g. + // ++i; i < max_iterations) in parallel to loop bodies. Since iteration + // index handling is usually cheap when this while loop is parallelizable, + // we don't want to merge it with loop bodies into one stream of inline + // execution. The quickest way to achieve this is to disable stream merging + // for loop functions. The potential downside is stream merging won't be + // applied to other part of the loop body, so there might be excessive + // threading overhead. + // + // TODO(chky): Consider a better way of parallizing while ops, so that we + // can have stream merging applied within the loop body. One option is to + // perform compiler transformation to extract iteration index handling logic + // out of the loop body and convert it to a parallel_map-like op. + body_fn->setAttr("tfrt.cost_threshold", rewriter.getI64IntegerAttr(1)); + } + + auto *block = body_fn.addEntryBlock(); + rewriter.setInsertionPointToStart(block); + + // Insert a call to the original body function. + auto call_original_body_fn = + tfrt::compiler::CallOp::create(rewriter, op.getLoc(), while_result_types, + original_body_fn, block->getArguments()); + + // Insert a call to the pred function, which contains a call to the original + // cond function and the predicate kernel that converts the tensor to boolean + // value. + auto call_pred_fn = tfrt::compiler::CallOp::create( + rewriter, op.getLoc(), pred_fn.getFunctionType().getResults(), + pred_fn.getSymName(), call_original_body_fn.getResults()); + + auto pred_chain = call_pred_fn.getResult(0); + + llvm::SmallVector body_results; + // The first result should be the chain returned from the pred function. + body_results.push_back(pred_chain); + // Then comes the results from the orignal while body excluding the first + // chain (which is replaced by the `pred_chain`). + for (auto res : llvm::drop_begin(call_original_body_fn.getResults())) { + body_results.push_back(res); + } + // The last result should be the boolean value converted from the condition. + auto bool_cond = call_pred_fn.getResult(1); + body_results.push_back(bool_cond); + + tfrt::compiler::ReturnOp::create(rewriter, op.getLoc(), body_results); + + symbol_table_.insert(body_fn); + + return body_fn; +} + +// Helper function for specifying legal dialects for conversion to CoreRT. +void SetUpTFToTFRTConversionLegality(mlir::ConversionTarget *target, + mlir::TypeConverter *func_type_converter, + mlir::Type chain_type) { + target->addLegalDialect(); + target->addLegalDialect(); + target->addLegalDialect(); + target->addLegalDialect(); + target->addIllegalDialect(); + target->addIllegalDialect(); + target->addDynamicallyLegalOp([func_type_converter, + chain_type]( + func::FuncOp op) { + auto func_type = op.getFunctionType(); + if (func_type.getNumInputs() == 0 || func_type.getInput(0) != chain_type) + return false; + if (func_type.getNumResults() == 0 || func_type.getResult(0) != chain_type) + return false; + + return func_type_converter->isSignatureLegal(op.getFunctionType()) && + func_type_converter->isLegal(&op.getBody()); + }); +} + +// Helper function for inserting TF dialect to TFRT dialect op conversion +// patterns. +void PopulateTFToTFRTConversionPatterns( + mlir::MLIRContext *context, mlir::RewritePatternSet *patterns, + CoreRTConverter *corert_converter, + tfrt_compiler::FallbackConverter *fallback_converter, + mlir::SymbolTable *symbol_table, + const tfrt_compiler::CostAnalysis *cost_analysis, + const tfrt_compiler::TensorArraySideEffectAnalysis + *tensor_array_side_effect_analysis, + bool enable_while_parallel_iterations, bool tpu_lower_to_fallback, + bool target_tpurt, bool use_gpu_compile_and_execute_op) { + // By default, we lower all TF ops to fallback ops. + patterns->add( + context, corert_converter, fallback_converter, symbol_table, + cost_analysis, tpu_lower_to_fallback, target_tpurt, + use_gpu_compile_and_execute_op); + patterns->add(context, corert_converter); + if (use_gpu_compile_and_execute_op) { + patterns->add(context); + } + + // For control flow ops, we handle them according to the option. + mlir::TypeConverter *func_type_converter; + func_type_converter = fallback_converter; + patterns->add(context, func_type_converter); + patterns->add(context, corert_converter); + patterns->add( + context, func_type_converter, corert_converter, symbol_table, + tensor_array_side_effect_analysis, enable_while_parallel_iterations); + patterns->add, + TFRTCallOpConversion, + TFRTCallOpConversion, + TFRTCaseOpConversion, TFRTCondOpConversion>( + context, func_type_converter, corert_converter); + + // For tf.BatchFunction, we need a special fallback op to batch a BEF + // function. + patterns->add(context, corert_converter); + + // Below patterns are preferred over fallback lowering as we want to use + // CoreRT interface for native kernels. This is only temporary and it will + // refactored to use SSOT interface. + + // Here we use specialized patterns for tf.Const on CPU as it is incorrect to + // use ExecuteOp pattern to convert string tensor attribute. + patterns->add(context, + corert_converter); +} + +// Lower TF dialect MLIR to TFRT dialect. +class TfToTfrtConversionPass + : public mlir::PassWrapper> { + void getDependentDialects(mlir::DialectRegistry ®istry) const override { + getDependentConversionDialects(registry); + + if (target_tpurt_) RegisterTPUDialects(®istry); + if (target_gpu_) RegisterGpuDialects(®istry); + } + + llvm::StringRef getArgument() const final { return "tf-to-tfrt"; } + llvm::StringRef getDescription() const final { + return "Convert Tensorflow dialect (generated from tf.function) to TFRT " + "dialect."; + } + + public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TfToTfrtConversionPass) + + TfToTfrtConversionPass() = default; + explicit TfToTfrtConversionPass(const TfrtPipelineOptions &options) { + target_tpurt_ = options.target_tpurt; + tpu_use_core_selector_ = options.tpu_use_core_selector; + tpu_use_bundled_transfer_ = options.tpu_use_bundled_transfer; + tpu_lower_to_fallback_ = options.tpu_lower_to_fallback; + tpu_transfer_result_to_host_ = options.tpu_transfer_result_to_host; + use_tpu_host_allocator_for_inputs_ = + options.use_tpu_host_allocator_for_inputs; + tpu_allow_unpadded_batch_ = options.tpu_allow_unpadded_batch; + cost_threshold_ = options.cost_threshold; + merge_inter_dependent_streams_ = options.merge_inter_dependent_streams; + enable_while_parallel_iterations_ = + options.enable_while_parallel_iterations; + target_gpu_ = options.target_gpu; + use_gpu_compile_and_execute_op_ = options.use_gpu_compile_and_execute_op; + } + TfToTfrtConversionPass(const TfToTfrtConversionPass &) {} + + mlir::LogicalResult runOnFunction( + mlir::func::FuncOp func, + const mlir::TF::SideEffectAnalysis::Info &side_effect_analysis, + const tfrt_compiler::TensorArraySideEffectAnalysis + &tensor_array_side_effect_analysis, + tfrt_compiler::FallbackConverter &fallback_converter, + mlir::SymbolTable &symbol_table) { + auto &context = getContext(); + mlir::ConversionTarget target(context); + mlir::RewritePatternSet patterns(&getContext()); + CoreRTConverter corert_converter(&context, &side_effect_analysis); + tfrt_compiler::CostAnalysis cost_analysis(func); + + if (target_tpurt_) + AddTPUTargetDialectAndPatterns( + &target, &patterns, &context, &corert_converter, &fallback_converter, + TfrtTpuExecuteOpConversionOptions{ + tpu_use_core_selector_, tpu_use_bundled_transfer_, + tpu_transfer_result_to_host_, use_tpu_host_allocator_for_inputs_, + tpu_allow_unpadded_batch_}, + tpu_lower_to_fallback_); + + if (target_gpu_) { + AddGpuTargetDialectAndPatterns(&context, &target, &patterns); + } + + mlir::TypeConverter *func_type_converter; + func_type_converter = &fallback_converter; + SetUpTFToTFRTConversionLegality(&target, func_type_converter, + corert_converter.chain_type()); + + PopulateTFToTFRTConversionPatterns( + &context, &patterns, &corert_converter, &fallback_converter, + &symbol_table, &cost_analysis, &tensor_array_side_effect_analysis, + enable_while_parallel_iterations_, tpu_lower_to_fallback_, + target_tpurt_, use_gpu_compile_and_execute_op_); + + return mlir::applyPartialConversion(func, target, std::move(patterns)); + } + + void runOnOperation() override { + auto module = getOperation(); + const auto &side_effect_analysis = + getAnalysis(); + + tfrt_compiler::TensorArraySideEffectAnalysis + tensor_array_side_effect_analysis(module); + tfrt_compiler::FallbackConverter fallback_converter(&getContext()); + + mlir::SymbolTable symbol_table(module); + + auto func_op_range = module.getOps(); + llvm::SmallVector func_ops(func_op_range.begin(), + func_op_range.end()); + for (auto func : func_ops) { + if (!func.isExternal()) { + if (mlir::failed(runOnFunction( + func, side_effect_analysis.GetAnalysisForFunc(func), + tensor_array_side_effect_analysis, fallback_converter, + symbol_table))) { + signalPassFailure(); + return; + } + + ChainDanglingValuesinFunction(func); + } + } + + CreateFallbackInitializationFunction(module, fallback_converter); + + // Set cost threshold as a module attribute. It will be used later in the + // runtime to decide whether certain execution is cheap enough to be inline + // executed. + mlir::Builder builder(module); + module->setAttr("tfrt.cost_threshold", + builder.getI64IntegerAttr(cost_threshold_)); + module->setAttr("tfrt.merge_inter_dependent_streams", + builder.getBoolAttr(merge_inter_dependent_streams_)); + } + + private: + // Chain all dangling values (ie. values with no users) together and merge it + // with the first returned chain. This merged chain can be used to signal the + // completion of all execution including side-effets. + void ChainDanglingValuesinFunction(mlir::func::FuncOp func_op) { + auto &block = func_op.front(); + + llvm::SmallVector dangling_values; + + // Find dangling function arguments. + for (auto arg : block.getArguments()) { + if (arg.use_empty()) { + dangling_values.push_back(arg); + } + } + + // Find dangling values produced by ops in the function. + for (auto &op : block) { + for (mlir::Value result : op.getResults()) { + if (result.use_empty()) { + dangling_values.push_back(result); + } + } + } + + if (dangling_values.empty()) return; + + // Merge these dangling values with the first returned chain. + auto return_op = + llvm::cast(block.getTerminator()); + auto chain = return_op->getOperand(0); + assert(mlir::isa(chain.getType())); + dangling_values.push_back(chain); + + mlir::OpBuilder builder(return_op); + + auto new_chain = tfrt::compiler::MergeChainsOp::create( + builder, return_op->getLoc(), + builder.getType(), dangling_values); + + return_op->setOperand(0, new_chain); + } + + void CreateFallbackInitializationFunction( + mlir::ModuleOp module, + tfrt_compiler::FallbackConverter &fallback_converter) { + mlir::OpBuilder builder(&module.getBodyRegion()); + + auto chain_type = builder.getType(); + + auto func_op = mlir::func::FuncOp::create( + builder, module.getLoc(), "_tfrt_fallback_init", + mlir::FunctionType::get(module.getContext(), /*inputs=*/chain_type, + /*results=*/chain_type)); + + auto *block = func_op.addEntryBlock(); + builder.setInsertionPointToStart(block); + + mlir::Value chain_value = block->getArgument(0); + + // Create operations for all fallback kernels in the module. + for (auto *op : fallback_converter.GetFallbackOps()) { + auto create_op = tfrt::fallback_async::CreateOp::create( + builder, func_op.getLoc(), chain_type, chain_value); + + create_op->setAttrs(op->getAttrs()); + create_op->setAttr("num_args", builder.getI64IntegerAttr(GetNumArgs(op))); + + chain_value = create_op; + } + + tfrt::compiler::ReturnOp::create(builder, func_op.getLoc(), chain_value); + } + + int64_t GetNumArgs(mlir::Operation *fallback_op) { + if (auto execute_op = + llvm::dyn_cast(fallback_op)) { + return execute_op.getArgs().size(); + } else if (auto execute_op_seq = + llvm::dyn_cast( + fallback_op)) { + return execute_op_seq.getArgs().size(); + } else if (auto execute_op_allocator = + llvm::dyn_cast( + fallback_op)) { + return execute_op_allocator.getArgs().size(); + } else if (auto execute_op_seq_allocator = llvm::dyn_cast< + tfrt::fallback_async::ExecuteOpSeqWithAllocator>( + fallback_op)) { + return execute_op_seq_allocator.getArgs().size(); + } + llvm_unreachable("invalid fallback op type"); + } + + Option target_tpurt_{*this, "target-tpurt", + llvm::cl::desc("Target TPURT dialect if true."), + llvm::cl::init(false)}; + + Option tpu_use_core_selector_{ + *this, "tpu-use-core-selector", + llvm::cl::desc("If true, use ServingCoreSelector to pick TPU core. " + "Otherwise, use the assigned core."), + llvm::cl::init(true)}; + + Option tpu_use_bundled_transfer_{ + *this, "tpu-use-bundled-transfer", + llvm::cl::desc("If true, use BundledTransferToTpuOp to transfer " + "variables and input tensors to TPU."), + llvm::cl::init(true)}; + + Option tpu_lower_to_fallback_{ + *this, "tpu-lower-to-fallback", + llvm::cl::desc("If true, lower a TF op that's placed on TPU device " + "to be executed by tfrt_fallback.execute. Note that this " + "applies to ops other than TPU compile and execute ops."), + llvm::cl::init(true)}; + + // TODO(b/194081364): remove this option once we unify servo TPU serving + // result transfer behavior. + Option tpu_transfer_result_to_host_{ + *this, "tpu-transfer-result-to-host", + llvm::cl::desc("If true, transfer the result of tpurt.execute from TPU " + "to host."), + llvm::cl::init(true)}; + + Option use_tpu_host_allocator_for_inputs_{ + *this, "use-tpu-host-allocator-for-inputs", + llvm::cl::desc("If true, fallback executeops that produce inputs to tpu " + "program will use tpu host allocator."), + llvm::cl::init(false)}; + + Option tpu_allow_unpadded_batch_{ + *this, "tpu-allow-unpadded-batch", + llvm::cl::desc("To allow unpadded batch for TPU execution."), + llvm::cl::values( + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kDisabled, + "disabled", "Disable this feature."), + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kAuto, "auto", + "Enable this feature when in-graph batching is detected."), + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kEnforced, + "enforced", "Force to enable this feature.")), + llvm::cl::init(TfrtCompileOptions::TpuAllowUnpaddedBatch::kDisabled)}; + + Option target_gpu_{ + *this, "target-gpu", + llvm::cl::desc("If true, target GPU compiler passes."), + llvm::cl::init(false)}; + + // TODO(b/294895431): Remove the flag and default to the fused op. + Option use_gpu_compile_and_execute_op_{ + *this, "use-gpu-compile-and-execute-op", + llvm::cl::desc("If true, gpurt.compile_and_execute is used for GPU"), + llvm::cl::init(false)}; + + Option cost_threshold_{ + *this, "tfrt-cost-threshold", + llvm::cl::desc( + "The cost threshold to decide whether a sequence of operations is " + "cheap, and then whether it can be executed inline."), + llvm::cl::init(1)}; + + Option merge_inter_dependent_streams_{ + *this, "tfrt-merge-inter-dependent-streams", + llvm::cl::desc("If true, streams with inter data depenedencies will be " + "preferred to be merged for inline execution."), + llvm::cl::init(false)}; + + Option enable_while_parallel_iterations_{ + *this, "enable-while-parallel-iterations", + llvm::cl::desc("If true, tf.While op will be parallelized. This is " + "currently experimental."), + llvm::cl::init(false)}; +}; +} // namespace + +std::unique_ptr> +CreateTfToTfrtConversionPass(const TfrtPipelineOptions &options) { + return std::make_unique(options); +} + +// -------------------------------------------------------------------------- // +void CreateTfToTfrtPipeline(mlir::OpPassManager &pm, + const TfrtPipelineOptions &options) { + pm.addPass(CreateTfToTfrtConversionPass(options)); + + pm.addPass(CreateRemoveDeviceAttributePass()); + + // Run optimizer on the MLIR module in CoreRT dialect. + if (options.enable_optimizer) { + pm.addNestedPass(mlir::createCanonicalizerPass()); + pm.addPass(mlir::createInlinerPass()); + pm.addNestedPass(mlir::createCSEPass()); + pm.addNestedPass( + tfrt_compiler::CreateInsertFallbackTensorCopyPass()); + } +} + +static void CreateTfExecutorToTfrtPipelineHelper( + mlir::OpPassManager &pm, const TfrtPipelineOptions &options) { + CreateTFExecutorToTFPreInvariantOptimizationPipelineHelper(pm, options); + CreateTFInvariantOptimizationPipelineHelper(pm, options); + CreateTfToTfrtPipeline(pm, options); +} + +// If verbose logging is on, dump the output of each pass to a file directory, +// set via env var TF_DUMP_GRAPH_PREFIX. e.g.: +// export TF_DUMP_GRAPH_PREFIX=/tmp/mlir +absl::Status CreateTfExecutorToTfrtPipeline( + mlir::PassManager &pm, const TfrtPipelineOptions &options) { + TF_RETURN_IF_ERROR( + CreateTFExecutorToTFPreInvariantOptimizationPipeline(pm, options)); + CreateTFInvariantOptimizationPipelineHelper(pm, options); + CreateTfToTfrtPipeline(pm, options); + return absl::OkStatus(); +} + +absl::Status CreateTFExecutorToTFPipeline(mlir::PassManager &pm, + const TfrtPipelineOptions &options) { + TF_RETURN_IF_ERROR( + CreateTFExecutorToTFPreInvariantOptimizationPipeline(pm, options)); + CreateTFInvariantOptimizationPipelineHelper(pm, options); + return absl::OkStatus(); +} + +static mlir::PassRegistration tf_to_tfrt_pass; + +static mlir::PassPipelineRegistration tf_pipeline( + "tf-executor-to-tfrt-pipeline", + "Convert Tensorflow Executor dialect to TFRT dialect and " + "also apply necessary optimization passes.", + CreateTfExecutorToTfrtPipelineHelper); + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h new file mode 100644 index 0000000000000000000000000000000000000000..924b56db2dfa1abc2f46baa2bce51e82a32ab3c7 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h @@ -0,0 +1,269 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TFRT_PIPELINE_OPTIONS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TFRT_PIPELINE_OPTIONS_H_ + +#include +#include + +#include "llvm/Support/CommandLine.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" + +namespace tensorflow { + +struct TfrtPipelineOptions + : public mlir::PassPipelineOptions { + Option saved_model_dir{*this, "saved-model-dir", + llvm::cl::desc(""), llvm::cl::init("")}; + Option default_device{ + *this, "default-device", llvm::cl::desc("default device assignment"), + llvm::cl::init("/job:localhost/replica:0/task:0/device:CPU:0")}; + Option enable_optimizer{ + *this, "enable-optimizer", + llvm::cl::desc("run optimization passes on corert dialect"), + llvm::cl::init(false)}; + Option decompose_resource_ops{ + *this, "decompose-resource-ops", + llvm::cl::desc("decompose composite resource ops into ReadVariableOp and " + "non-resource ops. This is currently used in TFRT " + "savedmodel pipeline."), + llvm::cl::init(false)}; + Option force_data_format{ + *this, "force-data-format", + llvm::cl::desc("force data format for all layout sensitive operations")}; + // TODO(tfrt-devs): consider making compiler to figure out whether to fold + // transpose or not instead of exposing the specific option. + Option skip_fold_transpose_in_ops{ + *this, "skip-fold-transpose-in-ops", + llvm::cl::desc("Skip folding transpose operands in Ops which can support " + "different layouts.")}; + Option target_tpurt{*this, "target-tpurt", + llvm::cl::desc("target TPURT dialect if true"), + llvm::cl::init(false)}; + Option tpu_use_core_selector{ + *this, "tpu-use-core-selector", + llvm::cl::desc("If true, use ServingCoreSelector to pick TPU core. " + "Otherwise, use the assigned core. Currently we use " + "core selector for Servo serving use cases."), + llvm::cl::init(true)}; + Option tpu_use_bundled_transfer{ + *this, "tpu-use-bundled-transfer", + llvm::cl::desc("If true, use BundledTransferToTpuOp to transfer " + "variables and input tensors to TPU."), + llvm::cl::init(true)}; + Option tpu_lower_to_fallback{ + *this, "tpu-lower-to-fallback", + llvm::cl::desc("If true, lower an TF op that's placed on TPU device " + "to be executed by tfrt_fallback.execute."), + llvm::cl::init(true)}; + Option tpu_fuse_ops{ + *this, "tpu-fuse-ops", + llvm::cl::desc("If true, use the TPU fused compile_and_execute kernel"), + llvm::cl::init(false)}; + // TODO(b/194081364): remove this option once we unify servo TPU serving + // result transfer behavior. + Option tpu_transfer_result_to_host{ + *this, "tpu-transfer-result-to-host", + llvm::cl::desc("If true, transfer the result of tpurt.execute from TPU " + "to host."), + llvm::cl::init(true)}; + Option use_tpu_host_allocator_for_inputs{ + *this, "use-tpu-host-allocator-for-inputs", + llvm::cl::desc("If true, fallback executeops that produce inputs to tpu " + "program will use tpu host allocator."), + llvm::cl::init(false)}; + Option tpu_allow_unpadded_batch{ + *this, "tpu-allow-unpadded-batch", + llvm::cl::desc("To allow unpadded batch for TPU execution."), + llvm::cl::values( + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kDisabled, + "disabled", "Disable this feature."), + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kAuto, "auto", + "Enable this feature when in-graph batching is detected."), + clEnumValN(TfrtCompileOptions::TpuAllowUnpaddedBatch::kEnforced, + "enforced", "Force to enable this feature.")), + llvm::cl::init(TfrtCompileOptions::TpuAllowUnpaddedBatch::kDisabled)}; + + Option target_gpu{ + *this, "target-gpu", + llvm::cl::desc("If true, target GPU compiler passes."), + llvm::cl::init(false)}; + + // TODO(b/294895431): Remove the flag and default to the fused op. + Option use_gpu_compile_and_execute_op{ + *this, "use-gpu-compile-and-execute-op", + llvm::cl::desc("If true, gpurt.compile_and_execute is used for GPU"), + llvm::cl::init(false)}; + + Option enable_while_parallel_iterations{ + *this, "enable-while-parallel-iterations", + llvm::cl::desc("If true, tf.While op will be parallelized. This is " + "currently experimental."), + llvm::cl::init(false)}; + + Option hoist_invariant_ops{ + *this, "hoist-invariant-ops", + llvm::cl::desc("If true, invariant ops in savedmodels will be hoisted " + "out to run during loading."), + llvm::cl::init(false)}; + + Option fuse_get_resource_ops_in_hoisting{ + *this, "fuse-get-resource-ops-in-hoisting", + llvm::cl::desc("If true, get_resource_op will be fused during hoisting"), + llvm::cl::init(true)}; + + Option sink_in_invariant_ops{ + *this, "sink-in-invariant-ops", + llvm::cl::desc("If true, sink the selected invariant ops in to the " + "nested functions to facilitate invariant ops hoisting."), + llvm::cl::init(false)}; + + Option cost_threshold{ + *this, "tfrt-cost-threshold", + llvm::cl::desc( + "The cost threshold to decide whether a sequence of operations is " + "cheap, and then whether it can be executed inline."), + llvm::cl::init(1)}; + + Option min_num_batch_threads{ + *this, "tfrt-min-num-batch-threads", + llvm::cl::desc("The minimum number of batch threads"), llvm::cl::init(1)}; + + Option min_max_enqueued_batches{ + *this, "tfrt-min-max-enqueued-batches", + llvm::cl::desc( + "The minimum of the maximum number of outstanding enqueued batches"), + llvm::cl::init(1)}; + + Option batch_queue_global_prioritization_num_threads{ + *this, "tfrt-batch-queue-global-prioritization-num-threads", + llvm::cl::desc( + "If non-zero, all models on this server are switched to use a " + "prioritized batching function using this number of global threads."), + llvm::cl::init(0)}; + + Option enable_priority_aware_batch_scheduler{ + *this, "tfrt-enable-priority-aware-batch-scheduler", + llvm::cl::desc("If true, the queue implementation will have a separate " + "subqueue for each criticality."), + llvm::cl::init(false)}; + + Option enable_priority_aware_batch_scheduler_resplit{ + *this, "tfrt-enable-priority-aware-batch-scheduler-resplit", + llvm::cl::desc("If true, the queue implementation will allow task " + "resplit for priority aware batch scheduler."), + llvm::cl::init(false)}; + + Option enable_batching_task_lazy_cancellation{ + *this, "tfrt-enable-batching-task-lazy-cancellation", + llvm::cl::desc("If true, enable lazy cancellation filtering in the " + "priority-aware batch scheduler."), + llvm::cl::init(false)}; + + Option batch_padding_policy{ + *this, "tfrt-batch-padding-policy", + llvm::cl::desc("The policy used when padding (or splitting) batches."), + llvm::cl::init("")}; + + Option num_batch_threads{ + *this, "tfrt-num-batch-threads", + llvm::cl::desc( + "The number of threads for processing batches in parallel"), + llvm::cl::init(0)}; + + Option max_batch_size{ + *this, "tfrt-max-batch-size", + llvm::cl::desc("The maximum allowed batch size"), llvm::cl::init(0)}; + + Option batch_timeout_micros{ + *this, "tfrt-batch-timeout-micros", + llvm::cl::desc("The maximum number of microseconds before outputting an " + "incomplete batch"), + llvm::cl::init(0)}; + + ListOption allowed_batch_sizes{ + *this, "tfrt-allowed-batch-sizes", + llvm::cl::desc("Allowed sizes for padding (or splitting) batches")}; + + Option max_enqueued_batches{ + *this, "tfrt-max-enqueued-batches", + llvm::cl::desc("The maximum number of batches enqueued for processing " + "before requests are failed fast"), + llvm::cl::init(0)}; + + /**experimental options to override legacy priority batching parameters**/ + + // The following options are for legacy priority batching, and are only for + // experimentation. + // TODO(b/488097129): Remove these options after the experiment is done. + Option low_priority_max_batch_size{ + *this, "tfrt-low-priority-max-batch-size", + llvm::cl::desc("The maximum allowed batch size for low priority " + "requests"), + llvm::cl::init(0)}; + + Option low_priority_batch_timeout_micros{ + *this, "tfrt-low-priority-batch-timeout-micros", + llvm::cl::desc("The maximum number of microseconds before outputting an " + "incomplete batch for low priority requests"), + llvm::cl::init(0)}; + + ListOption low_priority_allowed_batch_sizes{ + *this, "tfrt-low-priority-allowed-batch-sizes", + llvm::cl::desc("Allowed sizes for padding (or splitting) batches for low " + "priority requests")}; + + Option low_priority_max_enqueued_batches{ + *this, "tfrt-low-priority-max-enqueued-batches", + llvm::cl::desc("The maximum number of batches enqueued for processing " + "before low priority requests are failed fast"), + llvm::cl::init(0)}; + + /*experimental options end*/ + + Option num_warmup_batch_threads{ + *this, "tfrt-num-warmup-batch-threads", + llvm::cl::desc("The number of threads for processing warmup requests. " + "Useful to process warmup requests without starving the " + "regular batch threads when global scheduler is enabled."), + llvm::cl::init(0)}; + + Option enable_large_batch_splitting{ + *this, "tfrt-enable-large-batch-splitting", + llvm::cl::desc("If true, enables large batch splitting to reduce padding " + "inefficiency"), + llvm::cl::init(false)}; + + Option mixed_priority_batching_policy{ + *this, "tfrt-mixed-priority-batching-policy", + llvm::cl::desc("Policy for mixed priority batching"), llvm::cl::init("")}; + + Option merge_inter_dependent_streams{ + *this, "tfrt-merge-inter-dependent-streams", + llvm::cl::desc("If true, streams with inter data depenedencies will be " + "preferred to be merged for inline execution."), + llvm::cl::init(false)}; + Option allow_xla_cpu{ + *this, + "allow-xla-cpu", + llvm::cl::desc("If true, allow XLA:CPU for CPU computations."), + llvm::cl::init(true), + }; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TFRT_PIPELINE_OPTIONS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h new file mode 100644 index 0000000000000000000000000000000000000000..e2ad84ddd05f18f0f070c9e28eb2645f15d739a4 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h @@ -0,0 +1,87 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TPU_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TPU_PASSES_H_ + +// This file contains stub implementations for Google internal TPU APIs. + +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Support/LogicalResult.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/Pass/PassOptions.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" + +namespace tensorflow { + +class CoreRTConverter; + +namespace tfrt_compiler { + +class FallbackConverter; + +} + +struct TfrtTpuCompileOptions + : mlir::PassPipelineOptions { + Option move_resource_gather_to_host{ + *this, "move-resource-gather-to-host", + llvm::cl::desc("Move resource gather ops to host"), + llvm::cl::init(false)}; + Option gather_table_width_threshold_bytes{ + *this, "gather-table-width-threshold-bytes", + llvm::cl::desc( + "The threshold to control whether a TPU resource gather op should be " + "moved to host. A negative values means all are moved."), + llvm::cl::init(-1)}; +}; + +struct TfrtTpuExecuteOpConversionOptions { + bool use_core_selector = false; + bool use_bundled_transfer = false; + bool transfer_result_to_host = false; + bool use_tpu_host_allocator_for_inputs = false; + TfrtCompileOptions::TpuAllowUnpaddedBatch allow_unpadded_batch = + TfrtCompileOptions::TpuAllowUnpaddedBatch::kDisabled; +}; + +// Registers a set of dialects used in TFRT TPU lowering. +inline void RegisterTPUDialects(mlir::DialectRegistry *registry) {} + +// Adds a target dialect and a set of rewrite patterns for TFRT TPU lowering. +inline void AddTPUTargetDialectAndPatterns( + mlir::ConversionTarget *target, mlir::RewritePatternSet *patterns, + mlir::MLIRContext *context, CoreRTConverter *corert_converter, + tfrt_compiler::FallbackConverter *fallback_converter, + const TfrtTpuExecuteOpConversionOptions &tpu_exec_conv_opts, + bool tpu_lower_to_fallback) {} + +// Rewrites specific TF TPU ops to equivalent TF ops in a module. +inline mlir::LogicalResult RunTPUBackwardCompatConversion( + mlir::ModuleOp module, const TfrtTpuCompileOptions &options) { + return mlir::failure(); +} + +// The rewrite rules to support the fallback execution of TPUPartitionedCallOp. +inline mlir::LogicalResult RunTPUPartitionedCallFallbackCompatConversion( + mlir::ModuleOp module) { + return mlir::failure(); +} + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_TPU_PASSES_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.cc new file mode 100644 index 0000000000000000000000000000000000000000..7839c0871997fe18195a65b554a9c0acc5fcd145 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.cc @@ -0,0 +1,53 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.h" + +#include + +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/analysis/cost_analysis.h" +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" + +namespace tensorflow { +namespace tfrt_compiler { + +constexpr char kCostAttrName[] = "_tfrt_cost"; +constexpr char kOpKeyAttrName[] = "op_key"; + +void UpdateOpCostInTfrtMlir(mlir::ModuleOp op, + const tfrt_stub::CostRecorder& cost_recorder) { + mlir::Builder builder(op); + op.walk([&](mlir::Operation* op) { + // TODO(b/259602527): Add unit test for the precedence. + // Registered cost function has higher priority than online cost analysis. + if (HasCostFunctionRegistered(op->getName().getStringRef())) return; + // Only update ops with existing cost attr. + const auto cost_attr = op->getAttrOfType(kCostAttrName); + if (!cost_attr) return; + // Only fallback ops have `op_key`s. + const auto op_key_attr = + op->getAttrOfType(kOpKeyAttrName); + if (!op_key_attr) return; + // Set the cost attr with a new value. + const int64_t op_key = op_key_attr.getInt(); + op->setAttr(kCostAttrName, builder.getI64IntegerAttr( + cost_recorder.GetCost(op_key))); + }); +} + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.h new file mode 100644 index 0000000000000000000000000000000000000000..7e7eb491748fcf18cbba61245f22c6d7c53a3fc2 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/update_op_cost_in_tfrt_mlir.h @@ -0,0 +1,32 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UPDATE_OP_COST_IN_TFRT_MLIR_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UPDATE_OP_COST_IN_TFRT_MLIR_H_ + +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "tensorflow/core/tfrt/fallback/cost_recorder.h" + +namespace tensorflow { +namespace tfrt_compiler { + +// Updates the existing costs for all the fallback ops with the records in +// `cost_recorder`. +void UpdateOpCostInTfrtMlir(mlir::ModuleOp op, + const tfrt_stub::CostRecorder& cost_recorder); + +} // namespace tfrt_compiler +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UPDATE_OP_COST_IN_TFRT_MLIR_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.cc new file mode 100644 index 0000000000000000000000000000000000000000..53fac2bde0ee412e30d267bf651a40feba4284ce --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.cc @@ -0,0 +1,112 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/transforms/utils.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "llvm/Support/Casting.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/OpDefinition.h" // from @llvm-project +#include "mlir/IR/OperationSupport.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/host_runtime/tfrt_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace tensorflow { + +bool IsResourceArgument(mlir::Value value) { + auto arg = mlir::dyn_cast(value); + if (!arg) return false; + + auto func = llvm::cast(arg.getOwner()->getParentOp()); + + return func.getArgAttr(arg.getArgNumber(), "tf.resource_name") != nullptr; +} + +bool IsResultVariable(const mlir::Value &original_operand, + const mlir::Value &operand) { + if (mlir::isa(original_operand)) { + auto defining_op = original_operand.getDefiningOp(); + + // TODO(b/174753886): When device assignment is properly done, we + // should check that TF::ReadVariableOp is for TPU device here. + if (llvm::isa(defining_op) && + defining_op->getNumOperands() == 1) { + return true; + } else if (llvm::isa(defining_op)) { + return true; + } + return false; + } + return IsResourceArgument(operand); +} + +std::optional CanonicalizeTensorflowFunctionName( + const mlir::SymbolTable &symbol_table, absl::string_view mlir_func_name, + bool use_mlir_func_name) { + if (use_mlir_func_name) { + return std::string(mlir_func_name); + } + + // Currently in TF graph to MLIR importing, a "0" is appended to the original + // function name. The renaming is for TF/XLA v1 bridge use cases. Refer to + // b/142268695, b/141617294 for more context. + // + // TFRT currently uses the original function library. Hence, we retrieve the + // original function name from the function attributes. Longer term, we + // probably want to export the MLIR functions. + auto callee = + symbol_table.lookup(std::string(mlir_func_name)); + if (!callee) return std::nullopt; + + mlir::StringAttr original_func_name = + callee->getAttrOfType("tf._original_func_name"); + if (!original_func_name) { + // If there is no function attribute "tf._original_func_name" in the callee, + // we use the workaround to recover the original function name by removing + // the last char of the MLIR function name. + // TODO(b/259138201): Remove this workwaround after we make sure + // "tf._original_func_name" is present in callees in all code paths. + mlir_func_name.remove_suffix(1); + return std::string(mlir_func_name); + } + + return original_func_name.str(); +} + +bool IsSessionInitializer(mlir::func::FuncOp op) { + auto session_initializer_op = mlir::tf_saved_model::GetSessionInitializerOp( + op->getParentOfType()); + if (!session_initializer_op) return false; + + for (auto sym_ref : session_initializer_op.getInitializers()) { + if (op.getSymName() == + mlir::cast(sym_ref).getValue()) + return true; + } + + return false; +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..999f2cd58bf46f650d6e27de1edab970e1cc8070 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/utils.h @@ -0,0 +1,46 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UTILS_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project + +namespace tensorflow { + +// Checks if the given `value` is a resource argument. +bool IsResourceArgument(mlir::Value value); + +// Checks if an operand is the value of a variable. +bool IsResultVariable(const mlir::Value &original_operand, + const mlir::Value &operand); + +// Canonicalize the symbol attr to the original TF function name. +std::optional CanonicalizeTensorflowFunctionName( + const mlir::SymbolTable &symbol_table, absl::string_view mlir_func_name, + bool use_mlir_func_name = false); + +// Returns true if the function is a session initializer in tf_saved_model +// dialect. +bool IsSessionInitializer(mlir::func::FuncOp op); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSFORMS_UTILS_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/xla_rewrite_pass.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/xla_rewrite_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..20773daa632d9ea9d5a7ddf3e9c22c237a16ba17 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/transforms/xla_rewrite_pass.cc @@ -0,0 +1,121 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "mlir/Support/TypeID.h" // from @llvm-project +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_op_interfaces.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/core/ir/types/dialect.h" + +namespace tensorflow { +namespace tfrt_compiler { +namespace { + +template +struct RewriteFunctionCallToXlaLaunchOnCpu + : public mlir::OpRewritePattern { + public: + using mlir::OpRewritePattern::OpRewritePattern; + + mlir::LogicalResult matchAndRewrite( + OpType op, mlir::PatternRewriter& rewriter) const override { + if (auto xla_must_compile = + op->template getAttrOfType("_XlaMustCompile"); + !xla_must_compile || !xla_must_compile.getValue()) { + return mlir::failure(); + } + + llvm::StringRef device = GetDeviceOrEmpty(op); + + if (device.empty() || !device.contains("CPU")) return mlir::failure(); + + llvm::SmallVector constants; + llvm::SmallVector resources; + + for (int i = 0; i < op.getNumOperands(); ++i) { + auto value = op.getOperand(i); + if (llvm::isa( + llvm::cast(value.getType()).getElementType())) { + resources.push_back(i); + } else if (auto* def = value.getDefiningOp(); + def && llvm::isa(def)) { + constants.push_back(i); + } + } + + rewriter.replaceOpWithNewOp( + op, op.getResultTypes(), op.getOperands(), + rewriter.getI64ArrayAttr(constants), + rewriter.getI64ArrayAttr(resources), op.getFAttr()); + + return mlir::success(); + } +}; + +struct TfrtXlaRewritePass + : public mlir::PassWrapper> { + llvm::StringRef getArgument() const override { return "tfrt-xla-rewrite"; } + + llvm::StringRef getDescription() const override { + return "rewrites for XLA host ops."; + } + + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TfrtXlaRewritePass) + + void runOnOperation() override { + mlir::RewritePatternSet patterns(&getContext()); + + patterns + .add>( + &getContext()); + patterns.add>(&getContext()); + + if (mlir::failed( + mlir::applyPatternsGreedily(getOperation(), std::move(patterns)))) { + signalPassFailure(); + return; + } + } +}; + +} // namespace + +std::unique_ptr> +CreateTfrtXlaRewritePass() { + return std::make_unique(); +} + +static mlir::PassRegistration register_pass( + CreateTfrtXlaRewritePass); + +} // namespace tfrt_compiler +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.cc new file mode 100644 index 0000000000000000000000000000000000000000..59fbfbeb6c2975e30cb5fd1dfe29201e0a7d4cbe --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.cc @@ -0,0 +1,407 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/translate/import_model.h" + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/functional/function_ref.h" +#include "absl/log/log.h" +#include "absl/log/vlog_is_on.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" // from @llvm-project +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/host_runtime/lower_cluster_to_runtime_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_asset_sinking_pass.h" +#include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/error_util.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/serialize_mlir_module_utils.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/cluster_tf.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_dialect_to_executor.h" +#include "tensorflow/compiler/mlir/tf2xla/api/v2/tf_executor_to_graph.h" +#include "tensorflow/compiler/mlir/tfrt/backend_compiler.h" +#include "tensorflow/compiler/mlir/tfrt/function/function.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tpu_passes.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "xla/tsl/framework/device_type.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/common_runtime/function_def_utils.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tsl/platform/env.h" +#include "tsl/platform/errors.h" +#include "tsl/platform/statusor.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime +#include "tfrt/bef_converter/mlir_to_bef.h" // from @tf_runtime +#include "tfrt/support/forward_decls.h" // from @tf_runtime + +namespace tensorflow { + +namespace { + +// Exports all XLA functions in the form of XlaLaunch, and their nested +// functions. +absl::StatusOr> ExportXlaFunctions( + mlir::ModuleOp module, std::vector* added_xla_function_names) { + // Find all XLA functions. + std::vector xla_functions; + module.walk([&](mlir::TF::XlaLaunchOp xla_launch_op) { + std::string func_name = + xla_launch_op.getFunctionAttr().getRootReference().str(); + xla_functions.push_back(func_name); + if (added_xla_function_names != nullptr) { + added_xla_function_names->push_back(func_name); + } + }); + + // Convert all XLA functions and their nested functions. + std::deque queue; + for (const std::string& func : xla_functions) { + queue.push_back(func); + } + + const mlir::SymbolTable symbol_table(module); + absl::flat_hash_set visited; + std::vector xla_func_defs; + while (!queue.empty()) { + const std::string func_name = queue.front(); + queue.pop_front(); + + if (visited.contains(func_name)) continue; + + const auto func_op = symbol_table.lookup(func_name); + if (!func_op) { + return absl::InternalError( + absl::StrCat("Function ", func_name, " is not found.")); + } + FunctionDef func_def; + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::ConvertMlirFunctionToFunctionLibraryDef( + func_op, GraphExportConfig(), &func_def)); + xla_func_defs.push_back(func_def); + + // Visit each op in the function and find out referenced functions from the + // attributes. + func_op->walk([&](mlir::Operation* op) { + for (const mlir::NamedAttribute& attr : op->getAttrs()) { + if (const auto sym = + mlir::dyn_cast(attr.getValue())) { + mlir::Operation* func = + mlir::SymbolTable::lookupNearestSymbolFrom(op, sym); + if (func) { + queue.push_back(sym.getValue().str()); + } + } + } + }); + + // Remove the function from the module, as it will be handled by XLA. + // It is safe to remove the function, i.e., the function won't be invoked on + // CPU. This is because bridge guarantees that each function has only one + // use. We don't replace the uses of the function, because we iterate from + // the root caller and hence its uses should have been removed. + func_op->erase(); + + visited.insert(func_name); + } + return xla_func_defs; +} + +} // namespace + +absl::Status ConvertTfMlirToRuntimeExecutable( + const TfrtCompileOptions& options, mlir::ModuleOp module, + absl::FunctionRef< + absl::Status(mlir::PassManager&, mlir::ModuleOp, + const tensorflow::TfrtPipelineOptions& options)> + emit_executable, + tfrt_stub::ModelRuntimeContext& model_context, + tfrt_stub::FallbackState* fallback_state, + std::vector* added_xla_function_names) { + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + { + mlir::PassManager pm(module.getContext()); + pm.addNestedPass( + mlir::tf_executor::CreateTFExecutorGraphPruningPass()); + pm.addNestedPass( + mlir::CreateExecutorDialectToFunctionalConversionPass()); + if (!options.saved_model_dir.empty()) { + pm.addPass(mlir::tf_saved_model::CreateAssetSinkingPass( + options.saved_model_dir)); + } + if (mlir::failed(pm.run(module))) { + return diag_handler.Combine(absl::InternalError( + "Failed to sinking assets into initialization graphs.")); + } + } + + if (options.backend_compiler != nullptr) { + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tf_dialect_before_backend_compile", module); + } + TF_RETURN_IF_ERROR( + options.backend_compiler->CompileTensorflow(model_context, module)); + } else if (options.device_target == TfrtDeviceInfraTarget::kTpurt) { + VLOG(1) << "Running MLIR TPU bridge for tpurt"; + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tpu_bct_conversion_before", module); + } + + TfrtTpuCompileOptions tpu_compile_options; + tpu_compile_options.move_resource_gather_to_host = + options.tpu_move_resource_gather_to_host; + tpu_compile_options.gather_table_width_threshold_bytes = + options.tpu_gather_table_width_threshold_bytes; + + auto backward_compat_result = + tensorflow::RunTPUBackwardCompatConversion(module, tpu_compile_options); + if (mlir::failed(backward_compat_result)) { + return diag_handler.Combine( + absl::InternalError("Failed to handle legacy TPU Ops")); + } + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tpu_bct_conversion_after", module); + } + + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::RunFunctionTf2xlaClusteringBridge( + module, /*is_supported_by_replicated_brige*/ true, + /*is_in_fallback_enabled_mode=*/VLOG_IS_ON(1))); + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("after_tf2xla_clustering_bridge", module); + } + + TF_RETURN_IF_ERROR( + tensorflow::tfrt_compiler::RunLowerClusterToRuntimeOpsPassPipeline( + module, tsl::DeviceType(DEVICE_TPU_XLA_JIT))); + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("after_lower_cluster_to_runtime_ops", + module); + } + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::ExportFromTensorflowDialectToExecutor(module)); + } else if (options.device_target == TfrtDeviceInfraTarget::kTfFallback) { + auto tpu_partitioned_call_fallback_compat_result = + tensorflow::RunTPUPartitionedCallFallbackCompatConversion(module); + if (mlir::failed(tpu_partitioned_call_fallback_compat_result)) { + return diag_handler.Combine(absl::InternalError( + "Failed to process TPUPartitionedCallOp for fallback execution")); + } + } else if (options.device_target == TfrtDeviceInfraTarget::kGpu) { + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::RunFunctionTf2xlaClusteringBridge( + module, /*is_supported_by_replicated_brige*/ false, + /*is_in_fallback_enabled_mode=*/false)); + + TF_RETURN_IF_ERROR( + tensorflow::tfrt_compiler::RunLowerClusterToRuntimeOpsPassPipeline( + module, tsl::DeviceType(DEVICE_GPU_XLA_JIT))); + + TF_RETURN_IF_ERROR( + tensorflow::tf2xla::v2::ExportFromTensorflowDialectToExecutor(module)); + + if (options.serialize_mlir_module_to_aot_packages) { + const std::string mlir_string = SerializeMlirModule(module); + TF_RETURN_IF_ERROR(WriteStringToFile( + tsl::Env::Default(), options.aot_mlir_module_file, mlir_string)); + } + + // GPU XLA clusters are wrapped in functions, which could be transformed by + // bridge. Hence, the MLIR functions for XLA clusters are exported and added + // to the function library. + TF_RETURN_IF_ERROR( + AddXlaFunctions(fallback_state, module, added_xla_function_names)); + } + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tf_dialect", module); + } + + mlir::DialectRegistry registry; + mlir::func::registerAllExtensions(registry); + module.getContext()->appendDialectRegistry(registry); + + // Lower MLIR TF Dialect to MLIR TFRT CoreRT dialect. + mlir::PassManager pm(module.getContext()); + + auto pipeline_options = GetTfrtPipelineOptions(options); + + TF_RETURN_IF_ERROR( + tensorflow::CreateTFExecutorToTFPreInvariantOptimizationPipeline( + pm, *pipeline_options)); + + auto status = emit_executable(pm, module, *pipeline_options); + + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tfrt_dialect", module); + } + + return status; +} + +absl::Status ConvertTfMlirToBef( + const TfrtCompileOptions& options, mlir::ModuleOp module, + tfrt::BefBuffer* bef_buffer, tfrt_stub::ModelRuntimeContext& model_context, + tfrt_stub::FallbackState* fallback_state, + std::vector* added_xla_function_names) { + return ConvertTfMlirToRuntimeExecutable( + options, module, + [bef_buffer](mlir::PassManager& pm, mlir::ModuleOp module, + const tensorflow::TfrtPipelineOptions& options) { + mlir::StatusScopedDiagnosticHandler diag_handler(module.getContext()); + tensorflow::CreateTFInvariantOptimizationPipelineHelper(pm, options); + tensorflow::CreateTfToTfrtPipeline(pm, options); + + if (mlir::failed(pm.run(module))) { + if (VLOG_IS_ON(1)) { + tensorflow::DumpMlirOpToFile("tf_to_corert_failure", module); + } + return diag_handler.Combine(absl::InternalError( + "failed to lower TF Dialect to CoreRT dialect.")); + } + + *bef_buffer = + tfrt::ConvertMLIRToBEF(module, /*disable_optional_sections=*/true); + if (bef_buffer->empty()) + return diag_handler.Combine( + absl::InternalError("failed to convert MLIR to BEF.")); + + bef_buffer->shrink_to_fit(); + return absl::OkStatus(); + }, + model_context, fallback_state, added_xla_function_names); +} + +std::unique_ptr GetTfrtPipelineOptions( + const TfrtCompileOptions& options) { + auto pipeline_options = std::make_unique(); + + pipeline_options->saved_model_dir = options.saved_model_dir; + + if (!options.default_device.empty()) { + pipeline_options->default_device = options.default_device; + } + if (!options.force_data_format.empty()) { + pipeline_options->force_data_format = options.force_data_format; + } + + // TODO(b/187991150): Consider only decomposing read-only resource variable + // ops. + pipeline_options->decompose_resource_ops = options.decompose_resource_ops; + pipeline_options->enable_optimizer = options.enable_optimizer; + pipeline_options->target_tpurt = + (options.device_target == TfrtDeviceInfraTarget::kTpurt); + pipeline_options->target_gpu = + (options.device_target == TfrtDeviceInfraTarget::kGpu); + pipeline_options->use_gpu_compile_and_execute_op = + options.use_gpu_compile_and_execute_op; + pipeline_options->tpu_fuse_ops = options.tpu_fuse_ops; + pipeline_options->use_tpu_host_allocator_for_inputs = + options.use_tpu_host_allocator_for_inputs; + pipeline_options->tpu_allow_unpadded_batch = options.tpu_allow_unpadded_batch; + pipeline_options->sink_in_invariant_ops = options.sink_in_invariant_ops; + pipeline_options->hoist_invariant_ops = options.hoist_invariant_ops; + pipeline_options->fuse_get_resource_ops_in_hoisting = + options.fuse_get_resource_ops_in_hoisting; + pipeline_options->enable_while_parallel_iterations = + options.enable_while_parallel_iterations; + pipeline_options->cost_threshold = options.cost_threshold; + pipeline_options->min_num_batch_threads = options.min_num_batch_threads; + pipeline_options->min_max_enqueued_batches = options.min_max_enqueued_batches; + pipeline_options->batch_queue_global_prioritization_num_threads = + options.batch_queue_global_prioritization_num_threads; + pipeline_options->enable_priority_aware_batch_scheduler = + options.enable_priority_aware_batch_scheduler; + pipeline_options->enable_priority_aware_batch_scheduler_resplit = + options.enable_priority_aware_batch_scheduler_resplit; + pipeline_options->enable_batching_task_lazy_cancellation = + options.enable_batching_task_lazy_cancellation; + pipeline_options->batch_padding_policy = options.batch_padding_policy; + pipeline_options->num_batch_threads = + options.batch_options.num_batch_threads(); + pipeline_options->max_batch_size = options.batch_options.max_batch_size(); + pipeline_options->batch_timeout_micros = + options.batch_options.batch_timeout_micros(); + pipeline_options->allowed_batch_sizes = llvm::ArrayRef( + std::vector(options.batch_options.allowed_batch_sizes().begin(), + options.batch_options.allowed_batch_sizes().end())); + pipeline_options->max_enqueued_batches = + options.batch_options.max_enqueued_batches(); + pipeline_options->low_priority_max_batch_size = + options.batch_options.low_priority_max_batch_size(); + pipeline_options->low_priority_batch_timeout_micros = + options.batch_options.low_priority_batch_timeout_micros(); + pipeline_options->low_priority_allowed_batch_sizes = + llvm::ArrayRef(std::vector( + options.batch_options.low_priority_allowed_batch_sizes().begin(), + options.batch_options.low_priority_allowed_batch_sizes().end())); + pipeline_options->low_priority_max_enqueued_batches = + options.batch_options.low_priority_max_enqueued_batches(); + pipeline_options->num_warmup_batch_threads = + options.batch_options.num_warmup_batch_threads(); + pipeline_options->enable_large_batch_splitting = + options.batch_options.enable_large_batch_splitting(); + pipeline_options->mixed_priority_batching_policy = + options.batch_options.mixed_priority_batching_policy(); + pipeline_options->merge_inter_dependent_streams = + options.merge_inter_dependent_streams; + pipeline_options->allow_xla_cpu = options.allow_xla_cpu; + + return pipeline_options; +} + +absl::Status AddXlaFunctions( + tfrt_stub::FallbackState* fallback_state, mlir::ModuleOp mlir_module, + std::vector* added_xla_function_names) { + if (fallback_state != nullptr) { + TF_ASSIGN_OR_RETURN( + const std::vector xla_func_defs, + ExportXlaFunctions(mlir_module, added_xla_function_names)); + for (const auto& func_def : xla_func_defs) { + TF_RETURN_IF_ERROR(fallback_state->AddFunctionDef(func_def)); + } + } + + return absl::OkStatus(); +} + +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.h new file mode 100644 index 0000000000000000000000000000000000000000..7fe581271aff4d3589be4093344a2de1a8229c38 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model.h @@ -0,0 +1,74 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_IMPORT_MODEL_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_IMPORT_MODEL_H_ + +#include +#include +#include + +#include "absl/functional/function_ref.h" +#include "absl/status/status.h" +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/function/function.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/passes.h" +#include "tensorflow/compiler/mlir/tfrt/transforms/tfrt_pipeline_options.h" +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tfrt/fallback/fallback_state.h" +#include "tensorflow/core/tfrt/runtime/runtime.h" +#include "tfrt/bef/bef_buffer.h" // from @tf_runtime + +namespace tensorflow { + +struct FunctionBody; + +// Converts an MLIR `module` in TF dialect to TFRT's Binary Executable Format. +// If `fallback_state` is not null, the MLIR functions for XLA clusters in +// the form of XlaLaunch will be exported and added to the function library when +// needed. The nested functions will also be exported. If +// `added_xla_function_names` is not null, it will be populated with the names +// of the added XLA functions. +absl::Status ConvertTfMlirToBef( + const TfrtCompileOptions& options, mlir::ModuleOp module, + tfrt::BefBuffer* bef_buffer, tfrt_stub::ModelRuntimeContext& model_context, + tfrt_stub::FallbackState* fallback_state = nullptr, + std::vector* added_xla_function_names = nullptr); + +absl::Status ConvertTfMlirToRuntimeExecutable( + const TfrtCompileOptions& options, mlir::ModuleOp module, + absl::FunctionRef< + absl::Status(mlir::PassManager&, mlir::ModuleOp, + const tensorflow::TfrtPipelineOptions& options)> + emit_executable, + tfrt_stub::ModelRuntimeContext& model_context, + tfrt_stub::FallbackState* fallback_state = nullptr, + std::vector* added_xla_function_names = nullptr); + +std::unique_ptr GetTfrtPipelineOptions( + const TfrtCompileOptions& options); + +// Adds MLIR functions for XLA clusters to the function library. +absl::Status AddXlaFunctions( + tfrt_stub::FallbackState* fallback_state, mlir::ModuleOp mlir_module, + std::vector* added_xla_function_names = nullptr); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_IMPORT_MODEL_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a901e76afc1e970a70cf167e368cfd32e18f0f81 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/import_model_test.cc @@ -0,0 +1,123 @@ +/* Copyright 2024 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tfrt/translate/import_model.h" + +#include +#include +#include "tensorflow/compiler/mlir/tfrt/translate/tfrt_compile_options.h" + +namespace tensorflow { +namespace { + +using ::testing::SizeIs; + +TEST(GetTfrtPipelineOptions, BatchPaddingPolicy) { + tensorflow::TfrtCompileOptions options; + options.batch_padding_policy = "PAD_TEST_OPTION"; + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->batch_padding_policy, "PAD_TEST_OPTION"); +} + +TEST(GetTfrtPipelineOptions, NumBatchThreads) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_num_batch_threads(2); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->num_batch_threads, 2); +} + +TEST(GetTfrtPipelineOptions, MaxBatchSize) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_max_batch_size(8); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->max_batch_size, 8); +} + +TEST(GetTfrtPipelineOptions, BatchTimeoutMicros) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_batch_timeout_micros(5000); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->batch_timeout_micros, 5000); +} + +TEST(GetTfrtPipelineOptions, AllowedBatchSizes) { + tensorflow::TfrtCompileOptions options; + options.batch_options.add_allowed_batch_sizes(2); + options.batch_options.add_allowed_batch_sizes(4); + options.batch_options.add_allowed_batch_sizes(8); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_THAT(pipeline_options->allowed_batch_sizes, SizeIs(3)); +} + +TEST(GetTfrtPipelineOptions, MaxEnqueuedBatches) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_max_enqueued_batches(250); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->max_enqueued_batches, 250); +} + +TEST(GetTfrtPipelineOptions, LowPriorityMaxBatchSize) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_low_priority_max_batch_size(16); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->low_priority_max_batch_size, 16); +} + +TEST(GetTfrtPipelineOptions, LowPriorityBatchTimeoutMicros) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_low_priority_batch_timeout_micros(10000); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->low_priority_batch_timeout_micros, 10000); +} + +TEST(GetTfrtPipelineOptions, LowPriorityAllowedBatchSizes) { + tensorflow::TfrtCompileOptions options; + options.batch_options.add_low_priority_allowed_batch_sizes(4); + options.batch_options.add_low_priority_allowed_batch_sizes(8); + options.batch_options.add_low_priority_allowed_batch_sizes(16); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_THAT(pipeline_options->low_priority_allowed_batch_sizes, SizeIs(3)); +} + +TEST(GetTfrtPipelineOptions, LowPriorityMaxEnqueuedBatches) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_low_priority_max_enqueued_batches(100); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->low_priority_max_enqueued_batches, 100); +} + +TEST(GetTfrtPipelineOptions, NumWarmupBatchThreads) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_num_warmup_batch_threads(4); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->num_warmup_batch_threads, 4); +} + +TEST(GetTfrtPipelineOptions, EnableLargeBatchSplitting) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_enable_large_batch_splitting(true); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_TRUE(pipeline_options->enable_large_batch_splitting); +} + +TEST(GetTfrtPipelineOptions, MixedPriorityBatchingPolicy) { + tensorflow::TfrtCompileOptions options; + options.batch_options.set_mixed_priority_batching_policy("priority_merge"); + auto pipeline_options = GetTfrtPipelineOptions(options); + EXPECT_EQ(pipeline_options->mixed_priority_batching_policy, "priority_merge"); +} + +} // namespace +} // namespace tensorflow diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/BUILD b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..40e75a0bf93c5418c3258947615613d389f3e1b0 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/BUILD @@ -0,0 +1,94 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load( + "//tensorflow:tensorflow.bzl", + "tf_cc_test", +) + +package( + # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"], + default_visibility = [ + # copybara:uncomment "//learning/brain/experimental/tfrt:__subpackages__", + # copybara:uncomment "//smartass/brain/ops/tfrt_kernels:__subpackages__", + "//tensorflow/compiler/mlir/tfrt/transforms/mlrt:__subpackages__", + "//tensorflow/core/tfrt:__subpackages__", + ], +) + +cc_library( + name = "mlir_to_bytecode", + srcs = ["mlir_to_bytecode.cc"], + hdrs = ["mlir_to_bytecode.h"], + deps = [ + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/core/tfrt/mlrt/bytecode", + "//tensorflow/core/tfrt/mlrt/bytecode:executable", + "//tensorflow/core/tfrt/mlrt/bytecode:function", + "//tensorflow/core/tfrt/mlrt/bytecode:kernel", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + ], +) + +tf_cc_test( + name = "mlir_to_bytecode_test", + srcs = ["mlir_to_bytecode_test.cc"], + data = glob(["testdata/**"]), + deps = [ + ":mlir_to_bytecode", + "//tensorflow/compiler/mlir/tfrt/ir/mlrt:mlrt_ops", + "//tensorflow/core/tfrt/mlrt/bytecode", + "//tensorflow/core/tfrt/mlrt/bytecode:executable", + "//tensorflow/core/tfrt/mlrt/interpreter:attribute_span", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:status_matchers", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:string_view", + "@com_google_googletest//:gtest_main", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Support", + "@xla//xla/tsl/platform:resource_loader", + ], +) + +cc_library( + name = "test_utils", + testonly = 1, + srcs = ["test_utils.cc"], + hdrs = ["test_utils.h"], + deps = [ + # copybara:uncomment "//learning/brain/experimental/tfrt/native_lowering/stubs:tfrt_native_lowering_impl", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/types:span", + "@xla//xla/tsl/platform:errors", + "@xla//xla/tsl/platform:statusor", + "//tensorflow/core:core_cpu_base", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/tfrt/graph_executor:sync_resource_state", + "//tensorflow/core/tfrt/mlrt/attribute", + "//tensorflow/core/tfrt/mlrt/bytecode", + "//tensorflow/core/tfrt/mlrt/bytecode:kernel", + "//tensorflow/core/tfrt/mlrt/interpreter:context", + "//tensorflow/core/tfrt/mlrt/interpreter:interpreter_testutil", + "//tensorflow/core/tfrt/mlrt/interpreter:value", + "//tensorflow/core/tfrt/stubs:tfrt_native_lowering_stub", + "//tensorflow/core/tfrt/utils:tensor_util", + "@tf_runtime//:hostcontext", + "@tf_runtime//:support", + "@tf_runtime//:tensor", + ], +) diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.cc new file mode 100644 index 0000000000000000000000000000000000000000..e2de06516650449301ee78a6e4ede8a2ac8f4c02 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.cc @@ -0,0 +1,526 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/log/check.h" +#include "absl/log/log.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/Types.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/bytecode.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/executable.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/function.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/kernel.h" + +namespace mlrt { +namespace { + +// LINT.IfChange(mlrt_attributes) +bool CanBeInlined(mlir::Attribute attr, absl::string_view data) { + // FlatSymbolRefAttr is a special case as we are emitting it as integer. + return mlir::isa( + attr) && + data.size() <= sizeof(uint32_t); +} +// LINT.ThenChange(../../../../../core/tfrt/mlrt/interpreter/attribute_span.h:mlrt_attributes) + +// Encode integer or float-point numbers as bytes. +template +std::string EncodeIntegerOrFloat(T attr) { + std::string data(sizeof(attr), '\0'); + std::memcpy(data.data(), &attr, sizeof(attr)); + return data; +} + +// Encode a list of I64 integers as bytes using bc::Vector. The bytes +// can be decoded directly using bc::Vector. If `array` is not a list +// I64 integers, a nullopt will be returned. + +template +std::optional EncodeListOfInteger(mlir::ArrayAttr array) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto ctor = bc::New>(&allocator, array.size()); + + mlir::Type type; + + for (int i = 0; i < array.size(); ++i) { + if (auto integer_attr = mlir::dyn_cast(array[i])) { + if (type && integer_attr.getType() != type) return std::nullopt; + type = integer_attr.getType(); + llvm::APInt value = integer_attr.getValue(); + if (value.getBitWidth() != sizeof(T) * 8) return std::nullopt; + ctor.ConstructAt(i, value.getZExtValue()); + } else { + return std::nullopt; + } + } + + return std::string(buffer.data(), buffer.size()); +} + +std::optional EncodeListOfSymbolRef( + const ModuleEmitterContext& module_context, mlir::ArrayAttr array) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto ctor = bc::New>(&allocator, array.size()); + + for (int i = 0; i < array.size(); ++i) { + if (auto symbol_ref = mlir::dyn_cast(array[i])) { + ctor.ConstructAt(i, module_context.GetFunctionId(symbol_ref.getValue())); + } else { + return std::nullopt; + } + } + return std::string(buffer.data(), buffer.size()); +} + +template +std::optional EncodeDenseArray(llvm::ArrayRef array) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto ctor = bc::New>(&allocator, array.size()); + + if (!array.empty()) { + ctor.Place(reinterpret_cast(array.data()), + array.size() * sizeof(T)); + } + + return std::string(buffer.data(), buffer.size()); +} + +// bool values has special encoding in MLIR. It occupies one bit in MLIR +// but in bytecode it is one byte. +std::optional EncodeDenseBoolArray(llvm::ArrayRef array) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto ctor = bc::New>(&allocator, array.size()); + + if (!array.empty()) { + std::vector data(array.size()); + int i = 0; + for (auto v : array) { + data[i++] = static_cast(v); + } + ctor.Place(reinterpret_cast(data.data()), data.size()); + } + return std::string(buffer.data(), buffer.size()); +} + +// Encode a list of strings as bytes using bc::Vector. The bytes +// can be decoded directly using bc::Vector. If `array` is not a +// list of strings, a nullopt will be returned. +std::optional EncodeListOfString(mlir::ArrayAttr array) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto ctor = bc::New>(&allocator, array.size()); + + for (int i = 0; i < array.size(); ++i) { + if (auto string_attr = mlir::dyn_cast(array[i])) { + ctor.ConstructAt(i, string_attr.getValue().str()); + } else { + return std::nullopt; + } + } + + return std::string(buffer.data(), buffer.size()); +} + +struct FunctionEmitterContext { + explicit FunctionEmitterContext(const ModuleEmitterContext* module_context) + : module_context(*module_context) {} + + const ModuleEmitterContext& module_context; + + struct RegInfo { + int num_uses = 0; + int id = -1; + bool persistent = false; // True if the register should not be freed + }; + + int next_reg_id = 0; + llvm::DenseMap register_table; + std::vector free_regs; + + int AssignRegId(bool is_persistent) { + if (is_persistent) { + // Persistent types ALWAYS get a brand new ID. + return next_reg_id++; + } + + // Non-persistent types can reuse from free_regs. + if (!free_regs.empty()) { + int id = free_regs.back(); + free_regs.pop_back(); + return id; + } + return next_reg_id++; + } + + void FreeRegId(int id) { free_regs.push_back(id); } +}; + +// Emit the bytecode for a kernel. It uses the information in an MLIR operation +// and populates the bytecode using bc::Kernel::Constructor. For a kernel's +// bytecode format, please refer to kernel.h. +void EmitKernel(FunctionEmitterContext& function_context, + bc::Kernel::Constructor& constructor, mlir::Operation& op, + std::vector& function_output_regs, + std::vector& function_output_last_uses) { + // Assign reg ids for results first to make sure results does not reuse reg + // ids freed from args in the same operation. + std::vector results; + results.reserve(op.getNumResults()); + for (auto result : op.getResults()) { + auto iter = function_context.register_table.find(result); + CHECK(iter != function_context.register_table.end()); // Crash Ok + CHECK_EQ(iter->second.id, -1); // Crash Ok + iter->second.id = function_context.AssignRegId(iter->second.persistent); + results.push_back(iter->second.id); + } + constructor.construct_results(results.size()) + .Assign(results.begin(), results.end()); + + std::vector arguments; + std::vector last_uses; + arguments.reserve(op.getNumOperands()); + last_uses.reserve(op.getNumOperands()); + for (auto operand : op.getOperands()) { + auto iter = function_context.register_table.find(operand); + CHECK(iter != function_context.register_table.end()); // Crash Ok + int id = iter->second.id; + CHECK_NE(id, -1); // Crash Ok + last_uses.push_back(0); + auto& reg_info = iter->second; + if (!reg_info.persistent) { + if (--reg_info.num_uses == 0) { + function_context.FreeRegId(id); + last_uses.back() = 1; + } + } + arguments.push_back(id); + } + + constructor.construct_arguments(arguments.size()) + .Assign(arguments.begin(), arguments.end()); + constructor.construct_last_uses(last_uses.size()) + .Assign(last_uses.begin(), last_uses.end()); + + std::vector attributes; + attributes.reserve(op.getAttrs().size()); + for (auto attr : op.getAttrs()) { + int attr_id = + function_context.module_context.GetAttributeId(attr.getValue()); + absl::string_view attr_data = + function_context.module_context.attributes().at(attr_id); + + if (CanBeInlined(attr.getValue(), attr_data)) { + uint32_t data = 0; + std::memcpy(&data, attr_data.data(), attr_data.size()); + attributes.push_back(data); + } else { + attributes.push_back(attr_id); + } + } + constructor.construct_attributes(attributes.size()) + .Assign(attributes.begin(), attributes.end()); + + if (llvm::isa(&op)) { + constructor.set_code(function_context.module_context.GetKernelId("return")); + + function_output_regs = std::move(arguments); + function_output_last_uses = std::move(last_uses); + + } else if (llvm::isa(&op)) { + constructor.set_code(function_context.module_context.GetKernelId("call")); + } else { + llvm::StringRef op_name = op.getName().getStringRef(); + constructor.set_code(function_context.module_context.GetKernelId(op_name)); + } +} + +// Emit the bytecode for a function. It uses information in an MLIR function or +// an MLIR region, and populates the bytecode using bc::Function::Constructor. +// For a function's bytecode format, please refer to function.h. +void EmitFunction(const ModuleEmitterContext& module_context, + bc::Function::Constructor& constructor, llvm::StringRef name, + mlir::Region& region) { + FunctionEmitterContext function_context(&module_context); + + constructor.construct_name(name.str()); + + DCHECK(llvm::hasSingleElement(region)) << "should have a single block"; + + auto& block = region.front(); + + auto& register_table = function_context.register_table; + + std::vector input_regs; + input_regs.reserve(block.getNumArguments()); + for (auto arg : block.getArguments()) { + bool persistent = mlir::isa(arg.getType()); + int id = function_context.AssignRegId(persistent); + input_regs.push_back(id); + register_table[arg] = {static_cast(std::distance(arg.getUses().begin(), + arg.getUses().end())), + id, persistent}; + } + constructor.construct_input_regs(input_regs); + + for (auto& op : block) { + for (auto result : op.getResults()) { + bool persistent = + mlir::isa(result.getType()); + register_table[result] = { + static_cast( + std::distance(result.getUses().begin(), result.getUses().end())), + -1, persistent}; + } + } + + auto kernels_constructor = + constructor.construct_kernels(block.getOperations().size()); + + std::vector output_regs; + std::vector output_last_uses; + for (const auto& iter : llvm::enumerate(block.getOperations())) { + int i = iter.index(); + mlir::Operation& op = iter.value(); + auto kernel_ctor = kernels_constructor.ConstructAt(i); + EmitKernel(function_context, kernel_ctor, op, output_regs, + output_last_uses); + } + + constructor.set_num_regs(function_context.next_reg_id); + constructor.construct_output_regs(output_regs); + constructor.construct_output_last_uses(output_last_uses); +} + +// Emit the bytecode for an executable. It converts attributes, kernels, and +// functions in an MLIR module to bytecode using bc::Executable::Constructor. +// For an executable's bytecode format, please refer to executable.h. +absl::Status EmitExecutable(ModuleEmitterContext& module_context, + bc::Executable::Constructor& constructor, + mlir::ModuleOp module) { + module.walk( + [&](mlir::func::FuncOp func) { module_context.AddFunction(func); }); + + auto functions = module_context.functions(); + for (auto func : functions) { + if (!llvm::hasSingleElement(func.getRegion())) { + return absl::InvalidArgumentError("function should have a single block."); + } + auto& block = func.getRegion().front(); + + for (auto& op : block) { + if (llvm::isa(&op)) { + // Canonicalize the MLIR builtin call op's name to "call". + module_context.AddKernelName("call"); + } else if (llvm::isa(&op)) { + // Canonicalize the return op's name to "return". + if (op.getNumResults() != 0) { + return absl::InvalidArgumentError( + "Block terminator must be a return op."); + } + module_context.AddKernelName("return"); + } else { + module_context.AddKernelName(op.getName().getStringRef().str()); + } + + for (auto attr : op.getAttrs()) { + if (auto status = module_context.AddAttribute(&op, attr.getValue()); + !status.ok()) { + return status; + } + } + + // TODO(chky): Support inline regions. + } + } + + constructor.construct_kernel_names(module_context.kernels().size()) + .Assign(module_context.kernels().begin(), module_context.kernels().end()); + + auto functions_constructor = + constructor.construct_functions(functions.size()); + for (int i = 0; i < functions.size(); ++i) { + auto func = functions[i]; + auto function_ctor = functions_constructor.ConstructAt(i); + EmitFunction(module_context, function_ctor, func.getSymName(), + func.getRegion()); + } + + // Emit attributes after emitting functions as attributes might be large. + // Large attributes may result in large offsets that do not fit into a + // unit32_t integer. Since functions section should fit into 2GB size limit, + // so we emit functions first. + constructor.construct_attributes(module_context.attributes().size()) + .Assign(module_context.attributes().begin(), + module_context.attributes().end()); + + return absl::OkStatus(); +} + +} // namespace + +absl::Status ModuleEmitterContext::AddAttribute(mlir::Operation* op, + mlir::Attribute attr) { + absl::StatusOr attr_data; + if (auto* encoder = attribute_encoder_registry_.Get( + op->getName().getDialectNamespace())) { + attr_data = (*encoder)(*this, attr); + } else { + attr_data = DefaultEncodeAttribute(attr); + } + if (!attr_data.ok()) return std::move(attr_data).status(); + + int id = AddData(std::move(*attr_data), attributes_, attribute_data_id_map_); + attribute_id_map_[attr] = id; + + return absl::OkStatus(); +} + +int ModuleEmitterContext::AddFunction(mlir::func::FuncOp func) { + int id = functions_.size(); + functions_.push_back(func); + DCHECK(!function_name_id_map_.contains(func.getSymName())); + function_name_id_map_[func.getSymName()] = id; + return id; +} + +std::optional EncodeSimpleAttribute( + const ModuleEmitterContext& module_context, mlir::Attribute attr) { + return llvm::TypeSwitch>(attr) + .Case( + [](const auto& str_attr) { return str_attr.str(); }) + .Case( + [](const auto& integer_attr) -> std::optional { + switch (llvm::APInt value = integer_attr.getValue(); + value.getBitWidth()) { + case 1: + return EncodeIntegerOrFloat(value.getZExtValue()); + case 32: + return EncodeIntegerOrFloat(value.getZExtValue()); + case 64: + return EncodeIntegerOrFloat(value.getZExtValue()); + default: + return std::nullopt; + } + }) + .Case( + [](const auto& float_attr) -> std::optional { + llvm::APFloat value = float_attr.getValue(); + if (float_attr.getType().isF32()) { + return EncodeIntegerOrFloat(value.convertToFloat()); + } + return std::nullopt; + }) + .Case([&](const auto& array_attr) + -> std::optional { + if (auto encoded_list_i32 = EncodeListOfInteger(array_attr)) { + return std::move(*encoded_list_i32); + } else if (auto encoded_list_i64 = + EncodeListOfInteger(array_attr)) { + return std::move(*encoded_list_i64); + } else if (auto encoded_list_string = EncodeListOfString(array_attr)) { + return std::move(*encoded_list_string); + } else if (auto encoded_list_symbol_ref = + EncodeListOfSymbolRef(module_context, array_attr)) { + return std::move(*encoded_list_symbol_ref); + } else { + return std::nullopt; + } + }) + .Case( + [](const auto& dense_array_i32) -> std::optional { + return EncodeDenseArray(dense_array_i32); + }) + .Case( + [](const auto& dense_array_i64) -> std::optional { + return EncodeDenseArray(dense_array_i64); + }) + .Case( + [](const auto& dense_array_bool) -> std::optional { + return EncodeDenseBoolArray(dense_array_bool.asArrayRef()); + }) + .Case([&](const auto& symbol_ref) { + return EncodeIntegerOrFloat( + module_context.GetFunctionId(symbol_ref.getValue())); + }) + .Default([](const auto& attr) { return std::nullopt; }); +} + +// Encode mlir attributes with a limited support such as I64, string and array +// of I64. Returns an error if the attribute is not supported. +absl::StatusOr ModuleEmitterContext::DefaultEncodeAttribute( + mlir::Attribute attr) { + if (auto result = EncodeSimpleAttribute(*this, attr)) { + return std::move(*result); + } + + // TODO(chky): Add a unit test for the error below. This requires we + // propagate the error all the way back to the entry point. + std ::string attr_str; + llvm::raw_string_ostream os(attr_str); + attr.print(os); + + return absl::InvalidArgumentError( + absl::StrCat("Try to encode unsupported attribute: ", attr_str)); +} + +absl::StatusOr EmitExecutable( + const AttributeEncoderRegistry& attribute_encoder_registry, + mlir::ModuleOp module) { + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + + ModuleEmitterContext module_context(&attribute_encoder_registry); + + auto executable_ctor = bc::New(&allocator); + + if (auto status = EmitExecutable(module_context, executable_ctor, module); + !status.ok()) { + return status; + } + + buffer.shrink_to_fit(); + + return buffer; +} + +} // namespace mlrt diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h new file mode 100644 index 0000000000000000000000000000000000000000..e20300993ad288c45023c10a88fdf4b3cc35bbcb --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h @@ -0,0 +1,135 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_MLRT_MLIR_TO_BYTECODE_H_ +#define TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_MLRT_MLIR_TO_BYTECODE_H_ + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringRef.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "tensorflow/core/tfrt/mlrt/bytecode/bytecode.h" + +namespace mlrt { + +class ModuleEmitterContext; + +// Defines a custom attribute encoding registry. Users can register custom +// attribute encoding for their dialects in this registry. If no custom encoder +// is registered for a dialect, the default encoding with a limited support, the +// EncodeSimpleAttribute() below, will be used. +class AttributeEncoderRegistry { + public: + using EncoderFn = std::function( + const ModuleEmitterContext&, mlir::Attribute)>; + + void Register(absl::string_view dialect, EncoderFn encoder) { + encoders_[dialect] = std::move(encoder); + } + + // Returns the encoder for the specified dialect. It can be nullptr if it is + // not registered for this dialect. The returned reference will be invalidated + // if Register() is called. + const EncoderFn* Get(absl::string_view dialect) const { + auto iter = encoders_.find(dialect); + if (iter != encoders_.end()) return &iter->second; + return nullptr; + } + + private: + absl::flat_hash_map encoders_; +}; + +class ModuleEmitterContext { + public: + explicit ModuleEmitterContext( + const AttributeEncoderRegistry* attribute_encoder_registry) + : attribute_encoder_registry_(*attribute_encoder_registry) {} + + void AddKernelName(std::string name) { + AddData(std::move(name), kernels_, kernel_id_map_); + } + + int GetKernelId(llvm::StringRef name) const { + return kernel_id_map_.at(name); + } + + absl::Status AddAttribute(mlir::Operation* op, mlir::Attribute attr); + + int GetAttributeId(mlir::Attribute attr) const { + return attribute_id_map_.lookup(attr); + } + + int AddFunction(mlir::func::FuncOp func); + + int GetFunctionId(absl::string_view name) const { + return function_name_id_map_.at(name); + } + + absl::Span kernels() const { return kernels_; } + absl::Span attributes() const { return attributes_; } + absl::Span functions() const { return functions_; } + + private: + int AddData(std::string data, std::vector& data_vector, + absl::flat_hash_map& data_map) { + auto iter = data_map.find(data); + if (iter != data_map.end()) return iter->second; + + int id = data_vector.size(); + data_map[data] = id; + data_vector.push_back(std::move(data)); + return id; + } + + absl::StatusOr DefaultEncodeAttribute(mlir::Attribute attr); + + const AttributeEncoderRegistry& attribute_encoder_registry_; + + std::vector kernels_; + absl::flat_hash_map kernel_id_map_; + + std::vector attributes_; + llvm::DenseMap attribute_id_map_; + absl::flat_hash_map attribute_data_id_map_; + + std::vector functions_; + absl::flat_hash_map function_name_id_map_; +}; + +// Encodes a few simple attributes. Users can use this function in their custom +// attribute encoder. +std::optional EncodeSimpleAttribute( + const ModuleEmitterContext& module_context, mlir::Attribute attr); + +absl::StatusOr EmitExecutable( + const AttributeEncoderRegistry& attribute_encoder_registry, + mlir::ModuleOp module); + +} // namespace mlrt + +#endif // TENSORFLOW_COMPILER_MLIR_TFRT_TRANSLATE_MLRT_MLIR_TO_BYTECODE_H_ diff --git a/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode_test.cc b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..77e5881e818931c14630074c50fbe2fc2efa3be3 --- /dev/null +++ b/western_ai/usa/tensorflow/tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode_test.cc @@ -0,0 +1,511 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/compiler/mlir/tfrt/translate/mlrt/mlir_to_bytecode.h" + +#include +#include +#include +#include + +#include +#include +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" // from @llvm-project +#include "mlir/IR/Attributes.h" // from @llvm-project +#include "mlir/IR/BuiltinAttributes.h" // from @llvm-project +#include "mlir/IR/BuiltinOps.h" // from @llvm-project +#include "mlir/IR/BuiltinTypes.h" // from @llvm-project +#include "mlir/IR/DialectRegistry.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/Parser/Parser.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tfrt/ir/mlrt/mlrt_dialect.h" +#include "xla/tsl/platform/resource_loader.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/bytecode.h" +#include "tensorflow/core/tfrt/mlrt/bytecode/executable.h" +#include "tensorflow/core/tfrt/mlrt/interpreter/attribute_span.h" + +namespace mlrt { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::FloatEq; +using ::testing::IsEmpty; + +TEST(MlirToByteCodeTest, Basic) { + constexpr char kBasicMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/basic.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kBasicMlir), &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + bc::Buffer buffer = + EmitExecutable(attribute_encoder_registry, mlir_module.get()).value(); + + bc::Executable executable(buffer.data()); + + auto kernel_names = executable.kernel_names(); + EXPECT_THAT(kernel_names, + ElementsAreArray({"test_mlbc.add.i32", "test_mlbc.sub.i32", + "call", "return"})); + + auto functions = executable.functions(); + ASSERT_GE(functions.size(), 1); + + auto function = functions[0]; + EXPECT_EQ(function.name().str(), "add_i32_10"); + EXPECT_EQ(function.num_regs(), 5); + EXPECT_THAT(function.input_regs(), ElementsAreArray({0})); + EXPECT_THAT(function.output_regs(), ElementsAreArray({0, 2, 2})); + EXPECT_THAT(function.output_last_uses(), + ElementsAreArray({true, false, true})); + + auto kernels = function.kernels(); + ASSERT_EQ(kernels.size(), 11); + + EXPECT_EQ(kernels[0].code(), 0); + EXPECT_THAT(kernels[0].arguments(), ElementsAreArray({0, 0})); + EXPECT_THAT(kernels[0].results(), ElementsAreArray({1})); + EXPECT_THAT(kernels[0].last_uses(), ElementsAreArray({0, 0})); + + for (int i = 1; i < 9; i++) { + EXPECT_EQ(kernels[i].code(), i % 2); + EXPECT_THAT(kernels[i].arguments(), ElementsAreArray({(i - 1) % 2 + 1, 0})); + EXPECT_THAT(kernels[i].results(), ElementsAreArray({i % 2 + 1})); + EXPECT_THAT(kernels[i].last_uses(), ElementsAreArray({1, 0})); + } + + EXPECT_EQ(kernels[9].code(), 2); + EXPECT_THAT(kernels[9].arguments(), ElementsAreArray({1})); + EXPECT_THAT(kernels[9].last_uses(), ElementsAreArray({true})); + EXPECT_THAT(kernels[9].results(), ElementsAreArray({2, 3, 4})); + + EXPECT_EQ(kernels[10].code(), 3); + EXPECT_THAT(kernels[10].arguments(), ElementsAreArray({0, 2, 2})); + EXPECT_THAT(kernels[10].last_uses(), ElementsAreArray({true, false, true})); + EXPECT_TRUE(kernels[10].results().empty()); +} + +template +absl::StatusOr DecodeAttribute(absl::string_view data) { + if (data.size() < sizeof(T)) + return absl::InvalidArgumentError("Invalid data size for attribute."); + + T value; + std::memcpy(&value, data.data(), sizeof(T)); + return value; +} + +TEST(MlirToByteCodeTest, BasicAttributes) { + constexpr char kBasicAttributesMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/" + "basic_attributes.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kBasicAttributesMlir), &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + bc::Buffer buffer = + EmitExecutable(attribute_encoder_registry, mlir_module.get()).value(); + + bc::Executable executable(buffer.data()); + + auto attributes = executable.attributes(); + + ASSERT_EQ(attributes.size(), 15); + + auto attr_iter = attributes.begin(); + + EXPECT_EQ(*attr_iter, "test string"); + ++attr_iter; + + EXPECT_EQ(*attr_iter, "ts"); + ++attr_iter; + + EXPECT_THAT(DecodeAttribute(*attr_iter), + absl_testing::IsOkAndHolds(100)); + ++attr_iter; + + EXPECT_THAT(DecodeAttribute(*attr_iter), + absl_testing::IsOkAndHolds(200)); + ++attr_iter; + + EXPECT_THAT(DecodeAttribute(*attr_iter), + absl_testing::IsOkAndHolds(FloatEq(3.0))); + ++attr_iter; + + EXPECT_THAT(DecodeAttribute(*attr_iter), + absl_testing::IsOkAndHolds(0)); + ++attr_iter; + + bc::Vector list_of_i64((*attr_iter).data()); + EXPECT_THAT(list_of_i64, ElementsAreArray({0, 1, 2, 3, 4})); + ++attr_iter; + + bc::Vector list_of_i32((*attr_iter).data()); + EXPECT_THAT(list_of_i32, ElementsAreArray({0, 1, 2, 3})); + ++attr_iter; + + bc::Vector list_of_str((*attr_iter).data()); + EXPECT_THAT(list_of_str, ElementsAreArray({"string 0", "string 1"})); + ++attr_iter; + + EXPECT_THAT(DecodeAttribute(*attr_iter), + absl_testing::IsOkAndHolds(1)); + EXPECT_EQ(executable.functions()[1].name().Get(), "callee"); + ++attr_iter; + + bc::Vector list_of_symbol_ref((*attr_iter).data()); + EXPECT_EQ(executable.functions()[2].name().Get(), "callee0"); + EXPECT_EQ(executable.functions()[3].name().Get(), "callee1"); + EXPECT_THAT(list_of_symbol_ref, ElementsAreArray({2, 3})); + ++attr_iter; + + bc::Vector dense_array_of_i32((*attr_iter).data()); + EXPECT_THAT(dense_array_of_i32, ElementsAreArray({0, 1, 2})); + ++attr_iter; + + bc::Vector dense_array_of_i64((*attr_iter).data()); + EXPECT_THAT(dense_array_of_i64, ElementsAreArray({0, 1, 2})); + ++attr_iter; + + bc::Vector empty_dense_array((*attr_iter).data()); + EXPECT_TRUE(empty_dense_array.empty()); + ++attr_iter; + + bc::Vector dense_array_of_bool((*attr_iter).data()); + EXPECT_THAT(dense_array_of_bool, ElementsAreArray({true, false})); + + auto kernels = executable.functions()[0].kernels(); + ASSERT_EQ(kernels.size(), 16); + auto kernel_iter = kernels.begin(); + + auto attribute_span = [&](auto kernel_iter) { + return mlrt::AttributeSpan((*kernel_iter).attributes(), attributes); + }; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0).Get(), + "test string"); + ++kernel_iter; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0).Get(), "ts"); + ++kernel_iter; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0), 100); + ++kernel_iter; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0), 200); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs(0), FloatEq(3.0)); + ++kernel_iter; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0), false); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({0, 1, 2, 3, 4})); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({0, 1, 2, 3})); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({"string 0", "string 1"})); + ++kernel_iter; + + EXPECT_EQ(attribute_span(kernel_iter).GetAs(0), 1); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({2, 3})); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({0, 1, 2})); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({0, 1, 2})); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + IsEmpty()); + ++kernel_iter; + + EXPECT_THAT(attribute_span(kernel_iter).GetAs>(0), + ElementsAreArray({true, false})); +} + +TEST(MlirToByteCodeTest, UnsupportedAttributes) { + constexpr char kUnsupportedAttributesMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/" + "unsupported_attributes.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kUnsupportedAttributesMlir), + &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + EXPECT_THAT( + EmitExecutable(attribute_encoder_registry, mlir_module.get()), + absl_testing::StatusIs(absl::StatusCode::kInvalidArgument, + "Try to encode unsupported attribute: unit")); +} + +class CustomDense { + public: + struct StorageType { + using Self = StorageType; + DEFINE_BYTECODE_FIELD(bc::Vector, shape); + DEFINE_BYTECODE_FIELD(bc::Vector, data); + }; + + class Constructor { + public: + Constructor(bc::Allocator* allocator, bc::BcAddr_t address) + : allocator_(allocator), address_(address) {} + + template + auto construct_shape(Args&&... args) { + return StorageType::construct_shape(allocator_, address_, + std::forward(args)...); + } + template + auto construct_data(Args&&... args) { + return StorageType::construct_data(allocator_, address_, + std::forward(args)...); + } + + bc::BcAddr_t address() const { return address_; } + + private: + bc::Allocator* allocator_; + bc::BcAddr_t address_; + }; + using NonTrivialConstructorType = Constructor; + + explicit CustomDense(const char* p) : p_(p) {} + + bc::Vector shape() const { return StorageType::read_shape(p_); } + bc::Vector data() const { return StorageType::read_data(p_); } + + private: + const char* p_ = nullptr; +}; + +absl::StatusOr EncodeCustomDense(const ModuleEmitterContext&, + mlir::Attribute attr) { + auto dense_int_attr = mlir::dyn_cast(attr); + if (!dense_int_attr) + return absl::InvalidArgumentError( + "The element of the custom dense attribute must be an integer."); + + if (mlir::cast(dense_int_attr.getElementType()) + .getWidth() != 32) { + return absl::InvalidArgumentError( + "The element of the custom dense attribute must be an i32 integer."); + } + + bc::Buffer buffer; + bc::Allocator allocator(&buffer); + auto custom_dense_ctor = bc::New(&allocator); + + auto shaped_type = dense_int_attr.getType(); + std::vector shape(shaped_type.getShape().begin(), + shaped_type.getShape().end()); + custom_dense_ctor.construct_shape(shape); + + custom_dense_ctor.construct_data(shaped_type.getNumElements()) + .Place(dense_int_attr.getRawData().data(), + dense_int_attr.getRawData().size()); + + return std::string(buffer.data(), buffer.size()); +} + +TEST(MlirToByteCodeTest, CustomDense) { + constexpr char kCustomAttributesMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/" + "custom_attributes.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kCustomAttributesMlir), &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + attribute_encoder_registry.Register("test_custom", &EncodeCustomDense); + bc::Buffer buffer = + EmitExecutable(attribute_encoder_registry, mlir_module.get()).value(); + + bc::Executable executable(buffer.data()); + + auto attributes = executable.attributes(); + + ASSERT_EQ(attributes.size(), 10); + for (int i = 0; i < 10; ++i) { + bc::String attr_data = attributes[i]; + + CustomDense custom_dense(attr_data.data()); + EXPECT_THAT(custom_dense.shape(), ElementsAreArray({1})); + EXPECT_THAT(custom_dense.data(), ElementsAreArray({i})); + } +} + +TEST(MlirToByteCodeTest, AsyncNotFreed) { + constexpr char kAsyncMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/async.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kAsyncMlir), &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + bc::Buffer buffer = + EmitExecutable(attribute_encoder_registry, mlir_module.get()).value(); + + bc::Executable executable(buffer.data()); + + auto kernel_names = executable.kernel_names(); + EXPECT_THAT(kernel_names, + ElementsAreArray({"test_mlbc.add.i32", "return", "mlrt.async", + "mlrt.await_handle"})); + + auto functions = executable.functions(); + ASSERT_EQ(functions.size(), 2); + + auto function = functions[1]; + EXPECT_EQ(function.name().str(), "main"); + EXPECT_EQ(function.num_regs(), 4); + EXPECT_THAT(function.input_regs(), ElementsAreArray({0, 1})); + EXPECT_THAT(function.output_regs(), ElementsAreArray({1})); + EXPECT_THAT(function.output_last_uses(), ElementsAreArray({true})); + + auto kernels = function.kernels(); + ASSERT_EQ(kernels.size(), 5); + + EXPECT_EQ(kernels[0].code(), 2); // mlrt.async + EXPECT_THAT(kernels[0].arguments(), ElementsAreArray({0, 1})); + // The returned handle is in register 2, which is never used by other kernels. + EXPECT_THAT(kernels[0].results(), ElementsAreArray({2})); + EXPECT_THAT(kernels[0].last_uses(), ElementsAreArray({false, false})); + + EXPECT_EQ(kernels[1].code(), 3); // mlrt.await_handle + EXPECT_THAT(kernels[1].arguments(), ElementsAreArray({2})); + EXPECT_THAT(kernels[1].results(), IsEmpty()); + + EXPECT_EQ(kernels[2].code(), 0); // test_mlbc.add.i32 + EXPECT_THAT(kernels[2].arguments(), ElementsAreArray({0, 1})); + EXPECT_THAT(kernels[2].results(), ElementsAreArray({3})); + EXPECT_THAT(kernels[2].last_uses(), ElementsAreArray({true, true})); + + EXPECT_EQ(kernels[3].code(), 0); // test_mlbc.add.i32 + EXPECT_THAT(kernels[3].arguments(), ElementsAreArray({3, 3})); + EXPECT_THAT(kernels[3].results(), ElementsAreArray({1})); + EXPECT_THAT(kernels[3].last_uses(), ElementsAreArray({false, true})); + + EXPECT_EQ(kernels[4].code(), 1); // return + EXPECT_THAT(kernels[4].arguments(), ElementsAreArray({1})); + EXPECT_THAT(kernels[4].results(), IsEmpty()); + EXPECT_THAT(kernels[4].last_uses(), ElementsAreArray({true})); +} + +TEST(MlirToByteCodeTest, AsyncUseNewId) { + constexpr char kAsyncMlir[] = + "tensorflow/compiler/mlir/tfrt/translate/mlrt/testdata/async2.mlir"; + + mlir::DialectRegistry registry; + registry.insert(); + mlir::MLIRContext mlir_context(registry); + mlir_context.allowUnregisteredDialects(); + auto mlir_module = mlir::parseSourceFile( + tsl::GetDataDependencyFilepath(kAsyncMlir), &mlir_context); + + AttributeEncoderRegistry attribute_encoder_registry; + bc::Buffer buffer = + EmitExecutable(attribute_encoder_registry, mlir_module.get()).value(); + + bc::Executable executable(buffer.data()); + + auto kernel_names = executable.kernel_names(); + EXPECT_THAT(kernel_names, + ElementsAreArray({"test_mlbc.add.i32", "return", "mlrt.async", + "mlrt.await_handle"})); + + auto functions = executable.functions(); + ASSERT_EQ(functions.size(), 2); + + auto function = functions[1]; + EXPECT_EQ(function.name().str(), "main"); + EXPECT_EQ(function.num_regs(), 4); + EXPECT_THAT(function.input_regs(), ElementsAreArray({0, 1})); + EXPECT_THAT(function.output_regs(), ElementsAreArray({1})); + EXPECT_THAT(function.output_last_uses(), ElementsAreArray({true})); + + auto kernels = function.kernels(); + ASSERT_EQ(kernels.size(), 5); + + EXPECT_EQ(kernels[0].code(), 0); // test_mlbc.add.i32 + EXPECT_THAT(kernels[0].arguments(), ElementsAreArray({0, 1})); + EXPECT_THAT(kernels[0].results(), ElementsAreArray({2})); + EXPECT_THAT(kernels[0].last_uses(), ElementsAreArray({true, true})); + + EXPECT_EQ(kernels[1].code(), 2); // mlrt.async + EXPECT_THAT(kernels[1].arguments(), ElementsAreArray({2, 2})); + // The returned handle is in register 3, which is never used by other kernels. + EXPECT_THAT(kernels[1].results(), ElementsAreArray({3})); + EXPECT_THAT(kernels[1].last_uses(), ElementsAreArray({false, false})); + + EXPECT_EQ(kernels[2].code(), 3); // mlrt.await_handle + EXPECT_THAT(kernels[2].arguments(), ElementsAreArray({3})); + EXPECT_THAT(kernels[2].results(), IsEmpty()); + EXPECT_THAT(kernels[2].last_uses(), ElementsAreArray({false})); + + EXPECT_EQ(kernels[3].code(), 0); // test_mlbc.add.i32 + EXPECT_THAT(kernels[3].arguments(), ElementsAreArray({2, 2})); + // AsyncHandle does not free its register. So this can only use 1. + EXPECT_THAT(kernels[3].results(), ElementsAreArray({1})); + EXPECT_THAT(kernels[3].last_uses(), ElementsAreArray({false, true})); + + EXPECT_EQ(kernels[4].code(), 1); // return + EXPECT_THAT(kernels[4].arguments(), ElementsAreArray({1})); + EXPECT_THAT(kernels[4].results(), IsEmpty()); + EXPECT_THAT(kernels[4].last_uses(), ElementsAreArray({true})); +} + +} // namespace +} // namespace mlrt